Compile .ds scripts under assets/ to .dsb before packing into dusk.dsk

tools/asset/pack now special-cases .ds files: each one is compiled with
duskscriptcc to a temp .dsb (no .map, nothing reads one yet) and that gets
packed under the same path with the extension swapped - the raw .ds source
never ends up in the archive. Every other asset type is unchanged, packed
byte for byte as before.

CMake passes --duskscriptcc "$<TARGET_FILE:duskscriptcc>" and adds a DEPENDS
on it, but only on the linux/knulli host target where duskscriptcc is
actually built (see its own gating in src/CMakeLists.txt) - a cross-compiled
console target has no host-built compiler to point at, so for now it can
only pack an assets/ with no .ds in it; the packer raises a clear error
rather than silently skipping or mis-packing one if it finds one anyway.

The manifest's cache-skip check also now folds in duskscriptcc's own mtime,
so a compiler rebuild invalidates every .ds's cached result even when none
of their own source files changed.

Verified by hand: a .ds under assets/scripts/ packs as the matching .dsb
(confirmed by unzipping the resulting dusk.dsk), an unrelated rebuild
correctly skips re-packing, touching duskscriptcc correctly forces a
re-pack, and a syntax error in the .ds fails the build loudly with
duskscriptcc's own error message rather than silently succeeding.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

CMakeLists.txt

@@ -142,13 +142,30 @@ endif()
 # asset edit. tools.asset.pack itself decides whether anything actually
 # changed (see its manifest check) and no-ops quickly when nothing did,
 # so this stays cheap on a build where assets are untouched.
+#
+# A .ds script under assets/ is compiled to .dsb by duskscriptcc as part of
+# packing, so dusk.dsk only ever carries bytecode. duskscriptcc is a host
+# tool, only built when target_system is the build host (linux/knulli, see
+# src/CMakeLists.txt) - a cross-compiled console target has no duskscriptcc
+# of its own to depend on or pass down, so it can only pack assets with no
+# .ds in them until something sets up a host-built duskscriptcc for it too.
+if(DUSK_TARGET_SYSTEM STREQUAL "linux" OR DUSK_TARGET_SYSTEM STREQUAL "knulli")
+  set(DUSK_ASSET_PACK_DUSKSCRIPTCC_ARGS --duskscriptcc "$<TARGET_FILE:duskscriptcc>")
+  set(DUSK_ASSET_PACK_DEPENDS duskscriptcc)
+else()
+  set(DUSK_ASSET_PACK_DUSKSCRIPTCC_ARGS "")
+  set(DUSK_ASSET_PACK_DEPENDS "")
+endif()
+
 add_custom_target(DUSK_ASSETS_BUILT ALL
   COMMAND ${CMAKE_COMMAND} -E make_directory "${DUSK_ASSETS_DIR}"
   COMMAND ${Python3_EXECUTABLE} -m tools.asset.pack
     --input "${DUSK_ASSETS_DIR}"
     --output "${DUSK_ASSETS_ZIP}"
+    ${DUSK_ASSET_PACK_DUSKSCRIPTCC_ARGS}
   WORKING_DIRECTORY "${DUSK_ROOT_DIR}"
   BYPRODUCTS "${DUSK_ASSETS_ZIP}"
+  DEPENDS ${DUSK_ASSET_PACK_DEPENDS}
   COMMENT "Packing assets into dusk.dsk"
   VERBATIM
 )

tools/asset/pack/__main__.py

@@ -24,8 +24,14 @@ DSK2 format:
 
 Usage:
   python3 -m tools.asset.pack --input <assets_dir> --output <dusk.dsk>
+    [--duskscriptcc <path to the duskscriptcc executable>]
   Every file under <assets_dir> is added, using its path relative to
-  <assets_dir> as the zip entry name.
+  <assets_dir> as the zip entry name - except a .ds script, which is
+  compiled with duskscriptcc first and added as the .dsb it compiles to
+  (same path, extension swapped), not as its own source. --duskscriptcc is
+  only required when a .ds file is actually found; a platform this project
+  cross-compiles for without a host-built duskscriptcc (see the CMake side)
+  can still pack assets that have none.
 
 Called unconditionally on every CMake build (see the DUSK_ASSETS_BUILT
 target) rather than gated behind file-level build-system dependencies -
@@ -45,6 +51,9 @@ import argparse
 import json
 import os
 import struct
+import subprocess
+import sys
+import tempfile
 import zipfile
 import zlib
 import io
@@ -55,11 +64,42 @@ HEADER_FORMAT = '<4sIII'
 HEADER_SIZE = struct.calcsize(HEADER_FORMAT)
 
 
-def build_zip_blob(root, relative_paths):
+def script_arcname(relative_path):
+  """.ds packs as the .dsb it compiles to, never as its own source."""
+  return relative_path[:-len('.ds')] + '.dsb' if relative_path.endswith('.ds') else relative_path
+
+
+def compile_script(duskscriptcc, full_path, relative_path, temp_dir):
+  # Mirrors relative_path's own directories under temp_dir, so two .ds with
+  # the same file name in different folders don't collide here.
+  output_path = os.path.join(temp_dir, script_arcname(relative_path))
+  os.makedirs(os.path.dirname(output_path), exist_ok=True)
+  result = subprocess.run(
+    [duskscriptcc, full_path, '-o', output_path, '--no-map'],
+    capture_output=True, text=True
+  )
+  if result.returncode != 0:
+    sys.stderr.write(result.stdout)
+    sys.stderr.write(result.stderr)
+    raise SystemExit(f'duskscriptcc failed to compile {relative_path}')
+  with open(output_path, 'rb') as f:
+    return f.read()
+
+
+def build_zip_blob(root, relative_paths, duskscriptcc):
   buf = io.BytesIO()
-  with zipfile.ZipFile(buf, 'w', compression=zipfile.ZIP_STORED) as zf:
+  with tempfile.TemporaryDirectory() as temp_dir, \
+    zipfile.ZipFile(buf, 'w', compression=zipfile.ZIP_STORED) as zf:
     for relative_path in sorted(relative_paths):
-      zf.write(os.path.join(root, relative_path), arcname=relative_path)
+      full_path = os.path.join(root, relative_path)
+      if relative_path.endswith('.ds'):
+        if not duskscriptcc:
+          raise SystemExit(
+            f'{relative_path} needs duskscriptcc to compile, but --duskscriptcc was not given'
+          )
+        zf.writestr(script_arcname(relative_path), compile_script(duskscriptcc, full_path, relative_path, temp_dir))
+      else:
+        zf.write(full_path, arcname=relative_path)
   return buf.getvalue()
 
 
@@ -67,12 +107,15 @@ def manifest_path(output_path):
   return output_path + '.manifest.json'
 
 
-def compute_manifest(input_dir, relative_paths):
+def compute_manifest(input_dir, relative_paths, duskscriptcc):
   entries = {}
   for relative_path in relative_paths:
     st = os.stat(os.path.join(input_dir, relative_path))
     entries[relative_path] = [st.st_mtime_ns, st.st_size]
-  return {'version': VERSION, 'files': entries}
+  # A duskscriptcc rebuild (format/bugfix) has to invalidate every .ds in the
+  # manifest too, even though none of their own mtimes changed.
+  compiler_mtime = os.stat(duskscriptcc).st_mtime_ns if duskscriptcc else None
+  return {'version': VERSION, 'compiler': compiler_mtime, 'files': entries}
 
 
 def load_manifest(path):
@@ -83,7 +126,7 @@ def load_manifest(path):
     return None
 
 
-def pack(input_dir, output_path):
+def pack(input_dir, output_path, duskscriptcc=None):
   relative_paths = []
   for dirpath, _dirnames, filenames in os.walk(input_dir):
     for filename in filenames:
@@ -92,13 +135,13 @@ def pack(input_dir, output_path):
         os.path.relpath(full_path, input_dir).replace(os.sep, '/')
       )
 
-  manifest = compute_manifest(input_dir, relative_paths)
+  manifest = compute_manifest(input_dir, relative_paths, duskscriptcc)
   manifest_file = manifest_path(output_path)
   if os.path.isfile(output_path) and load_manifest(manifest_file) == manifest:
     print(f'{output_path} is up to date, skipping pack')
     return
 
-  blob = build_zip_blob(input_dir, relative_paths)
+  blob = build_zip_blob(input_dir, relative_paths, duskscriptcc)
 
   header = struct.pack(
     HEADER_FORMAT, MAGIC, VERSION, len(blob), zlib.crc32(blob) & 0xFFFFFFFF
@@ -123,9 +166,14 @@ def main():
   )
   parser.add_argument('--input', required=True, help='Assets directory to pack')
   parser.add_argument('--output', required=True, help='Path to write dusk.dsk to')
+  parser.add_argument(
+    '--duskscriptcc', default=None,
+    help='Path to the duskscriptcc executable, to compile .ds assets with. '
+         'Only required when the assets directory actually has a .ds in it.'
+  )
   args = parser.parse_args()
 
-  pack(args.input, args.output)
+  pack(args.input, args.output, args.duskscriptcc)
 
 
 if __name__ == '__main__':