From 7e932e67a2e2ffa440288deaacf645bcf036a646 Mon Sep 17 00:00:00 2001 From: Dominic Masters Date: Wed, 14 Oct 2026 14:21:20 -0500 Subject: [PATCH] Compile .ds scripts under assets/ to .dsb before packing into dusk.dsk tools/asset/pack now special-cases .ds files: each one is compiled with duskscriptcc to a temp .dsb (no .map, nothing reads one yet) and that gets packed under the same path with the extension swapped - the raw .ds source never ends up in the archive. Every other asset type is unchanged, packed byte for byte as before. CMake passes --duskscriptcc "$" and adds a DEPENDS on it, but only on the linux/knulli host target where duskscriptcc is actually built (see its own gating in src/CMakeLists.txt) - a cross-compiled console target has no host-built compiler to point at, so for now it can only pack an assets/ with no .ds in it; the packer raises a clear error rather than silently skipping or mis-packing one if it finds one anyway. The manifest's cache-skip check also now folds in duskscriptcc's own mtime, so a compiler rebuild invalidates every .ds's cached result even when none of their own source files changed. Verified by hand: a .ds under assets/scripts/ packs as the matching .dsb (confirmed by unzipping the resulting dusk.dsk), an unrelated rebuild correctly skips re-packing, touching duskscriptcc correctly forces a re-pack, and a syntax error in the .ds fails the build loudly with duskscriptcc's own error message rather than silently succeeding. Co-Authored-By: Claude Sonnet 5 --- CMakeLists.txt | 17 +++++++++ tools/asset/pack/__main__.py | 68 ++++++++++++++++++++++++++++++------ 2 files changed, 75 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2aae71f4..cb0bc39e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -142,13 +142,30 @@ endif() # asset edit. tools.asset.pack itself decides whether anything actually # changed (see its manifest check) and no-ops quickly when nothing did, # so this stays cheap on a build where assets are untouched. +# +# A .ds script under assets/ is compiled to .dsb by duskscriptcc as part of +# packing, so dusk.dsk only ever carries bytecode. duskscriptcc is a host +# tool, only built when target_system is the build host (linux/knulli, see +# src/CMakeLists.txt) - a cross-compiled console target has no duskscriptcc +# of its own to depend on or pass down, so it can only pack assets with no +# .ds in them until something sets up a host-built duskscriptcc for it too. +if(DUSK_TARGET_SYSTEM STREQUAL "linux" OR DUSK_TARGET_SYSTEM STREQUAL "knulli") + set(DUSK_ASSET_PACK_DUSKSCRIPTCC_ARGS --duskscriptcc "$") + set(DUSK_ASSET_PACK_DEPENDS duskscriptcc) +else() + set(DUSK_ASSET_PACK_DUSKSCRIPTCC_ARGS "") + set(DUSK_ASSET_PACK_DEPENDS "") +endif() + add_custom_target(DUSK_ASSETS_BUILT ALL COMMAND ${CMAKE_COMMAND} -E make_directory "${DUSK_ASSETS_DIR}" COMMAND ${Python3_EXECUTABLE} -m tools.asset.pack --input "${DUSK_ASSETS_DIR}" --output "${DUSK_ASSETS_ZIP}" + ${DUSK_ASSET_PACK_DUSKSCRIPTCC_ARGS} WORKING_DIRECTORY "${DUSK_ROOT_DIR}" BYPRODUCTS "${DUSK_ASSETS_ZIP}" + DEPENDS ${DUSK_ASSET_PACK_DEPENDS} COMMENT "Packing assets into dusk.dsk" VERBATIM ) diff --git a/tools/asset/pack/__main__.py b/tools/asset/pack/__main__.py index 932346c8..b206ef9c 100644 --- a/tools/asset/pack/__main__.py +++ b/tools/asset/pack/__main__.py @@ -24,8 +24,14 @@ DSK2 format: Usage: python3 -m tools.asset.pack --input --output + [--duskscriptcc ] Every file under is added, using its path relative to - as the zip entry name. + as the zip entry name - except a .ds script, which is + compiled with duskscriptcc first and added as the .dsb it compiles to + (same path, extension swapped), not as its own source. --duskscriptcc is + only required when a .ds file is actually found; a platform this project + cross-compiles for without a host-built duskscriptcc (see the CMake side) + can still pack assets that have none. Called unconditionally on every CMake build (see the DUSK_ASSETS_BUILT target) rather than gated behind file-level build-system dependencies - @@ -45,6 +51,9 @@ import argparse import json import os import struct +import subprocess +import sys +import tempfile import zipfile import zlib import io @@ -55,11 +64,42 @@ HEADER_FORMAT = '<4sIII' HEADER_SIZE = struct.calcsize(HEADER_FORMAT) -def build_zip_blob(root, relative_paths): +def script_arcname(relative_path): + """.ds packs as the .dsb it compiles to, never as its own source.""" + return relative_path[:-len('.ds')] + '.dsb' if relative_path.endswith('.ds') else relative_path + + +def compile_script(duskscriptcc, full_path, relative_path, temp_dir): + # Mirrors relative_path's own directories under temp_dir, so two .ds with + # the same file name in different folders don't collide here. + output_path = os.path.join(temp_dir, script_arcname(relative_path)) + os.makedirs(os.path.dirname(output_path), exist_ok=True) + result = subprocess.run( + [duskscriptcc, full_path, '-o', output_path, '--no-map'], + capture_output=True, text=True + ) + if result.returncode != 0: + sys.stderr.write(result.stdout) + sys.stderr.write(result.stderr) + raise SystemExit(f'duskscriptcc failed to compile {relative_path}') + with open(output_path, 'rb') as f: + return f.read() + + +def build_zip_blob(root, relative_paths, duskscriptcc): buf = io.BytesIO() - with zipfile.ZipFile(buf, 'w', compression=zipfile.ZIP_STORED) as zf: + with tempfile.TemporaryDirectory() as temp_dir, \ + zipfile.ZipFile(buf, 'w', compression=zipfile.ZIP_STORED) as zf: for relative_path in sorted(relative_paths): - zf.write(os.path.join(root, relative_path), arcname=relative_path) + full_path = os.path.join(root, relative_path) + if relative_path.endswith('.ds'): + if not duskscriptcc: + raise SystemExit( + f'{relative_path} needs duskscriptcc to compile, but --duskscriptcc was not given' + ) + zf.writestr(script_arcname(relative_path), compile_script(duskscriptcc, full_path, relative_path, temp_dir)) + else: + zf.write(full_path, arcname=relative_path) return buf.getvalue() @@ -67,12 +107,15 @@ def manifest_path(output_path): return output_path + '.manifest.json' -def compute_manifest(input_dir, relative_paths): +def compute_manifest(input_dir, relative_paths, duskscriptcc): entries = {} for relative_path in relative_paths: st = os.stat(os.path.join(input_dir, relative_path)) entries[relative_path] = [st.st_mtime_ns, st.st_size] - return {'version': VERSION, 'files': entries} + # A duskscriptcc rebuild (format/bugfix) has to invalidate every .ds in the + # manifest too, even though none of their own mtimes changed. + compiler_mtime = os.stat(duskscriptcc).st_mtime_ns if duskscriptcc else None + return {'version': VERSION, 'compiler': compiler_mtime, 'files': entries} def load_manifest(path): @@ -83,7 +126,7 @@ def load_manifest(path): return None -def pack(input_dir, output_path): +def pack(input_dir, output_path, duskscriptcc=None): relative_paths = [] for dirpath, _dirnames, filenames in os.walk(input_dir): for filename in filenames: @@ -92,13 +135,13 @@ def pack(input_dir, output_path): os.path.relpath(full_path, input_dir).replace(os.sep, '/') ) - manifest = compute_manifest(input_dir, relative_paths) + manifest = compute_manifest(input_dir, relative_paths, duskscriptcc) manifest_file = manifest_path(output_path) if os.path.isfile(output_path) and load_manifest(manifest_file) == manifest: print(f'{output_path} is up to date, skipping pack') return - blob = build_zip_blob(input_dir, relative_paths) + blob = build_zip_blob(input_dir, relative_paths, duskscriptcc) header = struct.pack( HEADER_FORMAT, MAGIC, VERSION, len(blob), zlib.crc32(blob) & 0xFFFFFFFF @@ -123,9 +166,14 @@ def main(): ) parser.add_argument('--input', required=True, help='Assets directory to pack') parser.add_argument('--output', required=True, help='Path to write dusk.dsk to') + parser.add_argument( + '--duskscriptcc', default=None, + help='Path to the duskscriptcc executable, to compile .ds assets with. ' + 'Only required when the assets directory actually has a .ds in it.' + ) args = parser.parse_args() - pack(args.input, args.output) + pack(args.input, args.output, args.duskscriptcc) if __name__ == '__main__':