Collapse dusk.dsk from two zip archives into one, all uncompressed

Bulk assets (mp3, png) are already compressed at the file level, so a
DEFLATE pass on top bought negligible space for real CPU cost. Packs
everything into a single ZIP_STORED archive instead, so every entry
(not just locale/*) gets the reliable repeated seeking/re-opening that
libzip only supports for uncompressed entries. Header shrinks from 32
to 16 bytes; version bumped to 2 so a stale pre-change dusk.dsk is
rejected and rebuilt rather than misread.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

tools/asset/pack/__main__.py

@@ -6,35 +6,26 @@
 """
 Packs the assets directory into a dusk.dsk (DSK2) archive.
 
-Replaces a single plain zip with two independent zip archives back to
-back, so specific files can be stored uncompressed instead of DEFLATEd -
-useful for small files (locale strings, config) accessed at runtime via
-repeated seeks/re-opens, which is only reliably supported by libzip for
-uncompressed (STORED) entries. Reading a whole compressed archive into
-memory just to get reliable access isn't viable once that archive holds
-the bulk of a game's binary assets, so the two archives are kept
-separate and only the (small, by convention) stored one is expected to
-ever need to be buffered whole.
+A small fixed header (magic/version/size/checksum) followed by a single
+uncompressed (ZIP_STORED) zip archive of every asset. Uncompressed rather
+than DEFLATEd because the bulk of what's packed (audio, images) is
+already compressed at the file level, so a second, zip-level DEFLATE
+pass buys negligible space back for real CPU cost, and it lets every
+entry - not just a hand-picked subset - support the reliable repeated
+seeking/re-opening (e.g. locale line-by-line reads, MP3 loop restarts)
+that libzip only supports for uncompressed entries.
 
 DSK2 format:
-  Bytes 0-3:   "DSK2" (magic)
-  Bytes 4-7:   uint32_t version = 1 (little-endian)
-  Bytes 8-11:  uint32_t compressedOffset
-  Bytes 12-15: uint32_t compressedSize
-  Bytes 16-19: uint32_t compressedChecksum (CRC32 of the compressed blob)
-  Bytes 20-23: uint32_t storedOffset
-  Bytes 24-27: uint32_t storedSize
-  Bytes 28-31: uint32_t storedChecksum (CRC32 of the stored blob)
-  Bytes 32+:   compressed zip blob, then stored zip blob (each a
-               complete, independently valid zip archive)
+  Bytes 0-3:  "DSK2" (magic)
+  Bytes 4-7:  uint32_t version = 2 (little-endian)
+  Bytes 8-11: uint32_t size (byte length of the zip archive that follows)
+  Bytes 12-15: uint32_t checksum (CRC32 of the zip archive that follows)
+  Bytes 16+:  the zip archive itself
 
 Usage:
   python3 -m tools.asset.pack --input <assets_dir> --output <dusk.dsk>
-    [--stored <glob-pattern>]...
   Every file under <assets_dir> is added, using its path relative to
-  <assets_dir> as the zip entry name, to the stored archive if its
-  relative path matches any --stored pattern (fnmatch, default:
-  "locale/*"), otherwise to the compressed archive.
+  <assets_dir> as the zip entry name.
 
 Called unconditionally on every CMake build (see the DUSK_ASSETS_BUILT
 target) rather than gated behind file-level build-system dependencies -
@@ -44,7 +35,10 @@ existing files or lag a build behind on adds/removes. Instead this script
 itself decides whether anything actually needs repacking: it writes a
 manifest of every input file's (mtime, size) next to the output on each
 successful pack, and skips the real work if a fresh manifest compares
-equal to it and the output still exists.
+equal to it and the output still exists. The manifest also carries the
+format version, so a manifest written by an older version of this script
+(a stale single/dual-archive dusk.dsk sitting from before a format
+change) never compares equal and always triggers a fresh pack.
 """
 
 import argparse
@@ -53,19 +47,17 @@ import os
 import struct
 import zipfile
 import zlib
-import fnmatch
 import io
 
 MAGIC = b'DSK2'
-VERSION = 1
-HEADER_FORMAT = '<4sIIIIIII'
+VERSION = 2
+HEADER_FORMAT = '<4sIII'
 HEADER_SIZE = struct.calcsize(HEADER_FORMAT)
-DEFAULT_STORED_PATTERNS = ['locale/*']
 
 
-def build_zip_blob(root, relative_paths, compression):
+def build_zip_blob(root, relative_paths):
   buf = io.BytesIO()
-  with zipfile.ZipFile(buf, 'w', compression=compression) as zf:
+  with zipfile.ZipFile(buf, 'w', compression=zipfile.ZIP_STORED) as zf:
     for relative_path in sorted(relative_paths):
       zf.write(os.path.join(root, relative_path), arcname=relative_path)
   return buf.getvalue()
@@ -75,12 +67,12 @@ def manifest_path(output_path):
   return output_path + '.manifest.json'
 
 
-def compute_manifest(input_dir, relative_paths, stored_patterns):
+def compute_manifest(input_dir, relative_paths):
   entries = {}
   for relative_path in relative_paths:
     st = os.stat(os.path.join(input_dir, relative_path))
     entries[relative_path] = [st.st_mtime_ns, st.st_size]
-  return {'stored_patterns': sorted(stored_patterns), 'files': entries}
+  return {'version': VERSION, 'files': entries}
 
 
 def load_manifest(path):
@@ -91,7 +83,7 @@ def load_manifest(path):
     return None
 
 
-def pack(input_dir, output_path, stored_patterns):
+def pack(input_dir, output_path):
   relative_paths = []
   for dirpath, _dirnames, filenames in os.walk(input_dir):
     for filename in filenames:
@@ -100,52 +92,28 @@ def pack(input_dir, output_path, stored_patterns):
         os.path.relpath(full_path, input_dir).replace(os.sep, '/')
       )
 
-  manifest = compute_manifest(input_dir, relative_paths, stored_patterns)
+  manifest = compute_manifest(input_dir, relative_paths)
   manifest_file = manifest_path(output_path)
   if os.path.isfile(output_path) and load_manifest(manifest_file) == manifest:
     print(f'{output_path} is up to date, skipping pack')
     return
 
-  stored_paths = [
-    path for path in relative_paths
-    if any(fnmatch.fnmatch(path, pattern) for pattern in stored_patterns)
-  ]
-  compressed_paths = [
-    path for path in relative_paths if path not in stored_paths
-  ]
-
-  compressed_blob = build_zip_blob(
-    input_dir, compressed_paths, zipfile.ZIP_DEFLATED
-  )
-  stored_blob = build_zip_blob(input_dir, stored_paths, zipfile.ZIP_STORED)
-
-  compressed_offset = HEADER_SIZE
-  stored_offset = compressed_offset + len(compressed_blob)
+  blob = build_zip_blob(input_dir, relative_paths)
 
   header = struct.pack(
-    HEADER_FORMAT,
-    MAGIC,
-    VERSION,
-    compressed_offset,
-    len(compressed_blob),
-    zlib.crc32(compressed_blob) & 0xFFFFFFFF,
-    stored_offset,
-    len(stored_blob),
-    zlib.crc32(stored_blob) & 0xFFFFFFFF,
+    HEADER_FORMAT, MAGIC, VERSION, len(blob), zlib.crc32(blob) & 0xFFFFFFFF
   )
 
   with open(output_path, 'wb') as f:
     f.write(header)
-    f.write(compressed_blob)
-    f.write(stored_blob)
+    f.write(blob)
 
   with open(manifest_file, 'w', encoding='utf-8') as f:
     json.dump(manifest, f)
 
   print(
-    f'Wrote {output_path}: {len(compressed_paths)} compressed file(s) '
-    f'({len(compressed_blob)} bytes), {len(stored_paths)} stored file(s) '
-    f'({len(stored_blob)} bytes), {len(header) + len(compressed_blob) + len(stored_blob)} bytes total'
+    f'Wrote {output_path}: {len(relative_paths)} file(s) '
+    f'({len(blob)} bytes), {len(header) + len(blob)} bytes total'
   )
 
 
@@ -155,17 +123,9 @@ def main():
   )
   parser.add_argument('--input', required=True, help='Assets directory to pack')
   parser.add_argument('--output', required=True, help='Path to write dusk.dsk to')
-  parser.add_argument(
-    '--stored',
-    action='append',
-    dest='stored_patterns',
-    help='fnmatch pattern (relative to --input) of files to store '
-         'uncompressed instead of compressing. May be given multiple '
-         'times. Defaults to "locale/*" if never given.'
-  )
   args = parser.parse_args()
 
-  pack(args.input, args.output, args.stored_patterns or DEFAULT_STORED_PATTERNS)
+  pack(args.input, args.output)
 
 
 if __name__ == '__main__':