Collapse dusk.dsk from two zip archives into one, all uncompressed

Bulk assets (mp3, png) are already compressed at the file level, so a
DEFLATE pass on top bought negligible space for real CPU cost. Packs
everything into a single ZIP_STORED archive instead, so every entry
(not just locale/*) gets the reliable repeated seeking/re-opening that
libzip only supports for uncompressed entries. Header shrinks from 32
to 16 bytes; version bumped to 2 so a stale pre-change dusk.dsk is
rejected and rebuilt rather than misread.

Co-Authored-By: Claude Sonnet 5 <[email protected]>

src/dusk/asset/asset.c

@@ -25,11 +25,9 @@ errorret_t assetInit(void) {
     threadMutexInit(&ASSET.loading[i].mutex);
   }
 
-  // assetInitPlatform must either define both ASSET.zip/ASSET.zipStored or
-  // throw an error.
+  // assetInitPlatform must either define ASSET.zip or throw an error.
   errorChain(assetInitPlatform());
   assertNotNull(ASSET.zip, "Asset zip null without error.");
-  assertNotNull(ASSET.zipStored, "Asset stored zip null without error.");
   threadInit(&ASSET.loadThread, assetUpdateAsync);
   threadStart(&ASSET.loadThread);
 
@@ -40,9 +38,7 @@ bool_t assetFileExists(const char_t *filename) {
   assertStrLenMax(filename, ASSET_FILE_NAME_MAX, "Filename too long.");
 
   threadMutexLock(&ASSET.zipLock);
-  bool_t found =
-    zip_name_locate(ASSET.zip, filename, 0) >= 0 ||
-    zip_name_locate(ASSET.zipStored, filename, 0) >= 0;
+  bool_t found = zip_name_locate(ASSET.zip, filename, 0) >= 0;
   threadMutexUnlock(&ASSET.zipLock);
   return found;
 }
@@ -445,19 +441,13 @@ errorret_t assetDispose(void) {
 
   errorChain(assetReapUnused());
 
-  // Cleanup zip files.
+  // Cleanup zip file.
   if(ASSET.zip != NULL) {
     if(zip_close(ASSET.zip) != 0) {
-      errorThrow("Failed to close compressed asset zip archive.");
+      errorThrow("Failed to close asset zip archive.");
     }
     ASSET.zip = NULL;
   }
-  if(ASSET.zipStored != NULL) {
-    if(zip_close(ASSET.zipStored) != 0) {
-      errorThrow("Failed to close stored asset zip archive.");
-    }
-    ASSET.zipStored = NULL;
-  }
 
   errorChain(assetDisposePlatform());
   threadMutexDispose(&ASSET.zipLock);

src/dusk/asset/asset.h

@@ -29,16 +29,12 @@
 #define ASSET_BASE_DIRECTORY_MAX 256
 
 typedef struct asset_s {
-  // Compressed (DEFLATE) archive - expected to hold the bulk of a game's
-  // binary assets. Looked up first by assetFileInit().
+  // The dusk.dsk archive - a single uncompressed (ZIP_STORED) zip, so
+  // every entry gets the reliable repeated seeking/re-opening that
+  // libzip only supports reliably for uncompressed entries (locale
+  // strings, MP3 loop restarts, etc.).
   zip_t *zip;
 
-  // Stored (uncompressed) archive - expected to hold small files that need
-  // reliable repeated seeking/re-opening (e.g. locale strings), which
-  // libzip only supports reliably for uncompressed entries. Looked up by
-  // assetFileInit() only if the name isn't found in `zip`.
-  zip_t *zipStored;
-
   // Directory dusk.dsk was actually found/opened in (on PSP, the directory
   // EBOOT.PBP lives in, since dusk.dsk is embedded inside it rather than a
   // loose file) - populated by each platform's own assetInitXxx() once it
@@ -52,7 +48,7 @@ typedef struct asset_s {
 
   assetplatform_t platform;
 
-  // Guards every libzip call against ASSET.zip/ASSET.zipStored (zip_fopen,
+  // Guards every libzip call against ASSET.zip (zip_fopen,
   // zip_fread, zip_fclose, zip_fseek, zip_stat, zip_name_locate - see
   // assetfile.c/assetFileExists). libzip is documented as not thread-safe,
   // and this asset system genuinely calls it from multiple real threads at

src/dusk/asset/assetdsk.c

@@ -30,9 +30,9 @@ errorret_t assetDskParseHeader(
   // Every field is a little-endian uint32_t regardless of host - convert
   // to host order (a no-op on little-endian hosts, a real byteswap on
   // Dolphin's big-endian PowerPC).
-  uint32_t fields[7];
+  uint32_t fields[3];
   memoryCopy(fields, bytes + ASSET_DSK_MAGIC_SIZE, sizeof(fields));
-  for(uint8_t i = 0; i < 7; i++) {
+  for(uint8_t i = 0; i < 3; i++) {
     fields[i] = endianLittleToHost32(fields[i]);
   }
 
@@ -41,24 +41,17 @@ errorret_t assetDskParseHeader(
     errorThrow("dusk.dsk has an unsupported version: %u", version);
   }
 
-  outHeader->compressedOffset = fields[1];
-  outHeader->compressedSize = fields[2];
-  outHeader->compressedChecksum = fields[3];
-  outHeader->storedOffset = fields[4];
-  outHeader->storedSize = fields[5];
-  outHeader->storedChecksum = fields[6];
+  outHeader->size = fields[1];
+  outHeader->checksum = fields[2];
 
   errorOk();
 }
 
 errorret_t assetDskOpenFromPath(
   const char_t *path,
-  zip_t **outCompressed,
-  zip_t **outStored
+  zip_t **outZip
 ) {
-  errorChain(assetDskOpenFromPathRange(
-    path, 0, SIZE_MAX, outCompressed, outStored
-  ));
+  errorChain(assetDskOpenFromPathRange(path, 0, SIZE_MAX, outZip));
   errorOk();
 }
 
@@ -66,15 +59,12 @@ errorret_t assetDskOpenFromPathRange(
   const char_t *path,
   const size_t baseOffset,
   const size_t baseSize,
-  zip_t **outCompressed,
-  zip_t **outStored
+  zip_t **outZip
 ) {
   assertNotNull(path, "Path cannot be NULL.");
-  assertNotNull(outCompressed, "Out compressed cannot be NULL.");
-  assertNotNull(outStored, "Out stored cannot be NULL.");
+  assertNotNull(outZip, "Out zip cannot be NULL.");
 
-  *outCompressed = NULL;
-  *outStored = NULL;
+  *outZip = NULL;
 
   FILE *headerFile = fopen(path, "rb");
   if(headerFile == NULL) {
@@ -96,104 +86,30 @@ errorret_t assetDskOpenFromPathRange(
   assetdskheader_t header;
   errorChain(assetDskParseHeader(headerBytes, sizeof(headerBytes), &header));
 
-  if(
-    (size_t) header.compressedOffset + header.compressedSize > baseSize ||
-    (size_t) header.storedOffset + header.storedSize > baseSize
-  ) {
-    errorThrow("dusk.dsk header describes ranges beyond its containing file.");
+  if((size_t) ASSET_DSK_HEADER_SIZE + header.size > baseSize) {
+    errorThrow("dusk.dsk header describes a range beyond its containing file.");
   }
 
   zip_error_t zipError;
   zip_error_init(&zipError);
 
-  zip_source_t *compressedSource = zip_source_file_create(
+  zip_source_t *source = zip_source_file_create(
     path,
-    (zip_uint64_t) (baseOffset + header.compressedOffset),
-    (zip_int64_t) header.compressedSize,
+    (zip_uint64_t) (baseOffset + ASSET_DSK_HEADER_SIZE),
+    (zip_int64_t) header.size,
     &zipError
   );
-  if(compressedSource == NULL) {
+  if(source == NULL) {
     errorThrow(
-      "Failed to create compressed dusk.dsk source: %s",
-      zip_error_strerror(&zipError)
+      "Failed to create dusk.dsk source: %s", zip_error_strerror(&zipError)
     );
   }
 
-  *outCompressed =
-    zip_open_from_source(compressedSource, ZIP_RDONLY, &zipError);
-  if(*outCompressed == NULL) {
-    zip_source_free(compressedSource);
+  *outZip = zip_open_from_source(source, ZIP_RDONLY, &zipError);
+  if(*outZip == NULL) {
+    zip_source_free(source);
     errorThrow(
-      "Failed to open compressed dusk.dsk archive: %s",
-      zip_error_strerror(&zipError)
-    );
-  }
-
-  zip_source_t *storedSource = zip_source_file_create(
-    path,
-    (zip_uint64_t) (baseOffset + header.storedOffset),
-    (zip_int64_t) header.storedSize,
-    &zipError
-  );
-  if(storedSource == NULL) {
-    zip_close(*outCompressed);
-    *outCompressed = NULL;
-    errorThrow(
-      "Failed to create stored dusk.dsk source: %s",
-      zip_error_strerror(&zipError)
-    );
-  }
-
-  *outStored = zip_open_from_source(storedSource, ZIP_RDONLY, &zipError);
-  if(*outStored == NULL) {
-    zip_source_free(storedSource);
-    zip_close(*outCompressed);
-    *outCompressed = NULL;
-    errorThrow(
-      "Failed to open stored dusk.dsk archive: %s",
-      zip_error_strerror(&zipError)
-    );
-  }
-
-  // The stored archive is small by convention, so verifying its checksum
-  // here (one extra small read) is cheap; the compressed archive isn't
-  // checked since it's meant to be read lazily/on-demand from here on.
-  uint8_t *storedBytes = (uint8_t *) memoryAllocate(header.storedSize);
-  FILE *storedFile = fopen(path, "rb");
-  if(storedFile == NULL) {
-    memoryFree(storedBytes);
-    zip_close(*outStored);
-    zip_close(*outCompressed);
-    *outStored = NULL;
-    *outCompressed = NULL;
-    errorThrow(
-      "Failed to re-open dusk.dsk to verify stored checksum: %s", path
-    );
-  }
-  fseek(storedFile, (long) (baseOffset + header.storedOffset), SEEK_SET);
-  size_t storedRead = fread(storedBytes, 1, header.storedSize, storedFile);
-  fclose(storedFile);
-  if(storedRead != header.storedSize) {
-    memoryFree(storedBytes);
-    zip_close(*outStored);
-    zip_close(*outCompressed);
-    *outStored = NULL;
-    *outCompressed = NULL;
-    errorThrow(
-      "Failed to read dusk.dsk stored archive to verify checksum: %s", path
-    );
-  }
-
-  uint32_t checksum =
-    (uint32_t) crc32(0L, storedBytes, (uInt) header.storedSize);
-  memoryFree(storedBytes);
-  if(checksum != header.storedChecksum) {
-    zip_close(*outStored);
-    zip_close(*outCompressed);
-    *outStored = NULL;
-    *outCompressed = NULL;
-    errorThrow(
-      "dusk.dsk stored archive failed checksum verification: %s", path
+      "Failed to open dusk.dsk archive: %s", zip_error_strerror(&zipError)
     );
   }
 
@@ -203,85 +119,45 @@ errorret_t assetDskOpenFromPathRange(
 errorret_t assetDskOpenFromBuffer(
   uint8_t *buffer,
   const size_t bufferSize,
-  zip_t **outCompressed,
-  zip_t **outStored
+  zip_t **outZip
 ) {
   assertNotNull(buffer, "Buffer cannot be NULL.");
-  assertNotNull(outCompressed, "Out compressed cannot be NULL.");
-  assertNotNull(outStored, "Out stored cannot be NULL.");
+  assertNotNull(outZip, "Out zip cannot be NULL.");
 
-  *outCompressed = NULL;
-  *outStored = NULL;
+  *outZip = NULL;
 
   assetdskheader_t header;
   errorChain(assetDskParseHeader(buffer, bufferSize, &header));
 
-  if(
-    (size_t) header.compressedOffset + header.compressedSize > bufferSize ||
-    (size_t) header.storedOffset + header.storedSize > bufferSize
-  ) {
-    errorThrow("dusk.dsk header describes ranges beyond the buffer.");
+  if((size_t) ASSET_DSK_HEADER_SIZE + header.size > bufferSize) {
+    errorThrow("dusk.dsk header describes a range beyond the buffer.");
   }
 
-  uint32_t compressedChecksum = (uint32_t) crc32(
-    0L, buffer + header.compressedOffset, (uInt) header.compressedSize
-  );
-  if(compressedChecksum != header.compressedChecksum) {
-    errorThrow("dusk.dsk compressed archive failed checksum verification.");
-  }
-
-  uint32_t storedChecksum = (uint32_t) crc32(
-    0L, buffer + header.storedOffset, (uInt) header.storedSize
-  );
-  if(storedChecksum != header.storedChecksum) {
-    errorThrow("dusk.dsk stored archive failed checksum verification.");
+  uint8_t *data = buffer + ASSET_DSK_HEADER_SIZE;
+  uint32_t checksum = (uint32_t) crc32(0L, data, (uInt) header.size);
+  if(checksum != header.checksum) {
+    errorThrow("dusk.dsk archive failed checksum verification.");
   }
 
   zip_error_t zipError;
   zip_error_init(&zipError);
 
-  // freep=0 for both - they're non-owning windows into the same caller-owned
-  // buffer, not independent allocations libzip should free.
-  zip_source_t *compressedSource = zip_source_buffer_create(
-    buffer + header.compressedOffset, header.compressedSize, 0, &zipError
+  // freep=0 - this is a non-owning window into the same caller-owned
+  // buffer, not an independent allocation libzip should free.
+  zip_source_t *source = zip_source_buffer_create(
+    data, header.size, 0, &zipError
   );
-  if(compressedSource == NULL) {
+  if(source == NULL) {
     errorThrow(
-      "Failed to create compressed dusk.dsk source: %s",
-      zip_error_strerror(&zipError)
+      "Failed to create dusk.dsk source: %s", zip_error_strerror(&zipError)
     );
   }
 
-  *outCompressed =
-    zip_open_from_source(compressedSource, ZIP_RDONLY, &zipError);
-  if(*outCompressed == NULL) {
-    zip_source_free(compressedSource);
+  *outZip = zip_open_from_source(source, ZIP_RDONLY, &zipError);
+  if(*outZip == NULL) {
+    zip_source_free(source);
     errorThrow(
-      "Failed to open compressed dusk.dsk archive: %s",
-      zip_error_strerror(&zipError)
-    );
-  }
-
-  zip_source_t *storedSource = zip_source_buffer_create(
-    buffer + header.storedOffset, header.storedSize, 0, &zipError
-  );
-  if(storedSource == NULL) {
-    zip_close(*outCompressed);
-    *outCompressed = NULL;
-    errorThrow(
-      "Failed to create stored dusk.dsk source: %s",
-      zip_error_strerror(&zipError)
-    );
-  }
-
-  *outStored = zip_open_from_source(storedSource, ZIP_RDONLY, &zipError);
-  if(*outStored == NULL) {
-    zip_source_free(storedSource);
-    zip_close(*outCompressed);
-    *outCompressed = NULL;
-    errorThrow(
-      "Failed to open stored dusk.dsk archive: %s",
-      zip_error_strerror(&zipError)
+      "Failed to open dusk.dsk archive: %s", zip_error_strerror(&zipError)
     );
   }
 

src/dusk/asset/assetdsk.h

@@ -13,30 +13,26 @@
 // never gets misread as a valid dusk.dsk.
 #define ASSET_DSK_MAGIC_SIZE 4
 #define ASSET_DSK_MAGIC "DSK2"
-#define ASSET_DSK_VERSION 1
+#define ASSET_DSK_VERSION 2
 
-// magic(4) + version(4) + compressedOffset(4) + compressedSize(4) +
-// compressedChecksum(4) + storedOffset(4) + storedSize(4) +
-// storedChecksum(4), all little-endian regardless of host - see
-// assetDskParseHeader.
-#define ASSET_DSK_HEADER_SIZE 32
+// magic(4) + version(4) + size(4) + checksum(4), all little-endian
+// regardless of host - see assetDskParseHeader. The zip archive itself
+// starts immediately after, at ASSET_DSK_HEADER_SIZE.
+#define ASSET_DSK_HEADER_SIZE 16
 
 /**
- * Parsed dusk.dsk (DSK2 format) header. dusk.dsk is two independent, back
- * to back zip archives (see tools/asset/pack) rather than a single plain
- * zip: a "compressed" one (DEFLATE, expected to hold the bulk of a game's
- * binary assets, opened lazily/on-demand) and a "stored" one (uncompressed,
- * expected to hold small files - locale strings, config - that need
- * reliable repeated seeking/re-opening, which libzip only supports for
- * uncompressed entries).
+ * Parsed dusk.dsk (DSK2 format) header. dusk.dsk is a single uncompressed
+ * (ZIP_STORED) zip archive (see tools/asset/pack) rather than DEFLATEd -
+ * the bulk of what's packed (audio, images) is already compressed at the
+ * file level, so DEFLATE-ing the whole archive on top buys negligible
+ * space for real CPU cost, and every entry gets the reliable repeated
+ * seeking/re-opening that libzip only supports for uncompressed entries
+ * (locale line-by-line reads, MP3 loop restarts, etc.) instead of just a
+ * hand-picked subset of files.
  */
 typedef struct {
-  uint32_t compressedOffset;
-  uint32_t compressedSize;
-  uint32_t compressedChecksum;
-  uint32_t storedOffset;
-  uint32_t storedSize;
-  uint32_t storedChecksum;
+  uint32_t size;
+  uint32_t checksum;
 } assetdskheader_t;
 
 /**
@@ -56,27 +52,24 @@ errorret_t assetDskParseHeader(
 );
 
 /**
- * Opens both archives of a dusk.dsk file given a filesystem path, using
- * lazy/windowed file-backed zip sources - no more memory used than the
+ * Opens the dusk.dsk archive given a filesystem path, using a lazy/
+ * windowed file-backed zip source - no more memory used than the
  * existing per-platform small read buffers, matching the memory
- * characteristics of a plain zip_open() on the whole file. Verifies the
- * (small, by convention) stored archive's checksum; the compressed
- * archive's checksum is intentionally not verified here since doing so
- * would require reading the bulk of the game's assets just to compute it.
+ * characteristics of a plain zip_open() on the whole file. The checksum
+ * is intentionally not verified here since doing so would require
+ * reading the whole (potentially large) archive just to compute it.
  *
  * See assetDskOpenFromPathRange() for the case where the DSK2 blob isn't
  * the whole file (e.g. embedded inside another container).
  *
  * @param path Filesystem path to the dusk.dsk file.
- * @param outCompressed Set to the opened compressed archive on success.
- * @param outStored Set to the opened stored archive on success.
+ * @param outZip Set to the opened archive on success.
  * @return OK on success, error if the file is missing, too short, has a
- *         bad header, or either archive fails to open/verify.
+ *         bad header, or the archive fails to open.
  */
 errorret_t assetDskOpenFromPath(
   const char_t *path,
-  zip_t **outCompressed,
-  zip_t **outStored
+  zip_t **outZip
 );
 
 /**
@@ -90,39 +83,35 @@ errorret_t assetDskOpenFromPath(
  * @param path Filesystem path to the container file.
  * @param baseOffset Byte offset within `path` where the DSK2 blob starts.
  * @param baseSize Number of bytes available at `baseOffset`.
- * @param outCompressed Set to the opened compressed archive on success.
- * @param outStored Set to the opened stored archive on success.
+ * @param outZip Set to the opened archive on success.
  * @return OK on success, error if the file is missing, too short, has a
- *         bad header, or either archive fails to open/verify.
+ *         bad header, or the archive fails to open.
  */
 errorret_t assetDskOpenFromPathRange(
   const char_t *path,
   const size_t baseOffset,
   const size_t baseSize,
-  zip_t **outCompressed,
-  zip_t **outStored
+  zip_t **outZip
 );
 
 /**
- * Opens both archives of a dusk.dsk file already fully resident in memory
- * (e.g. a PSAR embedded in an EBOOT.PBP, or an ISO-embedded file already
- * read via DVD_ReadAbs - platforms that already buffer the whole file for
- * reasons unrelated to this format). Neither archive takes ownership of
- * `buffer` (both are opened as non-owning sub-ranges of it) - the caller
+ * Opens the dusk.dsk archive already fully resident in memory (e.g. a
+ * PSAR embedded in an EBOOT.PBP, or an ISO-embedded file already read via
+ * DVD_ReadAbs - platforms that already buffer the whole file for reasons
+ * unrelated to this format). The returned archive doesn't take ownership
+ * of `buffer` (it's opened as a non-owning sub-range of it) - the caller
  * remains responsible for freeing it, and must keep it alive for as long
- * as either archive stays open. Verifies both archives' checksums, since
- * the bytes are already resident.
+ * as the archive stays open. Verifies the checksum, since the bytes are
+ * already resident.
  *
  * @param buffer The whole dusk.dsk file's bytes.
  * @param bufferSize Number of bytes at `buffer`.
- * @param outCompressed Set to the opened compressed archive on success.
- * @param outStored Set to the opened stored archive on success.
+ * @param outZip Set to the opened archive on success.
  * @return OK on success, error if too short, has a bad header/checksum, or
- *         either archive fails to open.
+ *         the archive fails to open.
  */
 errorret_t assetDskOpenFromBuffer(
   uint8_t *buffer,
   const size_t bufferSize,
-  zip_t **outCompressed,
-  zip_t **outStored
+  zip_t **outZip
 );

src/dusk/asset/assetfile.c

@@ -24,20 +24,11 @@ errorret_t assetFileInit(
   file->params = params;
   file->output = output;
 
-  // Stat the file, trying the compressed archive first and falling back to
-  // the stored one - remember which matched so assetFileOpen opens it from
-  // the right archive.
   zip_stat_init(&file->stat);
   threadMutexLock(&ASSET.zipLock);
-  if(zip_stat(ASSET.zip, filename, 0, &file->stat) == 0) {
-    file->sourceZip = ASSET.zip;
-  } else if(zip_stat(ASSET.zipStored, filename, 0, &file->stat) == 0) {
-    file->sourceZip = ASSET.zipStored;
-  } else {
-    file->sourceZip = NULL;
-  }
+  int statResult = zip_stat(ASSET.zip, filename, 0, &file->stat);
   threadMutexUnlock(&ASSET.zipLock);
-  if(file->sourceZip == NULL) {
+  if(statResult != 0) {
     errorThrow("Failed to stat asset file: %s", filename);
   }
 
@@ -85,11 +76,10 @@ errorret_t assetFileRewind(assetfile_t *file) {
 errorret_t assetFileOpen(assetfile_t *file) {
   assertNotNull(file, "Asset file cannot be NULL.");
   assertNotNull(file->filename, "Asset file filename cannot be NULL.");
-  assertNotNull(file->sourceZip, "Asset file must be inited before opening.");
   assertNull(file->zipFile, "Asset file already open.");
 
   threadMutexLock(&ASSET.zipLock);
-  file->zipFile = zip_fopen(file->sourceZip, file->filename, 0);
+  file->zipFile = zip_fopen(ASSET.zip, file->filename, 0);
   threadMutexUnlock(&ASSET.zipLock);
   if(file->zipFile == NULL) {
     errorThrow("Failed to open asset file: %s", file->filename);

src/dusk/asset/assetfile.h

@@ -32,11 +32,6 @@ typedef struct assetfile_s {
   zip_int64_t position;
   zip_int64_t lastRead;
   zip_file_t *zipFile;
-
-  // The archive this file was found in (ASSET.zip or ASSET.zipStored),
-  // set by assetFileInit and used by assetFileOpen so lookups fall back
-  // correctly between the two dusk.dsk archives.
-  zip_t *sourceZip;
 } assetfile_t;
 
 /**

src/dusk/audio/stream/audiostreampcm.c

@@ -87,10 +87,10 @@ errorret_t audioStreamPcmSeek(audiostream_t *stream, const size_t frame) {
 
   const size_t currentByte = (size_t) stream->pcm.file.position;
   if(targetByte < currentByte) {
-    // Only a full rewind can move a read cursor earlier once it's already
-    // advanced past a point - a compressed archive entry can't be decoded
-    // backward. See this function's own doc comment for the performance
-    // implications of a deep loopTo.
+    // A rewind is needed to move the read cursor earlier once it's
+    // already advanced past a point - assetFileRewind() takes the cheap
+    // in-place seek path since every dusk.dsk entry is stored
+    // uncompressed.
     errorChain(assetFileRewind(&stream->pcm.file));
     errorChain(assetFileRead(&stream->pcm.file, NULL, targetByte));
   } else if(targetByte > currentByte) {

src/dusk/audio/stream/audiostreampcm.h

@@ -63,11 +63,8 @@ size_t audioStreamPcmGetTotalFrames(const audiostream_t *stream);
  * (relative to the start of the PCM data - the same units as
  * audiostream_t's startFrame/loopStart/loopTo, once converted from
  * seconds). Seeking backward re-reads from the start of the underlying
- * asset file (see assetFileRewind()) since a compressed archive entry can
- * only be decoded forward - this makes a loop with a deep loopTo more
- * expensive to restart than one near the start, which is a real
- * performance caveat, not just a theoretical one, for anything backed by
- * a compressed (not stored) asset archive entry.
+ * asset file (see assetFileRewind()), which is a cheap in-place seek
+ * since every dusk.dsk entry is stored uncompressed.
  *
  * @param stream The audio stream to seek. Must be AUDIO_STREAM_TYPE_PCM.
  * @param frame Frame offset to seek to, relative to the start of the PCM

src/duskdolphin/asset/assetdolphindvd.c

@@ -91,9 +91,7 @@ errorret_t assetInitDolphinDVD(void) {
   );
   if(!data) errorThrow("Failed to read asset file from ISO.");
 
-  errorret_t ret = assetDskOpenFromBuffer(
-    data, fileSize, &ASSET.zip, &ASSET.zipStored
-  );
+  errorret_t ret = assetDskOpenFromBuffer(data, fileSize, &ASSET.zip);
   if(errorIsNotOk(ret)) {
     memoryFree(data);
     errorChain(ret);

src/duskdolphin/asset/assetdolphindvd.h

@@ -25,7 +25,7 @@
 
 typedef struct {
   // Whole dusk.dsk blob read from the ISO, kept alive for as long as
-  // ASSET.zip/ASSET.zipStored are open since they're non-owning windows
+  // ASSET.zip is open since it's a non-owning window
   // into it (see assetDskOpenFromBuffer). Freed in assetDisposeDolphinDVD.
   uint8_t *dskData;
 } assetdolphindvd_t;

src/duskdolphin/asset/assetdolphinfat.c

@@ -56,7 +56,7 @@ errorret_t assetInitDolphinFAT(void) {
   if(foundPath[0] == '\0')
     errorThrow("Failed to find asset file on FAT filesystem.");
 
-  errorChain(assetDskOpenFromPath(foundPath, &ASSET.zip, &ASSET.zipStored));
+  errorChain(assetDskOpenFromPath(foundPath, &ASSET.zip));
   errorOk();
 }
 

src/dusklinux/asset/assetlinux.c

@@ -76,9 +76,7 @@ errorret_t assetInitLinux(void) {
     printf("Try open asset file: %s\n", searchPath);
 
     // Try open
-    if(errorIsNotOk(
-      assetDskOpenFromPath(searchPath, &ASSET.zip, &ASSET.zipStored)
-    )) {
+    if(errorIsNotOk(assetDskOpenFromPath(searchPath, &ASSET.zip))) {
       printf("Failed to open asset file: %s\n", searchPath);
       continue;
     }

src/duskpsp/asset/assetpbp.c

@@ -80,8 +80,7 @@ errorret_t assetInitPBP(const char_t *pbpPath) {
     pbpPath,
     ASSET.platform.pbpHeader.psarOffset,
     psarSize,
-    &ASSET.zip,
-    &ASSET.zipStored
+    &ASSET.zip
   ));
 
   errorOk();

src/duskvita/asset/assetvita.c

@@ -10,9 +10,7 @@
 #include "assert/assert.h"
 
 errorret_t assetInitVita(void) {
-  errorChain(
-    assetDskOpenFromPath(ASSET_VITA_DSK_PATH, &ASSET.zip, &ASSET.zipStored)
-  );
+  errorChain(assetDskOpenFromPath(ASSET_VITA_DSK_PATH, &ASSET.zip));
   errorOk();
 }
 

tools/asset/pack/__main__.py

@@ -6,35 +6,26 @@
 """
 Packs the assets directory into a dusk.dsk (DSK2) archive.
 
-Replaces a single plain zip with two independent zip archives back to
-back, so specific files can be stored uncompressed instead of DEFLATEd -
-useful for small files (locale strings, config) accessed at runtime via
-repeated seeks/re-opens, which is only reliably supported by libzip for
-uncompressed (STORED) entries. Reading a whole compressed archive into
-memory just to get reliable access isn't viable once that archive holds
-the bulk of a game's binary assets, so the two archives are kept
-separate and only the (small, by convention) stored one is expected to
-ever need to be buffered whole.
+A small fixed header (magic/version/size/checksum) followed by a single
+uncompressed (ZIP_STORED) zip archive of every asset. Uncompressed rather
+than DEFLATEd because the bulk of what's packed (audio, images) is
+already compressed at the file level, so a second, zip-level DEFLATE
+pass buys negligible space back for real CPU cost, and it lets every
+entry - not just a hand-picked subset - support the reliable repeated
+seeking/re-opening (e.g. locale line-by-line reads, MP3 loop restarts)
+that libzip only supports for uncompressed entries.
 
 DSK2 format:
-  Bytes 0-3:   "DSK2" (magic)
-  Bytes 4-7:   uint32_t version = 1 (little-endian)
-  Bytes 8-11:  uint32_t compressedOffset
-  Bytes 12-15: uint32_t compressedSize
-  Bytes 16-19: uint32_t compressedChecksum (CRC32 of the compressed blob)
-  Bytes 20-23: uint32_t storedOffset
-  Bytes 24-27: uint32_t storedSize
-  Bytes 28-31: uint32_t storedChecksum (CRC32 of the stored blob)
-  Bytes 32+:   compressed zip blob, then stored zip blob (each a
-               complete, independently valid zip archive)
+  Bytes 0-3:  "DSK2" (magic)
+  Bytes 4-7:  uint32_t version = 2 (little-endian)
+  Bytes 8-11: uint32_t size (byte length of the zip archive that follows)
+  Bytes 12-15: uint32_t checksum (CRC32 of the zip archive that follows)
+  Bytes 16+:  the zip archive itself
 
 Usage:
   python3 -m tools.asset.pack --input <assets_dir> --output <dusk.dsk>
-    [--stored <glob-pattern>]...
   Every file under <assets_dir> is added, using its path relative to
-  <assets_dir> as the zip entry name, to the stored archive if its
-  relative path matches any --stored pattern (fnmatch, default:
-  "locale/*"), otherwise to the compressed archive.
+  <assets_dir> as the zip entry name.
 
 Called unconditionally on every CMake build (see the DUSK_ASSETS_BUILT
 target) rather than gated behind file-level build-system dependencies -
@@ -44,7 +35,10 @@ existing files or lag a build behind on adds/removes. Instead this script
 itself decides whether anything actually needs repacking: it writes a
 manifest of every input file's (mtime, size) next to the output on each
 successful pack, and skips the real work if a fresh manifest compares
-equal to it and the output still exists.
+equal to it and the output still exists. The manifest also carries the
+format version, so a manifest written by an older version of this script
+(a stale single/dual-archive dusk.dsk sitting from before a format
+change) never compares equal and always triggers a fresh pack.
 """
 
 import argparse
@@ -53,19 +47,17 @@ import os
 import struct
 import zipfile
 import zlib
-import fnmatch
 import io
 
 MAGIC = b'DSK2'
-VERSION = 1
-HEADER_FORMAT = '<4sIIIIIII'
+VERSION = 2
+HEADER_FORMAT = '<4sIII'
 HEADER_SIZE = struct.calcsize(HEADER_FORMAT)
-DEFAULT_STORED_PATTERNS = ['locale/*']
 
 
-def build_zip_blob(root, relative_paths, compression):
+def build_zip_blob(root, relative_paths):
   buf = io.BytesIO()
-  with zipfile.ZipFile(buf, 'w', compression=compression) as zf:
+  with zipfile.ZipFile(buf, 'w', compression=zipfile.ZIP_STORED) as zf:
     for relative_path in sorted(relative_paths):
       zf.write(os.path.join(root, relative_path), arcname=relative_path)
   return buf.getvalue()
@@ -75,12 +67,12 @@ def manifest_path(output_path):
   return output_path + '.manifest.json'
 
 
-def compute_manifest(input_dir, relative_paths, stored_patterns):
+def compute_manifest(input_dir, relative_paths):
   entries = {}
   for relative_path in relative_paths:
     st = os.stat(os.path.join(input_dir, relative_path))
     entries[relative_path] = [st.st_mtime_ns, st.st_size]
-  return {'stored_patterns': sorted(stored_patterns), 'files': entries}
+  return {'version': VERSION, 'files': entries}
 
 
 def load_manifest(path):
@@ -91,7 +83,7 @@ def load_manifest(path):
     return None
 
 
-def pack(input_dir, output_path, stored_patterns):
+def pack(input_dir, output_path):
   relative_paths = []
   for dirpath, _dirnames, filenames in os.walk(input_dir):
     for filename in filenames:
@@ -100,52 +92,28 @@ def pack(input_dir, output_path, stored_patterns):
         os.path.relpath(full_path, input_dir).replace(os.sep, '/')
       )
 
-  manifest = compute_manifest(input_dir, relative_paths, stored_patterns)
+  manifest = compute_manifest(input_dir, relative_paths)
   manifest_file = manifest_path(output_path)
   if os.path.isfile(output_path) and load_manifest(manifest_file) == manifest:
     print(f'{output_path} is up to date, skipping pack')
     return
 
-  stored_paths = [
-    path for path in relative_paths
-    if any(fnmatch.fnmatch(path, pattern) for pattern in stored_patterns)
-  ]
-  compressed_paths = [
-    path for path in relative_paths if path not in stored_paths
-  ]
-
-  compressed_blob = build_zip_blob(
-    input_dir, compressed_paths, zipfile.ZIP_DEFLATED
-  )
-  stored_blob = build_zip_blob(input_dir, stored_paths, zipfile.ZIP_STORED)
-
-  compressed_offset = HEADER_SIZE
-  stored_offset = compressed_offset + len(compressed_blob)
+  blob = build_zip_blob(input_dir, relative_paths)
 
   header = struct.pack(
-    HEADER_FORMAT,
-    MAGIC,
-    VERSION,
-    compressed_offset,
-    len(compressed_blob),
-    zlib.crc32(compressed_blob) & 0xFFFFFFFF,
-    stored_offset,
-    len(stored_blob),
-    zlib.crc32(stored_blob) & 0xFFFFFFFF,
+    HEADER_FORMAT, MAGIC, VERSION, len(blob), zlib.crc32(blob) & 0xFFFFFFFF
   )
 
   with open(output_path, 'wb') as f:
     f.write(header)
-    f.write(compressed_blob)
-    f.write(stored_blob)
+    f.write(blob)
 
   with open(manifest_file, 'w', encoding='utf-8') as f:
     json.dump(manifest, f)
 
   print(
-    f'Wrote {output_path}: {len(compressed_paths)} compressed file(s) '
-    f'({len(compressed_blob)} bytes), {len(stored_paths)} stored file(s) '
-    f'({len(stored_blob)} bytes), {len(header) + len(compressed_blob) + len(stored_blob)} bytes total'
+    f'Wrote {output_path}: {len(relative_paths)} file(s) '
+    f'({len(blob)} bytes), {len(header) + len(blob)} bytes total'
   )
 
 
@@ -155,17 +123,9 @@ def main():
   )
   parser.add_argument('--input', required=True, help='Assets directory to pack')
   parser.add_argument('--output', required=True, help='Path to write dusk.dsk to')
-  parser.add_argument(
-    '--stored',
-    action='append',
-    dest='stored_patterns',
-    help='fnmatch pattern (relative to --input) of files to store '
-         'uncompressed instead of compressing. May be given multiple '
-         'times. Defaults to "locale/*" if never given.'
-  )
   args = parser.parse_args()
 
-  pack(args.input, args.output, args.stored_patterns or DEFAULT_STORED_PATTERNS)
+  pack(args.input, args.output)
 
 
 if __name__ == '__main__':