From c8a6325615df5cd9d93129b0d20304cf801dc209 Mon Sep 17 00:00:00 2001 From: Dominic Masters Date: Wed, 9 Sep 2026 10:17:34 -0500 Subject: [PATCH] Collapse dusk.dsk from two zip archives into one, all uncompressed Bulk assets (mp3, png) are already compressed at the file level, so a DEFLATE pass on top bought negligible space for real CPU cost. Packs everything into a single ZIP_STORED archive instead, so every entry (not just locale/*) gets the reliable repeated seeking/re-opening that libzip only supports for uncompressed entries. Header shrinks from 32 to 16 bytes; version bumped to 2 so a stale pre-change dusk.dsk is rejected and rebuilt rather than misread. Co-Authored-By: Claude Sonnet 5 --- src/dusk/asset/asset.c | 18 +-- src/dusk/asset/asset.h | 14 +- src/dusk/asset/assetdsk.c | 202 +++++------------------- src/dusk/asset/assetdsk.h | 83 +++++----- src/dusk/asset/assetfile.c | 16 +- src/dusk/asset/assetfile.h | 5 - src/dusk/audio/stream/audiostreampcm.c | 8 +- src/dusk/audio/stream/audiostreampcm.h | 7 +- src/duskdolphin/asset/assetdolphindvd.c | 4 +- src/duskdolphin/asset/assetdolphindvd.h | 2 +- src/duskdolphin/asset/assetdolphinfat.c | 2 +- src/dusklinux/asset/assetlinux.c | 4 +- src/duskpsp/asset/assetpbp.c | 3 +- src/duskvita/asset/assetvita.c | 4 +- tools/asset/pack/__main__.py | 104 ++++-------- 15 files changed, 131 insertions(+), 345 deletions(-) diff --git a/src/dusk/asset/asset.c b/src/dusk/asset/asset.c index e8bd0a06..19f42c37 100644 --- a/src/dusk/asset/asset.c +++ b/src/dusk/asset/asset.c @@ -25,11 +25,9 @@ errorret_t assetInit(void) { threadMutexInit(&ASSET.loading[i].mutex); } - // assetInitPlatform must either define both ASSET.zip/ASSET.zipStored or - // throw an error. + // assetInitPlatform must either define ASSET.zip or throw an error. errorChain(assetInitPlatform()); assertNotNull(ASSET.zip, "Asset zip null without error."); - assertNotNull(ASSET.zipStored, "Asset stored zip null without error."); threadInit(&ASSET.loadThread, assetUpdateAsync); threadStart(&ASSET.loadThread); @@ -40,9 +38,7 @@ bool_t assetFileExists(const char_t *filename) { assertStrLenMax(filename, ASSET_FILE_NAME_MAX, "Filename too long."); threadMutexLock(&ASSET.zipLock); - bool_t found = - zip_name_locate(ASSET.zip, filename, 0) >= 0 || - zip_name_locate(ASSET.zipStored, filename, 0) >= 0; + bool_t found = zip_name_locate(ASSET.zip, filename, 0) >= 0; threadMutexUnlock(&ASSET.zipLock); return found; } @@ -445,19 +441,13 @@ errorret_t assetDispose(void) { errorChain(assetReapUnused()); - // Cleanup zip files. + // Cleanup zip file. if(ASSET.zip != NULL) { if(zip_close(ASSET.zip) != 0) { - errorThrow("Failed to close compressed asset zip archive."); + errorThrow("Failed to close asset zip archive."); } ASSET.zip = NULL; } - if(ASSET.zipStored != NULL) { - if(zip_close(ASSET.zipStored) != 0) { - errorThrow("Failed to close stored asset zip archive."); - } - ASSET.zipStored = NULL; - } errorChain(assetDisposePlatform()); threadMutexDispose(&ASSET.zipLock); diff --git a/src/dusk/asset/asset.h b/src/dusk/asset/asset.h index b84d9830..15e98110 100644 --- a/src/dusk/asset/asset.h +++ b/src/dusk/asset/asset.h @@ -29,16 +29,12 @@ #define ASSET_BASE_DIRECTORY_MAX 256 typedef struct asset_s { - // Compressed (DEFLATE) archive - expected to hold the bulk of a game's - // binary assets. Looked up first by assetFileInit(). + // The dusk.dsk archive - a single uncompressed (ZIP_STORED) zip, so + // every entry gets the reliable repeated seeking/re-opening that + // libzip only supports reliably for uncompressed entries (locale + // strings, MP3 loop restarts, etc.). zip_t *zip; - // Stored (uncompressed) archive - expected to hold small files that need - // reliable repeated seeking/re-opening (e.g. locale strings), which - // libzip only supports reliably for uncompressed entries. Looked up by - // assetFileInit() only if the name isn't found in `zip`. - zip_t *zipStored; - // Directory dusk.dsk was actually found/opened in (on PSP, the directory // EBOOT.PBP lives in, since dusk.dsk is embedded inside it rather than a // loose file) - populated by each platform's own assetInitXxx() once it @@ -52,7 +48,7 @@ typedef struct asset_s { assetplatform_t platform; - // Guards every libzip call against ASSET.zip/ASSET.zipStored (zip_fopen, + // Guards every libzip call against ASSET.zip (zip_fopen, // zip_fread, zip_fclose, zip_fseek, zip_stat, zip_name_locate - see // assetfile.c/assetFileExists). libzip is documented as not thread-safe, // and this asset system genuinely calls it from multiple real threads at diff --git a/src/dusk/asset/assetdsk.c b/src/dusk/asset/assetdsk.c index ab995095..95397434 100644 --- a/src/dusk/asset/assetdsk.c +++ b/src/dusk/asset/assetdsk.c @@ -30,9 +30,9 @@ errorret_t assetDskParseHeader( // Every field is a little-endian uint32_t regardless of host - convert // to host order (a no-op on little-endian hosts, a real byteswap on // Dolphin's big-endian PowerPC). - uint32_t fields[7]; + uint32_t fields[3]; memoryCopy(fields, bytes + ASSET_DSK_MAGIC_SIZE, sizeof(fields)); - for(uint8_t i = 0; i < 7; i++) { + for(uint8_t i = 0; i < 3; i++) { fields[i] = endianLittleToHost32(fields[i]); } @@ -41,24 +41,17 @@ errorret_t assetDskParseHeader( errorThrow("dusk.dsk has an unsupported version: %u", version); } - outHeader->compressedOffset = fields[1]; - outHeader->compressedSize = fields[2]; - outHeader->compressedChecksum = fields[3]; - outHeader->storedOffset = fields[4]; - outHeader->storedSize = fields[5]; - outHeader->storedChecksum = fields[6]; + outHeader->size = fields[1]; + outHeader->checksum = fields[2]; errorOk(); } errorret_t assetDskOpenFromPath( const char_t *path, - zip_t **outCompressed, - zip_t **outStored + zip_t **outZip ) { - errorChain(assetDskOpenFromPathRange( - path, 0, SIZE_MAX, outCompressed, outStored - )); + errorChain(assetDskOpenFromPathRange(path, 0, SIZE_MAX, outZip)); errorOk(); } @@ -66,15 +59,12 @@ errorret_t assetDskOpenFromPathRange( const char_t *path, const size_t baseOffset, const size_t baseSize, - zip_t **outCompressed, - zip_t **outStored + zip_t **outZip ) { assertNotNull(path, "Path cannot be NULL."); - assertNotNull(outCompressed, "Out compressed cannot be NULL."); - assertNotNull(outStored, "Out stored cannot be NULL."); + assertNotNull(outZip, "Out zip cannot be NULL."); - *outCompressed = NULL; - *outStored = NULL; + *outZip = NULL; FILE *headerFile = fopen(path, "rb"); if(headerFile == NULL) { @@ -96,104 +86,30 @@ errorret_t assetDskOpenFromPathRange( assetdskheader_t header; errorChain(assetDskParseHeader(headerBytes, sizeof(headerBytes), &header)); - if( - (size_t) header.compressedOffset + header.compressedSize > baseSize || - (size_t) header.storedOffset + header.storedSize > baseSize - ) { - errorThrow("dusk.dsk header describes ranges beyond its containing file."); + if((size_t) ASSET_DSK_HEADER_SIZE + header.size > baseSize) { + errorThrow("dusk.dsk header describes a range beyond its containing file."); } zip_error_t zipError; zip_error_init(&zipError); - zip_source_t *compressedSource = zip_source_file_create( + zip_source_t *source = zip_source_file_create( path, - (zip_uint64_t) (baseOffset + header.compressedOffset), - (zip_int64_t) header.compressedSize, + (zip_uint64_t) (baseOffset + ASSET_DSK_HEADER_SIZE), + (zip_int64_t) header.size, &zipError ); - if(compressedSource == NULL) { + if(source == NULL) { errorThrow( - "Failed to create compressed dusk.dsk source: %s", - zip_error_strerror(&zipError) + "Failed to create dusk.dsk source: %s", zip_error_strerror(&zipError) ); } - *outCompressed = - zip_open_from_source(compressedSource, ZIP_RDONLY, &zipError); - if(*outCompressed == NULL) { - zip_source_free(compressedSource); + *outZip = zip_open_from_source(source, ZIP_RDONLY, &zipError); + if(*outZip == NULL) { + zip_source_free(source); errorThrow( - "Failed to open compressed dusk.dsk archive: %s", - zip_error_strerror(&zipError) - ); - } - - zip_source_t *storedSource = zip_source_file_create( - path, - (zip_uint64_t) (baseOffset + header.storedOffset), - (zip_int64_t) header.storedSize, - &zipError - ); - if(storedSource == NULL) { - zip_close(*outCompressed); - *outCompressed = NULL; - errorThrow( - "Failed to create stored dusk.dsk source: %s", - zip_error_strerror(&zipError) - ); - } - - *outStored = zip_open_from_source(storedSource, ZIP_RDONLY, &zipError); - if(*outStored == NULL) { - zip_source_free(storedSource); - zip_close(*outCompressed); - *outCompressed = NULL; - errorThrow( - "Failed to open stored dusk.dsk archive: %s", - zip_error_strerror(&zipError) - ); - } - - // The stored archive is small by convention, so verifying its checksum - // here (one extra small read) is cheap; the compressed archive isn't - // checked since it's meant to be read lazily/on-demand from here on. - uint8_t *storedBytes = (uint8_t *) memoryAllocate(header.storedSize); - FILE *storedFile = fopen(path, "rb"); - if(storedFile == NULL) { - memoryFree(storedBytes); - zip_close(*outStored); - zip_close(*outCompressed); - *outStored = NULL; - *outCompressed = NULL; - errorThrow( - "Failed to re-open dusk.dsk to verify stored checksum: %s", path - ); - } - fseek(storedFile, (long) (baseOffset + header.storedOffset), SEEK_SET); - size_t storedRead = fread(storedBytes, 1, header.storedSize, storedFile); - fclose(storedFile); - if(storedRead != header.storedSize) { - memoryFree(storedBytes); - zip_close(*outStored); - zip_close(*outCompressed); - *outStored = NULL; - *outCompressed = NULL; - errorThrow( - "Failed to read dusk.dsk stored archive to verify checksum: %s", path - ); - } - - uint32_t checksum = - (uint32_t) crc32(0L, storedBytes, (uInt) header.storedSize); - memoryFree(storedBytes); - if(checksum != header.storedChecksum) { - zip_close(*outStored); - zip_close(*outCompressed); - *outStored = NULL; - *outCompressed = NULL; - errorThrow( - "dusk.dsk stored archive failed checksum verification: %s", path + "Failed to open dusk.dsk archive: %s", zip_error_strerror(&zipError) ); } @@ -203,85 +119,45 @@ errorret_t assetDskOpenFromPathRange( errorret_t assetDskOpenFromBuffer( uint8_t *buffer, const size_t bufferSize, - zip_t **outCompressed, - zip_t **outStored + zip_t **outZip ) { assertNotNull(buffer, "Buffer cannot be NULL."); - assertNotNull(outCompressed, "Out compressed cannot be NULL."); - assertNotNull(outStored, "Out stored cannot be NULL."); + assertNotNull(outZip, "Out zip cannot be NULL."); - *outCompressed = NULL; - *outStored = NULL; + *outZip = NULL; assetdskheader_t header; errorChain(assetDskParseHeader(buffer, bufferSize, &header)); - if( - (size_t) header.compressedOffset + header.compressedSize > bufferSize || - (size_t) header.storedOffset + header.storedSize > bufferSize - ) { - errorThrow("dusk.dsk header describes ranges beyond the buffer."); + if((size_t) ASSET_DSK_HEADER_SIZE + header.size > bufferSize) { + errorThrow("dusk.dsk header describes a range beyond the buffer."); } - uint32_t compressedChecksum = (uint32_t) crc32( - 0L, buffer + header.compressedOffset, (uInt) header.compressedSize - ); - if(compressedChecksum != header.compressedChecksum) { - errorThrow("dusk.dsk compressed archive failed checksum verification."); - } - - uint32_t storedChecksum = (uint32_t) crc32( - 0L, buffer + header.storedOffset, (uInt) header.storedSize - ); - if(storedChecksum != header.storedChecksum) { - errorThrow("dusk.dsk stored archive failed checksum verification."); + uint8_t *data = buffer + ASSET_DSK_HEADER_SIZE; + uint32_t checksum = (uint32_t) crc32(0L, data, (uInt) header.size); + if(checksum != header.checksum) { + errorThrow("dusk.dsk archive failed checksum verification."); } zip_error_t zipError; zip_error_init(&zipError); - // freep=0 for both - they're non-owning windows into the same caller-owned - // buffer, not independent allocations libzip should free. - zip_source_t *compressedSource = zip_source_buffer_create( - buffer + header.compressedOffset, header.compressedSize, 0, &zipError + // freep=0 - this is a non-owning window into the same caller-owned + // buffer, not an independent allocation libzip should free. + zip_source_t *source = zip_source_buffer_create( + data, header.size, 0, &zipError ); - if(compressedSource == NULL) { + if(source == NULL) { errorThrow( - "Failed to create compressed dusk.dsk source: %s", - zip_error_strerror(&zipError) + "Failed to create dusk.dsk source: %s", zip_error_strerror(&zipError) ); } - *outCompressed = - zip_open_from_source(compressedSource, ZIP_RDONLY, &zipError); - if(*outCompressed == NULL) { - zip_source_free(compressedSource); + *outZip = zip_open_from_source(source, ZIP_RDONLY, &zipError); + if(*outZip == NULL) { + zip_source_free(source); errorThrow( - "Failed to open compressed dusk.dsk archive: %s", - zip_error_strerror(&zipError) - ); - } - - zip_source_t *storedSource = zip_source_buffer_create( - buffer + header.storedOffset, header.storedSize, 0, &zipError - ); - if(storedSource == NULL) { - zip_close(*outCompressed); - *outCompressed = NULL; - errorThrow( - "Failed to create stored dusk.dsk source: %s", - zip_error_strerror(&zipError) - ); - } - - *outStored = zip_open_from_source(storedSource, ZIP_RDONLY, &zipError); - if(*outStored == NULL) { - zip_source_free(storedSource); - zip_close(*outCompressed); - *outCompressed = NULL; - errorThrow( - "Failed to open stored dusk.dsk archive: %s", - zip_error_strerror(&zipError) + "Failed to open dusk.dsk archive: %s", zip_error_strerror(&zipError) ); } diff --git a/src/dusk/asset/assetdsk.h b/src/dusk/asset/assetdsk.h index 345e950b..6be8644a 100644 --- a/src/dusk/asset/assetdsk.h +++ b/src/dusk/asset/assetdsk.h @@ -13,30 +13,26 @@ // never gets misread as a valid dusk.dsk. #define ASSET_DSK_MAGIC_SIZE 4 #define ASSET_DSK_MAGIC "DSK2" -#define ASSET_DSK_VERSION 1 +#define ASSET_DSK_VERSION 2 -// magic(4) + version(4) + compressedOffset(4) + compressedSize(4) + -// compressedChecksum(4) + storedOffset(4) + storedSize(4) + -// storedChecksum(4), all little-endian regardless of host - see -// assetDskParseHeader. -#define ASSET_DSK_HEADER_SIZE 32 +// magic(4) + version(4) + size(4) + checksum(4), all little-endian +// regardless of host - see assetDskParseHeader. The zip archive itself +// starts immediately after, at ASSET_DSK_HEADER_SIZE. +#define ASSET_DSK_HEADER_SIZE 16 /** - * Parsed dusk.dsk (DSK2 format) header. dusk.dsk is two independent, back - * to back zip archives (see tools/asset/pack) rather than a single plain - * zip: a "compressed" one (DEFLATE, expected to hold the bulk of a game's - * binary assets, opened lazily/on-demand) and a "stored" one (uncompressed, - * expected to hold small files - locale strings, config - that need - * reliable repeated seeking/re-opening, which libzip only supports for - * uncompressed entries). + * Parsed dusk.dsk (DSK2 format) header. dusk.dsk is a single uncompressed + * (ZIP_STORED) zip archive (see tools/asset/pack) rather than DEFLATEd - + * the bulk of what's packed (audio, images) is already compressed at the + * file level, so DEFLATE-ing the whole archive on top buys negligible + * space for real CPU cost, and every entry gets the reliable repeated + * seeking/re-opening that libzip only supports for uncompressed entries + * (locale line-by-line reads, MP3 loop restarts, etc.) instead of just a + * hand-picked subset of files. */ typedef struct { - uint32_t compressedOffset; - uint32_t compressedSize; - uint32_t compressedChecksum; - uint32_t storedOffset; - uint32_t storedSize; - uint32_t storedChecksum; + uint32_t size; + uint32_t checksum; } assetdskheader_t; /** @@ -56,27 +52,24 @@ errorret_t assetDskParseHeader( ); /** - * Opens both archives of a dusk.dsk file given a filesystem path, using - * lazy/windowed file-backed zip sources - no more memory used than the + * Opens the dusk.dsk archive given a filesystem path, using a lazy/ + * windowed file-backed zip source - no more memory used than the * existing per-platform small read buffers, matching the memory - * characteristics of a plain zip_open() on the whole file. Verifies the - * (small, by convention) stored archive's checksum; the compressed - * archive's checksum is intentionally not verified here since doing so - * would require reading the bulk of the game's assets just to compute it. + * characteristics of a plain zip_open() on the whole file. The checksum + * is intentionally not verified here since doing so would require + * reading the whole (potentially large) archive just to compute it. * * See assetDskOpenFromPathRange() for the case where the DSK2 blob isn't * the whole file (e.g. embedded inside another container). * * @param path Filesystem path to the dusk.dsk file. - * @param outCompressed Set to the opened compressed archive on success. - * @param outStored Set to the opened stored archive on success. + * @param outZip Set to the opened archive on success. * @return OK on success, error if the file is missing, too short, has a - * bad header, or either archive fails to open/verify. + * bad header, or the archive fails to open. */ errorret_t assetDskOpenFromPath( const char_t *path, - zip_t **outCompressed, - zip_t **outStored + zip_t **outZip ); /** @@ -90,39 +83,35 @@ errorret_t assetDskOpenFromPath( * @param path Filesystem path to the container file. * @param baseOffset Byte offset within `path` where the DSK2 blob starts. * @param baseSize Number of bytes available at `baseOffset`. - * @param outCompressed Set to the opened compressed archive on success. - * @param outStored Set to the opened stored archive on success. + * @param outZip Set to the opened archive on success. * @return OK on success, error if the file is missing, too short, has a - * bad header, or either archive fails to open/verify. + * bad header, or the archive fails to open. */ errorret_t assetDskOpenFromPathRange( const char_t *path, const size_t baseOffset, const size_t baseSize, - zip_t **outCompressed, - zip_t **outStored + zip_t **outZip ); /** - * Opens both archives of a dusk.dsk file already fully resident in memory - * (e.g. a PSAR embedded in an EBOOT.PBP, or an ISO-embedded file already - * read via DVD_ReadAbs - platforms that already buffer the whole file for - * reasons unrelated to this format). Neither archive takes ownership of - * `buffer` (both are opened as non-owning sub-ranges of it) - the caller + * Opens the dusk.dsk archive already fully resident in memory (e.g. a + * PSAR embedded in an EBOOT.PBP, or an ISO-embedded file already read via + * DVD_ReadAbs - platforms that already buffer the whole file for reasons + * unrelated to this format). The returned archive doesn't take ownership + * of `buffer` (it's opened as a non-owning sub-range of it) - the caller * remains responsible for freeing it, and must keep it alive for as long - * as either archive stays open. Verifies both archives' checksums, since - * the bytes are already resident. + * as the archive stays open. Verifies the checksum, since the bytes are + * already resident. * * @param buffer The whole dusk.dsk file's bytes. * @param bufferSize Number of bytes at `buffer`. - * @param outCompressed Set to the opened compressed archive on success. - * @param outStored Set to the opened stored archive on success. + * @param outZip Set to the opened archive on success. * @return OK on success, error if too short, has a bad header/checksum, or - * either archive fails to open. + * the archive fails to open. */ errorret_t assetDskOpenFromBuffer( uint8_t *buffer, const size_t bufferSize, - zip_t **outCompressed, - zip_t **outStored + zip_t **outZip ); diff --git a/src/dusk/asset/assetfile.c b/src/dusk/asset/assetfile.c index 06aaaa7f..e6de7462 100644 --- a/src/dusk/asset/assetfile.c +++ b/src/dusk/asset/assetfile.c @@ -24,20 +24,11 @@ errorret_t assetFileInit( file->params = params; file->output = output; - // Stat the file, trying the compressed archive first and falling back to - // the stored one - remember which matched so assetFileOpen opens it from - // the right archive. zip_stat_init(&file->stat); threadMutexLock(&ASSET.zipLock); - if(zip_stat(ASSET.zip, filename, 0, &file->stat) == 0) { - file->sourceZip = ASSET.zip; - } else if(zip_stat(ASSET.zipStored, filename, 0, &file->stat) == 0) { - file->sourceZip = ASSET.zipStored; - } else { - file->sourceZip = NULL; - } + int statResult = zip_stat(ASSET.zip, filename, 0, &file->stat); threadMutexUnlock(&ASSET.zipLock); - if(file->sourceZip == NULL) { + if(statResult != 0) { errorThrow("Failed to stat asset file: %s", filename); } @@ -85,11 +76,10 @@ errorret_t assetFileRewind(assetfile_t *file) { errorret_t assetFileOpen(assetfile_t *file) { assertNotNull(file, "Asset file cannot be NULL."); assertNotNull(file->filename, "Asset file filename cannot be NULL."); - assertNotNull(file->sourceZip, "Asset file must be inited before opening."); assertNull(file->zipFile, "Asset file already open."); threadMutexLock(&ASSET.zipLock); - file->zipFile = zip_fopen(file->sourceZip, file->filename, 0); + file->zipFile = zip_fopen(ASSET.zip, file->filename, 0); threadMutexUnlock(&ASSET.zipLock); if(file->zipFile == NULL) { errorThrow("Failed to open asset file: %s", file->filename); diff --git a/src/dusk/asset/assetfile.h b/src/dusk/asset/assetfile.h index 970647e0..2d02e1c9 100644 --- a/src/dusk/asset/assetfile.h +++ b/src/dusk/asset/assetfile.h @@ -32,11 +32,6 @@ typedef struct assetfile_s { zip_int64_t position; zip_int64_t lastRead; zip_file_t *zipFile; - - // The archive this file was found in (ASSET.zip or ASSET.zipStored), - // set by assetFileInit and used by assetFileOpen so lookups fall back - // correctly between the two dusk.dsk archives. - zip_t *sourceZip; } assetfile_t; /** diff --git a/src/dusk/audio/stream/audiostreampcm.c b/src/dusk/audio/stream/audiostreampcm.c index fc6862f9..c29492a0 100644 --- a/src/dusk/audio/stream/audiostreampcm.c +++ b/src/dusk/audio/stream/audiostreampcm.c @@ -87,10 +87,10 @@ errorret_t audioStreamPcmSeek(audiostream_t *stream, const size_t frame) { const size_t currentByte = (size_t) stream->pcm.file.position; if(targetByte < currentByte) { - // Only a full rewind can move a read cursor earlier once it's already - // advanced past a point - a compressed archive entry can't be decoded - // backward. See this function's own doc comment for the performance - // implications of a deep loopTo. + // A rewind is needed to move the read cursor earlier once it's + // already advanced past a point - assetFileRewind() takes the cheap + // in-place seek path since every dusk.dsk entry is stored + // uncompressed. errorChain(assetFileRewind(&stream->pcm.file)); errorChain(assetFileRead(&stream->pcm.file, NULL, targetByte)); } else if(targetByte > currentByte) { diff --git a/src/dusk/audio/stream/audiostreampcm.h b/src/dusk/audio/stream/audiostreampcm.h index 1e42a83b..0fc5e02c 100644 --- a/src/dusk/audio/stream/audiostreampcm.h +++ b/src/dusk/audio/stream/audiostreampcm.h @@ -63,11 +63,8 @@ size_t audioStreamPcmGetTotalFrames(const audiostream_t *stream); * (relative to the start of the PCM data - the same units as * audiostream_t's startFrame/loopStart/loopTo, once converted from * seconds). Seeking backward re-reads from the start of the underlying - * asset file (see assetFileRewind()) since a compressed archive entry can - * only be decoded forward - this makes a loop with a deep loopTo more - * expensive to restart than one near the start, which is a real - * performance caveat, not just a theoretical one, for anything backed by - * a compressed (not stored) asset archive entry. + * asset file (see assetFileRewind()), which is a cheap in-place seek + * since every dusk.dsk entry is stored uncompressed. * * @param stream The audio stream to seek. Must be AUDIO_STREAM_TYPE_PCM. * @param frame Frame offset to seek to, relative to the start of the PCM diff --git a/src/duskdolphin/asset/assetdolphindvd.c b/src/duskdolphin/asset/assetdolphindvd.c index 19d6ea67..b0c984f5 100644 --- a/src/duskdolphin/asset/assetdolphindvd.c +++ b/src/duskdolphin/asset/assetdolphindvd.c @@ -91,9 +91,7 @@ errorret_t assetInitDolphinDVD(void) { ); if(!data) errorThrow("Failed to read asset file from ISO."); - errorret_t ret = assetDskOpenFromBuffer( - data, fileSize, &ASSET.zip, &ASSET.zipStored - ); + errorret_t ret = assetDskOpenFromBuffer(data, fileSize, &ASSET.zip); if(errorIsNotOk(ret)) { memoryFree(data); errorChain(ret); diff --git a/src/duskdolphin/asset/assetdolphindvd.h b/src/duskdolphin/asset/assetdolphindvd.h index 9e5154eb..19197e25 100644 --- a/src/duskdolphin/asset/assetdolphindvd.h +++ b/src/duskdolphin/asset/assetdolphindvd.h @@ -25,7 +25,7 @@ typedef struct { // Whole dusk.dsk blob read from the ISO, kept alive for as long as - // ASSET.zip/ASSET.zipStored are open since they're non-owning windows + // ASSET.zip is open since it's a non-owning window // into it (see assetDskOpenFromBuffer). Freed in assetDisposeDolphinDVD. uint8_t *dskData; } assetdolphindvd_t; diff --git a/src/duskdolphin/asset/assetdolphinfat.c b/src/duskdolphin/asset/assetdolphinfat.c index 88ea98ac..32b0413e 100644 --- a/src/duskdolphin/asset/assetdolphinfat.c +++ b/src/duskdolphin/asset/assetdolphinfat.c @@ -56,7 +56,7 @@ errorret_t assetInitDolphinFAT(void) { if(foundPath[0] == '\0') errorThrow("Failed to find asset file on FAT filesystem."); - errorChain(assetDskOpenFromPath(foundPath, &ASSET.zip, &ASSET.zipStored)); + errorChain(assetDskOpenFromPath(foundPath, &ASSET.zip)); errorOk(); } diff --git a/src/dusklinux/asset/assetlinux.c b/src/dusklinux/asset/assetlinux.c index 2e6a5a11..36239586 100644 --- a/src/dusklinux/asset/assetlinux.c +++ b/src/dusklinux/asset/assetlinux.c @@ -76,9 +76,7 @@ errorret_t assetInitLinux(void) { printf("Try open asset file: %s\n", searchPath); // Try open - if(errorIsNotOk( - assetDskOpenFromPath(searchPath, &ASSET.zip, &ASSET.zipStored) - )) { + if(errorIsNotOk(assetDskOpenFromPath(searchPath, &ASSET.zip))) { printf("Failed to open asset file: %s\n", searchPath); continue; } diff --git a/src/duskpsp/asset/assetpbp.c b/src/duskpsp/asset/assetpbp.c index dc7dd424..2877d6e1 100644 --- a/src/duskpsp/asset/assetpbp.c +++ b/src/duskpsp/asset/assetpbp.c @@ -80,8 +80,7 @@ errorret_t assetInitPBP(const char_t *pbpPath) { pbpPath, ASSET.platform.pbpHeader.psarOffset, psarSize, - &ASSET.zip, - &ASSET.zipStored + &ASSET.zip )); errorOk(); diff --git a/src/duskvita/asset/assetvita.c b/src/duskvita/asset/assetvita.c index 4dcd1d6a..76197b80 100644 --- a/src/duskvita/asset/assetvita.c +++ b/src/duskvita/asset/assetvita.c @@ -10,9 +10,7 @@ #include "assert/assert.h" errorret_t assetInitVita(void) { - errorChain( - assetDskOpenFromPath(ASSET_VITA_DSK_PATH, &ASSET.zip, &ASSET.zipStored) - ); + errorChain(assetDskOpenFromPath(ASSET_VITA_DSK_PATH, &ASSET.zip)); errorOk(); } diff --git a/tools/asset/pack/__main__.py b/tools/asset/pack/__main__.py index 851740b1..c549ea0e 100644 --- a/tools/asset/pack/__main__.py +++ b/tools/asset/pack/__main__.py @@ -6,35 +6,26 @@ """ Packs the assets directory into a dusk.dsk (DSK2) archive. -Replaces a single plain zip with two independent zip archives back to -back, so specific files can be stored uncompressed instead of DEFLATEd - -useful for small files (locale strings, config) accessed at runtime via -repeated seeks/re-opens, which is only reliably supported by libzip for -uncompressed (STORED) entries. Reading a whole compressed archive into -memory just to get reliable access isn't viable once that archive holds -the bulk of a game's binary assets, so the two archives are kept -separate and only the (small, by convention) stored one is expected to -ever need to be buffered whole. +A small fixed header (magic/version/size/checksum) followed by a single +uncompressed (ZIP_STORED) zip archive of every asset. Uncompressed rather +than DEFLATEd because the bulk of what's packed (audio, images) is +already compressed at the file level, so a second, zip-level DEFLATE +pass buys negligible space back for real CPU cost, and it lets every +entry - not just a hand-picked subset - support the reliable repeated +seeking/re-opening (e.g. locale line-by-line reads, MP3 loop restarts) +that libzip only supports for uncompressed entries. DSK2 format: - Bytes 0-3: "DSK2" (magic) - Bytes 4-7: uint32_t version = 1 (little-endian) - Bytes 8-11: uint32_t compressedOffset - Bytes 12-15: uint32_t compressedSize - Bytes 16-19: uint32_t compressedChecksum (CRC32 of the compressed blob) - Bytes 20-23: uint32_t storedOffset - Bytes 24-27: uint32_t storedSize - Bytes 28-31: uint32_t storedChecksum (CRC32 of the stored blob) - Bytes 32+: compressed zip blob, then stored zip blob (each a - complete, independently valid zip archive) + Bytes 0-3: "DSK2" (magic) + Bytes 4-7: uint32_t version = 2 (little-endian) + Bytes 8-11: uint32_t size (byte length of the zip archive that follows) + Bytes 12-15: uint32_t checksum (CRC32 of the zip archive that follows) + Bytes 16+: the zip archive itself Usage: python3 -m tools.asset.pack --input --output - [--stored ]... Every file under is added, using its path relative to - as the zip entry name, to the stored archive if its - relative path matches any --stored pattern (fnmatch, default: - "locale/*"), otherwise to the compressed archive. + as the zip entry name. Called unconditionally on every CMake build (see the DUSK_ASSETS_BUILT target) rather than gated behind file-level build-system dependencies - @@ -44,7 +35,10 @@ existing files or lag a build behind on adds/removes. Instead this script itself decides whether anything actually needs repacking: it writes a manifest of every input file's (mtime, size) next to the output on each successful pack, and skips the real work if a fresh manifest compares -equal to it and the output still exists. +equal to it and the output still exists. The manifest also carries the +format version, so a manifest written by an older version of this script +(a stale single/dual-archive dusk.dsk sitting from before a format +change) never compares equal and always triggers a fresh pack. """ import argparse @@ -53,19 +47,17 @@ import os import struct import zipfile import zlib -import fnmatch import io MAGIC = b'DSK2' -VERSION = 1 -HEADER_FORMAT = '<4sIIIIIII' +VERSION = 2 +HEADER_FORMAT = '<4sIII' HEADER_SIZE = struct.calcsize(HEADER_FORMAT) -DEFAULT_STORED_PATTERNS = ['locale/*'] -def build_zip_blob(root, relative_paths, compression): +def build_zip_blob(root, relative_paths): buf = io.BytesIO() - with zipfile.ZipFile(buf, 'w', compression=compression) as zf: + with zipfile.ZipFile(buf, 'w', compression=zipfile.ZIP_STORED) as zf: for relative_path in sorted(relative_paths): zf.write(os.path.join(root, relative_path), arcname=relative_path) return buf.getvalue() @@ -75,12 +67,12 @@ def manifest_path(output_path): return output_path + '.manifest.json' -def compute_manifest(input_dir, relative_paths, stored_patterns): +def compute_manifest(input_dir, relative_paths): entries = {} for relative_path in relative_paths: st = os.stat(os.path.join(input_dir, relative_path)) entries[relative_path] = [st.st_mtime_ns, st.st_size] - return {'stored_patterns': sorted(stored_patterns), 'files': entries} + return {'version': VERSION, 'files': entries} def load_manifest(path): @@ -91,7 +83,7 @@ def load_manifest(path): return None -def pack(input_dir, output_path, stored_patterns): +def pack(input_dir, output_path): relative_paths = [] for dirpath, _dirnames, filenames in os.walk(input_dir): for filename in filenames: @@ -100,52 +92,28 @@ def pack(input_dir, output_path, stored_patterns): os.path.relpath(full_path, input_dir).replace(os.sep, '/') ) - manifest = compute_manifest(input_dir, relative_paths, stored_patterns) + manifest = compute_manifest(input_dir, relative_paths) manifest_file = manifest_path(output_path) if os.path.isfile(output_path) and load_manifest(manifest_file) == manifest: print(f'{output_path} is up to date, skipping pack') return - stored_paths = [ - path for path in relative_paths - if any(fnmatch.fnmatch(path, pattern) for pattern in stored_patterns) - ] - compressed_paths = [ - path for path in relative_paths if path not in stored_paths - ] - - compressed_blob = build_zip_blob( - input_dir, compressed_paths, zipfile.ZIP_DEFLATED - ) - stored_blob = build_zip_blob(input_dir, stored_paths, zipfile.ZIP_STORED) - - compressed_offset = HEADER_SIZE - stored_offset = compressed_offset + len(compressed_blob) + blob = build_zip_blob(input_dir, relative_paths) header = struct.pack( - HEADER_FORMAT, - MAGIC, - VERSION, - compressed_offset, - len(compressed_blob), - zlib.crc32(compressed_blob) & 0xFFFFFFFF, - stored_offset, - len(stored_blob), - zlib.crc32(stored_blob) & 0xFFFFFFFF, + HEADER_FORMAT, MAGIC, VERSION, len(blob), zlib.crc32(blob) & 0xFFFFFFFF ) with open(output_path, 'wb') as f: f.write(header) - f.write(compressed_blob) - f.write(stored_blob) + f.write(blob) with open(manifest_file, 'w', encoding='utf-8') as f: json.dump(manifest, f) print( - f'Wrote {output_path}: {len(compressed_paths)} compressed file(s) ' - f'({len(compressed_blob)} bytes), {len(stored_paths)} stored file(s) ' - f'({len(stored_blob)} bytes), {len(header) + len(compressed_blob) + len(stored_blob)} bytes total' + f'Wrote {output_path}: {len(relative_paths)} file(s) ' + f'({len(blob)} bytes), {len(header) + len(blob)} bytes total' ) @@ -155,17 +123,9 @@ def main(): ) parser.add_argument('--input', required=True, help='Assets directory to pack') parser.add_argument('--output', required=True, help='Path to write dusk.dsk to') - parser.add_argument( - '--stored', - action='append', - dest='stored_patterns', - help='fnmatch pattern (relative to --input) of files to store ' - 'uncompressed instead of compressing. May be given multiple ' - 'times. Defaults to "locale/*" if never given.' - ) args = parser.parse_args() - pack(args.input, args.output, args.stored_patterns or DEFAULT_STORED_PATTERNS) + pack(args.input, args.output) if __name__ == '__main__':