YourWishes/dusk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Collapse dusk.dsk from two zip archives into one, all uncompressed

Browse Source 
Bulk assets (mp3, png) are already compressed at the file level, so a
DEFLATE pass on top bought negligible space for real CPU cost. Packs
everything into a single ZIP_STORED archive instead, so every entry
(not just locale/*) gets the reliable repeated seeking/re-opening that
libzip only supports for uncompressed entries. Header shrinks from 32
to 16 bytes; version bumped to 2 so a stale pre-change dusk.dsk is
rejected and rebuilt rather than misread.

Co-Authored-By: Claude Sonnet 5 <[email protected]>
This commit is contained in:
YourWishes
2026-09-09 10:17:34 -05:00
co-authored by Claude Sonnet 5
parent 798db14bb5
commit c8a6325615
15 changed files with 131 additions and 345 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/dusk/asset/asset.c
+4 -14
View File
@@ -25,11 +25,9 @@ errorret_t assetInit(void) {
threadMutexInit(&ASSET.loading[i].mutex); threadMutexInit(&ASSET.loading[i].mutex);
} }
// assetInitPlatform must either define both ASSET.zip/ASSET.zipStored or // assetInitPlatform must either define ASSET.zip or throw an error.
// throw an error.
errorChain(assetInitPlatform()); errorChain(assetInitPlatform());
assertNotNull(ASSET.zip, "Asset zip null without error."); assertNotNull(ASSET.zip, "Asset zip null without error.");
assertNotNull(ASSET.zipStored, "Asset stored zip null without error.");
threadInit(&ASSET.loadThread, assetUpdateAsync); threadInit(&ASSET.loadThread, assetUpdateAsync);
threadStart(&ASSET.loadThread); threadStart(&ASSET.loadThread);
@@ -40,9 +38,7 @@ bool_t assetFileExists(const char_t *filename) {
assertStrLenMax(filename, ASSET_FILE_NAME_MAX, "Filename too long."); assertStrLenMax(filename, ASSET_FILE_NAME_MAX, "Filename too long.");
threadMutexLock(&ASSET.zipLock); threadMutexLock(&ASSET.zipLock);
bool_t found = bool_t found = zip_name_locate(ASSET.zip, filename, 0) >= 0;
zip_name_locate(ASSET.zip, filename, 0) >= 0 ||
zip_name_locate(ASSET.zipStored, filename, 0) >= 0;
threadMutexUnlock(&ASSET.zipLock); threadMutexUnlock(&ASSET.zipLock);
return found; return found;
} }
@@ -445,19 +441,13 @@ errorret_t assetDispose(void) {
errorChain(assetReapUnused()); errorChain(assetReapUnused());
// Cleanup zip files. // Cleanup zip file.
if(ASSET.zip != NULL) { if(ASSET.zip != NULL) {
if(zip_close(ASSET.zip) != 0) { if(zip_close(ASSET.zip) != 0) {
errorThrow("Failed to close compressed asset zip archive."); errorThrow("Failed to close asset zip archive.");
} }
ASSET.zip = NULL; ASSET.zip = NULL;
} }
if(ASSET.zipStored != NULL) {
if(zip_close(ASSET.zipStored) != 0) {
errorThrow("Failed to close stored asset zip archive.");
}
ASSET.zipStored = NULL;
}
errorChain(assetDisposePlatform()); errorChain(assetDisposePlatform());
threadMutexDispose(&ASSET.zipLock); threadMutexDispose(&ASSET.zipLock);
src/dusk/asset/asset.h
+5 -9
View File
@@ -29,16 +29,12 @@
#define ASSET_BASE_DIRECTORY_MAX 256 #define ASSET_BASE_DIRECTORY_MAX 256
typedef struct asset_s { typedef struct asset_s {
// Compressed (DEFLATE) archive - expected to hold the bulk of a game's // The dusk.dsk archive - a single uncompressed (ZIP_STORED) zip, so
// binary assets. Looked up first by assetFileInit(). // every entry gets the reliable repeated seeking/re-opening that
// libzip only supports reliably for uncompressed entries (locale
// strings, MP3 loop restarts, etc.).
zip_t *zip; zip_t *zip;
// Stored (uncompressed) archive - expected to hold small files that need
// reliable repeated seeking/re-opening (e.g. locale strings), which
// libzip only supports reliably for uncompressed entries. Looked up by
// assetFileInit() only if the name isn't found in `zip`.
zip_t *zipStored;
// Directory dusk.dsk was actually found/opened in (on PSP, the directory // Directory dusk.dsk was actually found/opened in (on PSP, the directory
// EBOOT.PBP lives in, since dusk.dsk is embedded inside it rather than a // EBOOT.PBP lives in, since dusk.dsk is embedded inside it rather than a
// loose file) - populated by each platform's own assetInitXxx() once it // loose file) - populated by each platform's own assetInitXxx() once it
@@ -52,7 +48,7 @@ typedef struct asset_s {
assetplatform_t platform; assetplatform_t platform;
// Guards every libzip call against ASSET.zip/ASSET.zipStored (zip_fopen, // Guards every libzip call against ASSET.zip (zip_fopen,
// zip_fread, zip_fclose, zip_fseek, zip_stat, zip_name_locate - see // zip_fread, zip_fclose, zip_fseek, zip_stat, zip_name_locate - see
// assetfile.c/assetFileExists). libzip is documented as not thread-safe, // assetfile.c/assetFileExists). libzip is documented as not thread-safe,
// and this asset system genuinely calls it from multiple real threads at // and this asset system genuinely calls it from multiple real threads at
src/dusk/asset/assetdsk.c
+39 -163
View File
@@ -30,9 +30,9 @@ errorret_t assetDskParseHeader(
// Every field is a little-endian uint32_t regardless of host - convert // Every field is a little-endian uint32_t regardless of host - convert
// to host order (a no-op on little-endian hosts, a real byteswap on // to host order (a no-op on little-endian hosts, a real byteswap on
// Dolphin's big-endian PowerPC). // Dolphin's big-endian PowerPC).
uint32_t fields[7]; uint32_t fields[3];
memoryCopy(fields, bytes + ASSET_DSK_MAGIC_SIZE, sizeof(fields)); memoryCopy(fields, bytes + ASSET_DSK_MAGIC_SIZE, sizeof(fields));
for(uint8_t i = 0; i < 7; i++) { for(uint8_t i = 0; i < 3; i++) {
fields[i] = endianLittleToHost32(fields[i]); fields[i] = endianLittleToHost32(fields[i]);
} }
@@ -41,24 +41,17 @@ errorret_t assetDskParseHeader(
errorThrow("dusk.dsk has an unsupported version: %u", version); errorThrow("dusk.dsk has an unsupported version: %u", version);
} }
outHeader->compressedOffset = fields[1]; outHeader->size = fields[1];
outHeader->compressedSize = fields[2]; outHeader->checksum = fields[2];
outHeader->compressedChecksum = fields[3];
outHeader->storedOffset = fields[4];
outHeader->storedSize = fields[5];
outHeader->storedChecksum = fields[6];
errorOk(); errorOk();
} }
errorret_t assetDskOpenFromPath( errorret_t assetDskOpenFromPath(
const char_t *path, const char_t *path,
zip_t **outCompressed, zip_t **outZip
zip_t **outStored
) { ) {
errorChain(assetDskOpenFromPathRange( errorChain(assetDskOpenFromPathRange(path, 0, SIZE_MAX, outZip));
path, 0, SIZE_MAX, outCompressed, outStored
));
errorOk(); errorOk();
} }
@@ -66,15 +59,12 @@ errorret_t assetDskOpenFromPathRange(
const char_t *path, const char_t *path,
const size_t baseOffset, const size_t baseOffset,
const size_t baseSize, const size_t baseSize,
zip_t **outCompressed, zip_t **outZip
zip_t **outStored
) { ) {
assertNotNull(path, "Path cannot be NULL."); assertNotNull(path, "Path cannot be NULL.");
assertNotNull(outCompressed, "Out compressed cannot be NULL."); assertNotNull(outZip, "Out zip cannot be NULL.");
assertNotNull(outStored, "Out stored cannot be NULL.");
*outCompressed = NULL; *outZip = NULL;
*outStored = NULL;
FILE *headerFile = fopen(path, "rb"); FILE *headerFile = fopen(path, "rb");
if(headerFile == NULL) { if(headerFile == NULL) {
@@ -96,104 +86,30 @@ errorret_t assetDskOpenFromPathRange(
assetdskheader_t header; assetdskheader_t header;
errorChain(assetDskParseHeader(headerBytes, sizeof(headerBytes), &header)); errorChain(assetDskParseHeader(headerBytes, sizeof(headerBytes), &header));
if( if((size_t) ASSET_DSK_HEADER_SIZE + header.size > baseSize) {
(size_t) header.compressedOffset + header.compressedSize > baseSize || errorThrow("dusk.dsk header describes a range beyond its containing file.");
(size_t) header.storedOffset + header.storedSize > baseSize
) {
errorThrow("dusk.dsk header describes ranges beyond its containing file.");
} }
zip_error_t zipError; zip_error_t zipError;
zip_error_init(&zipError); zip_error_init(&zipError);
zip_source_t *compressedSource = zip_source_file_create( zip_source_t *source = zip_source_file_create(
path, path,
(zip_uint64_t) (baseOffset + header.compressedOffset), (zip_uint64_t) (baseOffset + ASSET_DSK_HEADER_SIZE),
(zip_int64_t) header.compressedSize, (zip_int64_t) header.size,
&zipError &zipError
); );
if(compressedSource == NULL) { if(source == NULL) {
errorThrow( errorThrow(
"Failed to create compressed dusk.dsk source: %s", "Failed to create dusk.dsk source: %s", zip_error_strerror(&zipError)
zip_error_strerror(&zipError)
); );
} }
*outCompressed = *outZip = zip_open_from_source(source, ZIP_RDONLY, &zipError);
zip_open_from_source(compressedSource, ZIP_RDONLY, &zipError); if(*outZip == NULL) {
if(*outCompressed == NULL) { zip_source_free(source);
zip_source_free(compressedSource);
errorThrow( errorThrow(
"Failed to open compressed dusk.dsk archive: %s", "Failed to open dusk.dsk archive: %s", zip_error_strerror(&zipError)
zip_error_strerror(&zipError)
);
}
zip_source_t *storedSource = zip_source_file_create(
path,
(zip_uint64_t) (baseOffset + header.storedOffset),
(zip_int64_t) header.storedSize,
&zipError
);
if(storedSource == NULL) {
zip_close(*outCompressed);
*outCompressed = NULL;
errorThrow(
"Failed to create stored dusk.dsk source: %s",
zip_error_strerror(&zipError)
);
}
*outStored = zip_open_from_source(storedSource, ZIP_RDONLY, &zipError);
if(*outStored == NULL) {
zip_source_free(storedSource);
zip_close(*outCompressed);
*outCompressed = NULL;
errorThrow(
"Failed to open stored dusk.dsk archive: %s",
zip_error_strerror(&zipError)
);
}
// The stored archive is small by convention, so verifying its checksum
// here (one extra small read) is cheap; the compressed archive isn't
// checked since it's meant to be read lazily/on-demand from here on.
uint8_t *storedBytes = (uint8_t *) memoryAllocate(header.storedSize);
FILE *storedFile = fopen(path, "rb");
if(storedFile == NULL) {
memoryFree(storedBytes);
zip_close(*outStored);
zip_close(*outCompressed);
*outStored = NULL;
*outCompressed = NULL;
errorThrow(
"Failed to re-open dusk.dsk to verify stored checksum: %s", path
);
}
fseek(storedFile, (long) (baseOffset + header.storedOffset), SEEK_SET);
size_t storedRead = fread(storedBytes, 1, header.storedSize, storedFile);
fclose(storedFile);
if(storedRead != header.storedSize) {
memoryFree(storedBytes);
zip_close(*outStored);
zip_close(*outCompressed);
*outStored = NULL;
*outCompressed = NULL;
errorThrow(
"Failed to read dusk.dsk stored archive to verify checksum: %s", path
);
}
uint32_t checksum =
(uint32_t) crc32(0L, storedBytes, (uInt) header.storedSize);
memoryFree(storedBytes);
if(checksum != header.storedChecksum) {
zip_close(*outStored);
zip_close(*outCompressed);
*outStored = NULL;
*outCompressed = NULL;
errorThrow(
"dusk.dsk stored archive failed checksum verification: %s", path
); );
} }
@@ -203,85 +119,45 @@ errorret_t assetDskOpenFromPathRange(
errorret_t assetDskOpenFromBuffer( errorret_t assetDskOpenFromBuffer(
uint8_t *buffer, uint8_t *buffer,
const size_t bufferSize, const size_t bufferSize,
zip_t **outCompressed, zip_t **outZip
zip_t **outStored
) { ) {
assertNotNull(buffer, "Buffer cannot be NULL."); assertNotNull(buffer, "Buffer cannot be NULL.");
assertNotNull(outCompressed, "Out compressed cannot be NULL."); assertNotNull(outZip, "Out zip cannot be NULL.");
assertNotNull(outStored, "Out stored cannot be NULL.");
*outCompressed = NULL; *outZip = NULL;
*outStored = NULL;
assetdskheader_t header; assetdskheader_t header;
errorChain(assetDskParseHeader(buffer, bufferSize, &header)); errorChain(assetDskParseHeader(buffer, bufferSize, &header));
if( if((size_t) ASSET_DSK_HEADER_SIZE + header.size > bufferSize) {
(size_t) header.compressedOffset + header.compressedSize > bufferSize || errorThrow("dusk.dsk header describes a range beyond the buffer.");
(size_t) header.storedOffset + header.storedSize > bufferSize
) {
errorThrow("dusk.dsk header describes ranges beyond the buffer.");
} }
uint32_t compressedChecksum = (uint32_t) crc32( uint8_t *data = buffer + ASSET_DSK_HEADER_SIZE;
0L, buffer + header.compressedOffset, (uInt) header.compressedSize uint32_t checksum = (uint32_t) crc32(0L, data, (uInt) header.size);
); if(checksum != header.checksum) {
if(compressedChecksum != header.compressedChecksum) { errorThrow("dusk.dsk archive failed checksum verification.");
errorThrow("dusk.dsk compressed archive failed checksum verification.");
}
uint32_t storedChecksum = (uint32_t) crc32(
0L, buffer + header.storedOffset, (uInt) header.storedSize
);
if(storedChecksum != header.storedChecksum) {
errorThrow("dusk.dsk stored archive failed checksum verification.");
} }
zip_error_t zipError; zip_error_t zipError;
zip_error_init(&zipError); zip_error_init(&zipError);
// freep=0 for both - they're non-owning windows into the same caller-owned // freep=0 - this is a non-owning window into the same caller-owned
// buffer, not independent allocations libzip should free. // buffer, not an independent allocation libzip should free.
zip_source_t *compressedSource = zip_source_buffer_create( zip_source_t *source = zip_source_buffer_create(
buffer + header.compressedOffset, header.compressedSize, 0, &zipError data, header.size, 0, &zipError
); );
if(compressedSource == NULL) { if(source == NULL) {
errorThrow( errorThrow(
"Failed to create compressed dusk.dsk source: %s", "Failed to create dusk.dsk source: %s", zip_error_strerror(&zipError)
zip_error_strerror(&zipError)
); );
} }
*outCompressed = *outZip = zip_open_from_source(source, ZIP_RDONLY, &zipError);
zip_open_from_source(compressedSource, ZIP_RDONLY, &zipError); if(*outZip == NULL) {
if(*outCompressed == NULL) { zip_source_free(source);
zip_source_free(compressedSource);
errorThrow( errorThrow(
"Failed to open compressed dusk.dsk archive: %s", "Failed to open dusk.dsk archive: %s", zip_error_strerror(&zipError)
zip_error_strerror(&zipError)
);
}
zip_source_t *storedSource = zip_source_buffer_create(
buffer + header.storedOffset, header.storedSize, 0, &zipError
);
if(storedSource == NULL) {
zip_close(*outCompressed);
*outCompressed = NULL;
errorThrow(
"Failed to create stored dusk.dsk source: %s",
zip_error_strerror(&zipError)
);
}
*outStored = zip_open_from_source(storedSource, ZIP_RDONLY, &zipError);
if(*outStored == NULL) {
zip_source_free(storedSource);
zip_close(*outCompressed);
*outCompressed = NULL;
errorThrow(
"Failed to open stored dusk.dsk archive: %s",
zip_error_strerror(&zipError)
); );
} }
src/dusk/asset/assetdsk.h
+36 -47
View File
@@ -13,30 +13,26 @@
// never gets misread as a valid dusk.dsk. // never gets misread as a valid dusk.dsk.
#define ASSET_DSK_MAGIC_SIZE 4 #define ASSET_DSK_MAGIC_SIZE 4
#define ASSET_DSK_MAGIC "DSK2" #define ASSET_DSK_MAGIC "DSK2"
#define ASSET_DSK_VERSION 1 #define ASSET_DSK_VERSION 2
// magic(4) + version(4) + compressedOffset(4) + compressedSize(4) + // magic(4) + version(4) + size(4) + checksum(4), all little-endian
// compressedChecksum(4) + storedOffset(4) + storedSize(4) + // regardless of host - see assetDskParseHeader. The zip archive itself
// storedChecksum(4), all little-endian regardless of host - see // starts immediately after, at ASSET_DSK_HEADER_SIZE.
// assetDskParseHeader. #define ASSET_DSK_HEADER_SIZE 16
#define ASSET_DSK_HEADER_SIZE 32
/** /**
* Parsed dusk.dsk (DSK2 format) header. dusk.dsk is two independent, back * Parsed dusk.dsk (DSK2 format) header. dusk.dsk is a single uncompressed
* to back zip archives (see tools/asset/pack) rather than a single plain * (ZIP_STORED) zip archive (see tools/asset/pack) rather than DEFLATEd -
* zip: a "compressed" one (DEFLATE, expected to hold the bulk of a game's * the bulk of what's packed (audio, images) is already compressed at the
* binary assets, opened lazily/on-demand) and a "stored" one (uncompressed, * file level, so DEFLATE-ing the whole archive on top buys negligible
* expected to hold small files - locale strings, config - that need * space for real CPU cost, and every entry gets the reliable repeated
* reliable repeated seeking/re-opening, which libzip only supports for * seeking/re-opening that libzip only supports for uncompressed entries
* uncompressed entries). * (locale line-by-line reads, MP3 loop restarts, etc.) instead of just a
* hand-picked subset of files.
*/ */
typedef struct { typedef struct {
uint32_t compressedOffset; uint32_t size;
uint32_t compressedSize; uint32_t checksum;
uint32_t compressedChecksum;
uint32_t storedOffset;
uint32_t storedSize;
uint32_t storedChecksum;
} assetdskheader_t; } assetdskheader_t;
/** /**
@@ -56,27 +52,24 @@ errorret_t assetDskParseHeader(
); );
/** /**
* Opens both archives of a dusk.dsk file given a filesystem path, using * Opens the dusk.dsk archive given a filesystem path, using a lazy/
* lazy/windowed file-backed zip sources - no more memory used than the * windowed file-backed zip source - no more memory used than the
* existing per-platform small read buffers, matching the memory * existing per-platform small read buffers, matching the memory
* characteristics of a plain zip_open() on the whole file. Verifies the * characteristics of a plain zip_open() on the whole file. The checksum
* (small, by convention) stored archive's checksum; the compressed * is intentionally not verified here since doing so would require
* archive's checksum is intentionally not verified here since doing so * reading the whole (potentially large) archive just to compute it.
* would require reading the bulk of the game's assets just to compute it.
* *
* See assetDskOpenFromPathRange() for the case where the DSK2 blob isn't * See assetDskOpenFromPathRange() for the case where the DSK2 blob isn't
* the whole file (e.g. embedded inside another container). * the whole file (e.g. embedded inside another container).
* *
* @param path Filesystem path to the dusk.dsk file. * @param path Filesystem path to the dusk.dsk file.
* @param outCompressed Set to the opened compressed archive on success. * @param outZip Set to the opened archive on success.
* @param outStored Set to the opened stored archive on success.
* @return OK on success, error if the file is missing, too short, has a * @return OK on success, error if the file is missing, too short, has a
* bad header, or either archive fails to open/verify. * bad header, or the archive fails to open.
*/ */
errorret_t assetDskOpenFromPath( errorret_t assetDskOpenFromPath(
const char_t *path, const char_t *path,
zip_t **outCompressed, zip_t **outZip
zip_t **outStored
); );
/** /**
@@ -90,39 +83,35 @@ errorret_t assetDskOpenFromPath(
* @param path Filesystem path to the container file. * @param path Filesystem path to the container file.
* @param baseOffset Byte offset within `path` where the DSK2 blob starts. * @param baseOffset Byte offset within `path` where the DSK2 blob starts.
* @param baseSize Number of bytes available at `baseOffset`. * @param baseSize Number of bytes available at `baseOffset`.
* @param outCompressed Set to the opened compressed archive on success. * @param outZip Set to the opened archive on success.
* @param outStored Set to the opened stored archive on success.
* @return OK on success, error if the file is missing, too short, has a * @return OK on success, error if the file is missing, too short, has a
* bad header, or either archive fails to open/verify. * bad header, or the archive fails to open.
*/ */
errorret_t assetDskOpenFromPathRange( errorret_t assetDskOpenFromPathRange(
const char_t *path, const char_t *path,
const size_t baseOffset, const size_t baseOffset,
const size_t baseSize, const size_t baseSize,
zip_t **outCompressed, zip_t **outZip
zip_t **outStored
); );
/** /**
* Opens both archives of a dusk.dsk file already fully resident in memory * Opens the dusk.dsk archive already fully resident in memory (e.g. a
* (e.g. a PSAR embedded in an EBOOT.PBP, or an ISO-embedded file already * PSAR embedded in an EBOOT.PBP, or an ISO-embedded file already read via
* read via DVD_ReadAbs - platforms that already buffer the whole file for * DVD_ReadAbs - platforms that already buffer the whole file for reasons
* reasons unrelated to this format). Neither archive takes ownership of * unrelated to this format). The returned archive doesn't take ownership
* `buffer` (both are opened as non-owning sub-ranges of it) - the caller * of `buffer` (it's opened as a non-owning sub-range of it) - the caller
* remains responsible for freeing it, and must keep it alive for as long * remains responsible for freeing it, and must keep it alive for as long
* as either archive stays open. Verifies both archives' checksums, since * as the archive stays open. Verifies the checksum, since the bytes are
* the bytes are already resident. * already resident.
* *
* @param buffer The whole dusk.dsk file's bytes. * @param buffer The whole dusk.dsk file's bytes.
* @param bufferSize Number of bytes at `buffer`. * @param bufferSize Number of bytes at `buffer`.
* @param outCompressed Set to the opened compressed archive on success. * @param outZip Set to the opened archive on success.
* @param outStored Set to the opened stored archive on success.
* @return OK on success, error if too short, has a bad header/checksum, or * @return OK on success, error if too short, has a bad header/checksum, or
* either archive fails to open. * the archive fails to open.
*/ */
errorret_t assetDskOpenFromBuffer( errorret_t assetDskOpenFromBuffer(
uint8_t *buffer, uint8_t *buffer,
const size_t bufferSize, const size_t bufferSize,
zip_t **outCompressed, zip_t **outZip
zip_t **outStored
); );
src/dusk/asset/assetfile.c
+3 -13
View File
@@ -24,20 +24,11 @@ errorret_t assetFileInit(
file->params = params; file->params = params;
file->output = output; file->output = output;
// Stat the file, trying the compressed archive first and falling back to
// the stored one - remember which matched so assetFileOpen opens it from
// the right archive.
zip_stat_init(&file->stat); zip_stat_init(&file->stat);
threadMutexLock(&ASSET.zipLock); threadMutexLock(&ASSET.zipLock);
if(zip_stat(ASSET.zip, filename, 0, &file->stat) == 0) { int statResult = zip_stat(ASSET.zip, filename, 0, &file->stat);
file->sourceZip = ASSET.zip;
} else if(zip_stat(ASSET.zipStored, filename, 0, &file->stat) == 0) {
file->sourceZip = ASSET.zipStored;
} else {
file->sourceZip = NULL;
}
threadMutexUnlock(&ASSET.zipLock); threadMutexUnlock(&ASSET.zipLock);
if(file->sourceZip == NULL) { if(statResult != 0) {
errorThrow("Failed to stat asset file: %s", filename); errorThrow("Failed to stat asset file: %s", filename);
} }
@@ -85,11 +76,10 @@ errorret_t assetFileRewind(assetfile_t *file) {
errorret_t assetFileOpen(assetfile_t *file) { errorret_t assetFileOpen(assetfile_t *file) {
assertNotNull(file, "Asset file cannot be NULL."); assertNotNull(file, "Asset file cannot be NULL.");
assertNotNull(file->filename, "Asset file filename cannot be NULL."); assertNotNull(file->filename, "Asset file filename cannot be NULL.");
assertNotNull(file->sourceZip, "Asset file must be inited before opening.");
assertNull(file->zipFile, "Asset file already open."); assertNull(file->zipFile, "Asset file already open.");
threadMutexLock(&ASSET.zipLock); threadMutexLock(&ASSET.zipLock);
file->zipFile = zip_fopen(file->sourceZip, file->filename, 0); file->zipFile = zip_fopen(ASSET.zip, file->filename, 0);
threadMutexUnlock(&ASSET.zipLock); threadMutexUnlock(&ASSET.zipLock);
if(file->zipFile == NULL) { if(file->zipFile == NULL) {
errorThrow("Failed to open asset file: %s", file->filename); errorThrow("Failed to open asset file: %s", file->filename);
src/dusk/asset/assetfile.h
-5
View File
@@ -32,11 +32,6 @@ typedef struct assetfile_s {
zip_int64_t position; zip_int64_t position;
zip_int64_t lastRead; zip_int64_t lastRead;
zip_file_t *zipFile; zip_file_t *zipFile;
// The archive this file was found in (ASSET.zip or ASSET.zipStored),
// set by assetFileInit and used by assetFileOpen so lookups fall back
// correctly between the two dusk.dsk archives.
zip_t *sourceZip;
} assetfile_t; } assetfile_t;
/** /**
src/dusk/audio/stream/audiostreampcm.c
+4 -4
View File
@@ -87,10 +87,10 @@ errorret_t audioStreamPcmSeek(audiostream_t *stream, const size_t frame) {
const size_t currentByte = (size_t) stream->pcm.file.position; const size_t currentByte = (size_t) stream->pcm.file.position;
if(targetByte < currentByte) { if(targetByte < currentByte) {
// Only a full rewind can move a read cursor earlier once it's already // A rewind is needed to move the read cursor earlier once it's
// advanced past a point - a compressed archive entry can't be decoded // already advanced past a point - assetFileRewind() takes the cheap
// backward. See this function's own doc comment for the performance // in-place seek path since every dusk.dsk entry is stored
// implications of a deep loopTo. // uncompressed.
errorChain(assetFileRewind(&stream->pcm.file)); errorChain(assetFileRewind(&stream->pcm.file));
errorChain(assetFileRead(&stream->pcm.file, NULL, targetByte)); errorChain(assetFileRead(&stream->pcm.file, NULL, targetByte));
} else if(targetByte > currentByte) { } else if(targetByte > currentByte) {
src/dusk/audio/stream/audiostreampcm.h
+2 -5
View File
@@ -63,11 +63,8 @@ size_t audioStreamPcmGetTotalFrames(const audiostream_t *stream);
* (relative to the start of the PCM data - the same units as * (relative to the start of the PCM data - the same units as
* audiostream_t's startFrame/loopStart/loopTo, once converted from * audiostream_t's startFrame/loopStart/loopTo, once converted from
* seconds). Seeking backward re-reads from the start of the underlying * seconds). Seeking backward re-reads from the start of the underlying
* asset file (see assetFileRewind()) since a compressed archive entry can * asset file (see assetFileRewind()), which is a cheap in-place seek
* only be decoded forward - this makes a loop with a deep loopTo more * since every dusk.dsk entry is stored uncompressed.
* expensive to restart than one near the start, which is a real
* performance caveat, not just a theoretical one, for anything backed by
* a compressed (not stored) asset archive entry.
* *
* @param stream The audio stream to seek. Must be AUDIO_STREAM_TYPE_PCM. * @param stream The audio stream to seek. Must be AUDIO_STREAM_TYPE_PCM.
* @param frame Frame offset to seek to, relative to the start of the PCM * @param frame Frame offset to seek to, relative to the start of the PCM
src/duskdolphin/asset/assetdolphindvd.c
+1 -3
View File
@@ -91,9 +91,7 @@ errorret_t assetInitDolphinDVD(void) {
); );
if(!data) errorThrow("Failed to read asset file from ISO."); if(!data) errorThrow("Failed to read asset file from ISO.");
errorret_t ret = assetDskOpenFromBuffer( errorret_t ret = assetDskOpenFromBuffer(data, fileSize, &ASSET.zip);
data, fileSize, &ASSET.zip, &ASSET.zipStored
);
if(errorIsNotOk(ret)) { if(errorIsNotOk(ret)) {
memoryFree(data); memoryFree(data);
errorChain(ret); errorChain(ret);
src/duskdolphin/asset/assetdolphindvd.h
+1 -1
View File
@@ -25,7 +25,7 @@
typedef struct { typedef struct {
// Whole dusk.dsk blob read from the ISO, kept alive for as long as // Whole dusk.dsk blob read from the ISO, kept alive for as long as
// ASSET.zip/ASSET.zipStored are open since they're non-owning windows // ASSET.zip is open since it's a non-owning window
// into it (see assetDskOpenFromBuffer). Freed in assetDisposeDolphinDVD. // into it (see assetDskOpenFromBuffer). Freed in assetDisposeDolphinDVD.
uint8_t *dskData; uint8_t *dskData;
} assetdolphindvd_t; } assetdolphindvd_t;
src/duskdolphin/asset/assetdolphinfat.c
+1 -1
View File
@@ -56,7 +56,7 @@ errorret_t assetInitDolphinFAT(void) {
if(foundPath[0] == '\0') if(foundPath[0] == '\0')
errorThrow("Failed to find asset file on FAT filesystem."); errorThrow("Failed to find asset file on FAT filesystem.");
errorChain(assetDskOpenFromPath(foundPath, &ASSET.zip, &ASSET.zipStored)); errorChain(assetDskOpenFromPath(foundPath, &ASSET.zip));
errorOk(); errorOk();
} }
src/dusklinux/asset/assetlinux.c
+1 -3
View File
@@ -76,9 +76,7 @@ errorret_t assetInitLinux(void) {
printf("Try open asset file: %s\n", searchPath); printf("Try open asset file: %s\n", searchPath);
// Try open // Try open
if(errorIsNotOk( if(errorIsNotOk(assetDskOpenFromPath(searchPath, &ASSET.zip))) {
assetDskOpenFromPath(searchPath, &ASSET.zip, &ASSET.zipStored)
)) {
printf("Failed to open asset file: %s\n", searchPath); printf("Failed to open asset file: %s\n", searchPath);
continue; continue;
} }
src/duskpsp/asset/assetpbp.c
+1 -2
View File
@@ -80,8 +80,7 @@ errorret_t assetInitPBP(const char_t *pbpPath) {
pbpPath, pbpPath,
ASSET.platform.pbpHeader.psarOffset, ASSET.platform.pbpHeader.psarOffset,
psarSize, psarSize,
&ASSET.zip, &ASSET.zip
&ASSET.zipStored
)); ));
errorOk(); errorOk();
src/duskvita/asset/assetvita.c
+1 -3
View File
@@ -10,9 +10,7 @@
#include "assert/assert.h" #include "assert/assert.h"
errorret_t assetInitVita(void) { errorret_t assetInitVita(void) {
errorChain( errorChain(assetDskOpenFromPath(ASSET_VITA_DSK_PATH, &ASSET.zip));
assetDskOpenFromPath(ASSET_VITA_DSK_PATH, &ASSET.zip, &ASSET.zipStored)
);
errorOk(); errorOk();
} }
tools/asset/pack/__main__.py
+32 -72
View File
@@ -6,35 +6,26 @@
""" """
Packs the assets directory into a dusk.dsk (DSK2) archive. Packs the assets directory into a dusk.dsk (DSK2) archive.
Replaces a single plain zip with two independent zip archives back to A small fixed header (magic/version/size/checksum) followed by a single
back, so specific files can be stored uncompressed instead of DEFLATEd - uncompressed (ZIP_STORED) zip archive of every asset. Uncompressed rather
useful for small files (locale strings, config) accessed at runtime via than DEFLATEd because the bulk of what's packed (audio, images) is
repeated seeks/re-opens, which is only reliably supported by libzip for already compressed at the file level, so a second, zip-level DEFLATE
uncompressed (STORED) entries. Reading a whole compressed archive into pass buys negligible space back for real CPU cost, and it lets every
memory just to get reliable access isn't viable once that archive holds entry - not just a hand-picked subset - support the reliable repeated
the bulk of a game's binary assets, so the two archives are kept seeking/re-opening (e.g. locale line-by-line reads, MP3 loop restarts)
separate and only the (small, by convention) stored one is expected to that libzip only supports for uncompressed entries.
ever need to be buffered whole.
DSK2 format: DSK2 format:
Bytes 0-3: "DSK2" (magic) Bytes 0-3: "DSK2" (magic)
Bytes 4-7: uint32_t version = 1 (little-endian) Bytes 4-7: uint32_t version = 2 (little-endian)
Bytes 8-11: uint32_t compressedOffset Bytes 8-11: uint32_t size (byte length of the zip archive that follows)
Bytes 12-15: uint32_t compressedSize Bytes 12-15: uint32_t checksum (CRC32 of the zip archive that follows)
Bytes 16-19: uint32_t compressedChecksum (CRC32 of the compressed blob) Bytes 16+: the zip archive itself
Bytes 20-23: uint32_t storedOffset
Bytes 24-27: uint32_t storedSize
Bytes 28-31: uint32_t storedChecksum (CRC32 of the stored blob)
Bytes 32+: compressed zip blob, then stored zip blob (each a
complete, independently valid zip archive)
Usage: Usage:
python3 -m tools.asset.pack --input <assets_dir> --output <dusk.dsk> python3 -m tools.asset.pack --input <assets_dir> --output <dusk.dsk>
[--stored <glob-pattern>]...
Every file under <assets_dir> is added, using its path relative to Every file under <assets_dir> is added, using its path relative to
<assets_dir> as the zip entry name, to the stored archive if its <assets_dir> as the zip entry name.
relative path matches any --stored pattern (fnmatch, default:
"locale/*"), otherwise to the compressed archive.
Called unconditionally on every CMake build (see the DUSK_ASSETS_BUILT Called unconditionally on every CMake build (see the DUSK_ASSETS_BUILT
target) rather than gated behind file-level build-system dependencies - target) rather than gated behind file-level build-system dependencies -
@@ -44,7 +35,10 @@ existing files or lag a build behind on adds/removes. Instead this script
itself decides whether anything actually needs repacking: it writes a itself decides whether anything actually needs repacking: it writes a
manifest of every input file's (mtime, size) next to the output on each manifest of every input file's (mtime, size) next to the output on each
successful pack, and skips the real work if a fresh manifest compares successful pack, and skips the real work if a fresh manifest compares
equal to it and the output still exists. equal to it and the output still exists. The manifest also carries the
format version, so a manifest written by an older version of this script
(a stale single/dual-archive dusk.dsk sitting from before a format
change) never compares equal and always triggers a fresh pack.
""" """
import argparse import argparse
@@ -53,19 +47,17 @@ import os
import struct import struct
import zipfile import zipfile
import zlib import zlib
import fnmatch
import io import io
MAGIC = b'DSK2' MAGIC = b'DSK2'
VERSION = 1 VERSION = 2
HEADER_FORMAT = '<4sIIIIIII' HEADER_FORMAT = '<4sIII'
HEADER_SIZE = struct.calcsize(HEADER_FORMAT) HEADER_SIZE = struct.calcsize(HEADER_FORMAT)
DEFAULT_STORED_PATTERNS = ['locale/*']
def build_zip_blob(root, relative_paths, compression): def build_zip_blob(root, relative_paths):
buf = io.BytesIO() buf = io.BytesIO()
with zipfile.ZipFile(buf, 'w', compression=compression) as zf: with zipfile.ZipFile(buf, 'w', compression=zipfile.ZIP_STORED) as zf:
for relative_path in sorted(relative_paths): for relative_path in sorted(relative_paths):
zf.write(os.path.join(root, relative_path), arcname=relative_path) zf.write(os.path.join(root, relative_path), arcname=relative_path)
return buf.getvalue() return buf.getvalue()
@@ -75,12 +67,12 @@ def manifest_path(output_path):
return output_path + '.manifest.json' return output_path + '.manifest.json'
def compute_manifest(input_dir, relative_paths, stored_patterns): def compute_manifest(input_dir, relative_paths):
entries = {} entries = {}
for relative_path in relative_paths: for relative_path in relative_paths:
st = os.stat(os.path.join(input_dir, relative_path)) st = os.stat(os.path.join(input_dir, relative_path))
entries[relative_path] = [st.st_mtime_ns, st.st_size] entries[relative_path] = [st.st_mtime_ns, st.st_size]
return {'stored_patterns': sorted(stored_patterns), 'files': entries} return {'version': VERSION, 'files': entries}
def load_manifest(path): def load_manifest(path):
@@ -91,7 +83,7 @@ def load_manifest(path):
return None return None
def pack(input_dir, output_path, stored_patterns): def pack(input_dir, output_path):
relative_paths = [] relative_paths = []
for dirpath, _dirnames, filenames in os.walk(input_dir): for dirpath, _dirnames, filenames in os.walk(input_dir):
for filename in filenames: for filename in filenames:
@@ -100,52 +92,28 @@ def pack(input_dir, output_path, stored_patterns):
os.path.relpath(full_path, input_dir).replace(os.sep, '/') os.path.relpath(full_path, input_dir).replace(os.sep, '/')
) )
manifest = compute_manifest(input_dir, relative_paths, stored_patterns) manifest = compute_manifest(input_dir, relative_paths)
manifest_file = manifest_path(output_path) manifest_file = manifest_path(output_path)
if os.path.isfile(output_path) and load_manifest(manifest_file) == manifest: if os.path.isfile(output_path) and load_manifest(manifest_file) == manifest:
print(f'{output_path} is up to date, skipping pack') print(f'{output_path} is up to date, skipping pack')
return return
stored_paths = [ blob = build_zip_blob(input_dir, relative_paths)
path for path in relative_paths
if any(fnmatch.fnmatch(path, pattern) for pattern in stored_patterns)
]
compressed_paths = [
path for path in relative_paths if path not in stored_paths
]
compressed_blob = build_zip_blob(
input_dir, compressed_paths, zipfile.ZIP_DEFLATED
)
stored_blob = build_zip_blob(input_dir, stored_paths, zipfile.ZIP_STORED)
compressed_offset = HEADER_SIZE
stored_offset = compressed_offset + len(compressed_blob)
header = struct.pack( header = struct.pack(
HEADER_FORMAT, HEADER_FORMAT, MAGIC, VERSION, len(blob), zlib.crc32(blob) & 0xFFFFFFFF
MAGIC,
VERSION,
compressed_offset,
len(compressed_blob),
zlib.crc32(compressed_blob) & 0xFFFFFFFF,
stored_offset,
len(stored_blob),
zlib.crc32(stored_blob) & 0xFFFFFFFF,
) )
with open(output_path, 'wb') as f: with open(output_path, 'wb') as f:
f.write(header) f.write(header)
f.write(compressed_blob) f.write(blob)
f.write(stored_blob)
with open(manifest_file, 'w', encoding='utf-8') as f: with open(manifest_file, 'w', encoding='utf-8') as f:
json.dump(manifest, f) json.dump(manifest, f)
print( print(
f'Wrote {output_path}: {len(compressed_paths)} compressed file(s) ' f'Wrote {output_path}: {len(relative_paths)} file(s) '
f'({len(compressed_blob)} bytes), {len(stored_paths)} stored file(s) ' f'({len(blob)} bytes), {len(header) + len(blob)} bytes total'
f'({len(stored_blob)} bytes), {len(header) + len(compressed_blob) + len(stored_blob)} bytes total'
) )
@@ -155,17 +123,9 @@ def main():
) )
parser.add_argument('--input', required=True, help='Assets directory to pack') parser.add_argument('--input', required=True, help='Assets directory to pack')
parser.add_argument('--output', required=True, help='Path to write dusk.dsk to') parser.add_argument('--output', required=True, help='Path to write dusk.dsk to')
parser.add_argument(
'--stored',
action='append',
dest='stored_patterns',
help='fnmatch pattern (relative to --input) of files to store '
'uncompressed instead of compressing. May be given multiple '
'times. Defaults to "locale/*" if never given.'
)
args = parser.parse_args() args = parser.parse_args()
pack(args.input, args.output, args.stored_patterns or DEFAULT_STORED_PATTERNS) pack(args.input, args.output)
if __name__ == '__main__': if __name__ == '__main__':