diff --git a/CMakeLists.txt b/CMakeLists.txt index dfd74b4d..ce079434 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -128,8 +128,10 @@ add_custom_command( OUTPUT "${DUSK_ASSETS_ZIP}" COMMAND ${CMAKE_COMMAND} -E make_directory "${DUSK_ASSETS_DIR}" COMMAND ${CMAKE_COMMAND} -E rm -f "${DUSK_ASSETS_ZIP}" - COMMAND ${CMAKE_COMMAND} -E tar "cf" "${DUSK_ASSETS_ZIP}" --format=zip -- . - WORKING_DIRECTORY "${DUSK_ASSETS_DIR}" + COMMAND ${Python3_EXECUTABLE} -m tools.asset.pack + --input "${DUSK_ASSETS_DIR}" + --output "${DUSK_ASSETS_ZIP}" + WORKING_DIRECTORY "${DUSK_ROOT_DIR}" DEPENDS ${DUSK_ASSET_FILES} VERBATIM ) diff --git a/src/dusk/CMakeLists.txt b/src/dusk/CMakeLists.txt index d3f9e31c..90af7a3b 100644 --- a/src/dusk/CMakeLists.txt +++ b/src/dusk/CMakeLists.txt @@ -14,6 +14,13 @@ if(NOT libzip_FOUND) target_link_libraries(${DUSK_LIBRARY_TARGET_NAME} PUBLIC zip) endif() +# assetdsk.c calls crc32() directly (to verify dusk.dsk archive checksums). +# find_package(libzip) above already resolves ZLIB as a side effect, so +# ZLIB_FOUND may already be true without ZLIB::ZLIB having been linked to +# our target - link it unconditionally rather than guarding on ZLIB_FOUND. +find_package(ZLIB REQUIRED) +target_link_libraries(${DUSK_LIBRARY_TARGET_NAME} PUBLIC ZLIB::ZLIB) + if(NOT stb_image_FOUND) find_package(stb REQUIRED) if(STB_IMAGE_FOUND) diff --git a/src/dusk/asset/CMakeLists.txt b/src/dusk/asset/CMakeLists.txt index 39e32cab..a3d838a4 100644 --- a/src/dusk/asset/CMakeLists.txt +++ b/src/dusk/asset/CMakeLists.txt @@ -7,6 +7,7 @@ target_sources(${DUSK_LIBRARY_TARGET_NAME} PUBLIC asset.c + assetdsk.c assetfile.c ) diff --git a/src/dusk/asset/asset.c b/src/dusk/asset/asset.c index bcb33dfc..0ef0a6dd 100644 --- a/src/dusk/asset/asset.c +++ b/src/dusk/asset/asset.c @@ -23,9 +23,11 @@ errorret_t assetInit(void) { threadMutexInit(&ASSET.loading[i].mutex); } - // assetInitPlatform must either define ASSET.zip or throw an error. + // assetInitPlatform must either define both ASSET.zip/ASSET.zipStored or + // throw an error. errorChain(assetInitPlatform()); assertNotNull(ASSET.zip, "Asset zip null without error."); + assertNotNull(ASSET.zipStored, "Asset stored zip null without error."); threadInit(&ASSET.loadThread, assetUpdateAsync); threadStart(&ASSET.loadThread); @@ -35,9 +37,9 @@ errorret_t assetInit(void) { bool_t assetFileExists(const char_t *filename) { assertStrLenMax(filename, ASSET_FILE_NAME_MAX, "Filename too long."); - zip_int64_t idx = zip_name_locate(ASSET.zip, filename, 0); - if(idx < 0) return false; - return true; + if(zip_name_locate(ASSET.zip, filename, 0) >= 0) return true; + if(zip_name_locate(ASSET.zipStored, filename, 0) >= 0) return true; + return false; } assetentry_t * assetGetEntry( @@ -427,13 +429,19 @@ errorret_t assetDispose(void) { errorChain(assetReapUnused()); - // Cleanup zip file. + // Cleanup zip files. if(ASSET.zip != NULL) { if(zip_close(ASSET.zip) != 0) { - errorThrow("Failed to close asset zip archive."); + errorThrow("Failed to close compressed asset zip archive."); } ASSET.zip = NULL; } + if(ASSET.zipStored != NULL) { + if(zip_close(ASSET.zipStored) != 0) { + errorThrow("Failed to close stored asset zip archive."); + } + ASSET.zipStored = NULL; + } errorChain(assetDisposePlatform()); errorOk(); diff --git a/src/dusk/asset/asset.h b/src/dusk/asset/asset.h index ed09a854..0aa9ed72 100644 --- a/src/dusk/asset/asset.h +++ b/src/dusk/asset/asset.h @@ -27,7 +27,16 @@ #define ASSET_ENTRY_COUNT_MAX 64 typedef struct asset_s { + // Compressed (DEFLATE) archive - expected to hold the bulk of a game's + // binary assets. Looked up first by assetFileInit(). zip_t *zip; + + // Stored (uncompressed) archive - expected to hold small files that need + // reliable repeated seeking/re-opening (e.g. locale strings), which + // libzip only supports reliably for uncompressed entries. Looked up by + // assetFileInit() only if the name isn't found in `zip`. + zip_t *zipStored; + assetplatform_t platform; // Background loading thread. diff --git a/src/dusk/asset/assetdsk.c b/src/dusk/asset/assetdsk.c new file mode 100644 index 00000000..bc8fc374 --- /dev/null +++ b/src/dusk/asset/assetdsk.c @@ -0,0 +1,241 @@ +/** + * Copyright (c) 2026 Dominic Masters + * + * This software is released under the MIT License. + * https://opensource.org/licenses/MIT + */ + +#include "assetdsk.h" +#include "util/memory.h" +#include "util/endian.h" +#include "assert/assert.h" +#include + +errorret_t assetDskParseHeader( + const uint8_t *bytes, + const size_t bytesSize, + assetdskheader_t *outHeader +) { + assertNotNull(bytes, "Bytes cannot be NULL."); + assertNotNull(outHeader, "Out header cannot be NULL."); + + if(bytesSize < ASSET_DSK_HEADER_SIZE) { + errorThrow("dusk.dsk header is truncated."); + } + + if(memoryCompare(bytes, ASSET_DSK_MAGIC, ASSET_DSK_MAGIC_SIZE) != 0) { + errorThrow("dusk.dsk has an invalid magic header."); + } + + // Every field is a little-endian uint32_t regardless of host - convert + // to host order (a no-op on little-endian hosts, a real byteswap on + // Dolphin's big-endian PowerPC). + uint32_t fields[7]; + memoryCopy(fields, bytes + ASSET_DSK_MAGIC_SIZE, sizeof(fields)); + for(uint8_t i = 0; i < 7; i++) { + fields[i] = endianLittleToHost32(fields[i]); + } + + const uint32_t version = fields[0]; + if(version != ASSET_DSK_VERSION) { + errorThrow("dusk.dsk has an unsupported version: %u", version); + } + + outHeader->compressedOffset = fields[1]; + outHeader->compressedSize = fields[2]; + outHeader->compressedChecksum = fields[3]; + outHeader->storedOffset = fields[4]; + outHeader->storedSize = fields[5]; + outHeader->storedChecksum = fields[6]; + + errorOk(); +} + +errorret_t assetDskOpenFromPath( + const char_t *path, + zip_t **outCompressed, + zip_t **outStored +) { + assertNotNull(path, "Path cannot be NULL."); + assertNotNull(outCompressed, "Out compressed cannot be NULL."); + assertNotNull(outStored, "Out stored cannot be NULL."); + + *outCompressed = NULL; + *outStored = NULL; + + FILE *headerFile = fopen(path, "rb"); + if(headerFile == NULL) { + errorThrow("Failed to open dusk.dsk: %s", path); + } + + uint8_t headerBytes[ASSET_DSK_HEADER_SIZE]; + size_t headerRead = fread(headerBytes, 1, sizeof(headerBytes), headerFile); + fclose(headerFile); + if(headerRead != sizeof(headerBytes)) { + errorThrow("Failed to read dusk.dsk header: %s", path); + } + + assetdskheader_t header; + errorChain(assetDskParseHeader(headerBytes, sizeof(headerBytes), &header)); + + zip_error_t zipError; + zip_error_init(&zipError); + + zip_source_t *compressedSource = zip_source_file_create( + path, header.compressedOffset, (zip_int64_t) header.compressedSize, &zipError + ); + if(compressedSource == NULL) { + errorThrow( + "Failed to create compressed dusk.dsk source: %s", zip_error_strerror(&zipError) + ); + } + + *outCompressed = zip_open_from_source(compressedSource, ZIP_RDONLY, &zipError); + if(*outCompressed == NULL) { + zip_source_free(compressedSource); + errorThrow( + "Failed to open compressed dusk.dsk archive: %s", zip_error_strerror(&zipError) + ); + } + + zip_source_t *storedSource = zip_source_file_create( + path, header.storedOffset, (zip_int64_t) header.storedSize, &zipError + ); + if(storedSource == NULL) { + zip_close(*outCompressed); + *outCompressed = NULL; + errorThrow( + "Failed to create stored dusk.dsk source: %s", zip_error_strerror(&zipError) + ); + } + + *outStored = zip_open_from_source(storedSource, ZIP_RDONLY, &zipError); + if(*outStored == NULL) { + zip_source_free(storedSource); + zip_close(*outCompressed); + *outCompressed = NULL; + errorThrow( + "Failed to open stored dusk.dsk archive: %s", zip_error_strerror(&zipError) + ); + } + + // The stored archive is small by convention, so verifying its checksum + // here (one extra small read) is cheap; the compressed archive isn't + // checked since it's meant to be read lazily/on-demand from here on. + uint8_t *storedBytes = (uint8_t *) memoryAllocate(header.storedSize); + FILE *storedFile = fopen(path, "rb"); + if(storedFile == NULL) { + memoryFree(storedBytes); + zip_close(*outStored); + zip_close(*outCompressed); + *outStored = NULL; + *outCompressed = NULL; + errorThrow("Failed to re-open dusk.dsk to verify stored checksum: %s", path); + } + fseek(storedFile, (long) header.storedOffset, SEEK_SET); + size_t storedRead = fread(storedBytes, 1, header.storedSize, storedFile); + fclose(storedFile); + if(storedRead != header.storedSize) { + memoryFree(storedBytes); + zip_close(*outStored); + zip_close(*outCompressed); + *outStored = NULL; + *outCompressed = NULL; + errorThrow("Failed to read dusk.dsk stored archive to verify checksum: %s", path); + } + + uint32_t checksum = (uint32_t) crc32(0L, storedBytes, (uInt) header.storedSize); + memoryFree(storedBytes); + if(checksum != header.storedChecksum) { + zip_close(*outStored); + zip_close(*outCompressed); + *outStored = NULL; + *outCompressed = NULL; + errorThrow("dusk.dsk stored archive failed checksum verification: %s", path); + } + + errorOk(); +} + +errorret_t assetDskOpenFromBuffer( + uint8_t *buffer, + const size_t bufferSize, + zip_t **outCompressed, + zip_t **outStored +) { + assertNotNull(buffer, "Buffer cannot be NULL."); + assertNotNull(outCompressed, "Out compressed cannot be NULL."); + assertNotNull(outStored, "Out stored cannot be NULL."); + + *outCompressed = NULL; + *outStored = NULL; + + assetdskheader_t header; + errorChain(assetDskParseHeader(buffer, bufferSize, &header)); + + if( + (size_t) header.compressedOffset + header.compressedSize > bufferSize || + (size_t) header.storedOffset + header.storedSize > bufferSize + ) { + errorThrow("dusk.dsk header describes ranges beyond the buffer."); + } + + uint32_t compressedChecksum = (uint32_t) crc32( + 0L, buffer + header.compressedOffset, (uInt) header.compressedSize + ); + if(compressedChecksum != header.compressedChecksum) { + errorThrow("dusk.dsk compressed archive failed checksum verification."); + } + + uint32_t storedChecksum = (uint32_t) crc32( + 0L, buffer + header.storedOffset, (uInt) header.storedSize + ); + if(storedChecksum != header.storedChecksum) { + errorThrow("dusk.dsk stored archive failed checksum verification."); + } + + zip_error_t zipError; + zip_error_init(&zipError); + + // freep=0 for both - they're non-owning windows into the same caller-owned + // buffer, not independent allocations libzip should free. + zip_source_t *compressedSource = zip_source_buffer_create( + buffer + header.compressedOffset, header.compressedSize, 0, &zipError + ); + if(compressedSource == NULL) { + errorThrow( + "Failed to create compressed dusk.dsk source: %s", zip_error_strerror(&zipError) + ); + } + + *outCompressed = zip_open_from_source(compressedSource, ZIP_RDONLY, &zipError); + if(*outCompressed == NULL) { + zip_source_free(compressedSource); + errorThrow( + "Failed to open compressed dusk.dsk archive: %s", zip_error_strerror(&zipError) + ); + } + + zip_source_t *storedSource = zip_source_buffer_create( + buffer + header.storedOffset, header.storedSize, 0, &zipError + ); + if(storedSource == NULL) { + zip_close(*outCompressed); + *outCompressed = NULL; + errorThrow( + "Failed to create stored dusk.dsk source: %s", zip_error_strerror(&zipError) + ); + } + + *outStored = zip_open_from_source(storedSource, ZIP_RDONLY, &zipError); + if(*outStored == NULL) { + zip_source_free(storedSource); + zip_close(*outCompressed); + *outCompressed = NULL; + errorThrow( + "Failed to open stored dusk.dsk archive: %s", zip_error_strerror(&zipError) + ); + } + + errorOk(); +} diff --git a/src/dusk/asset/assetdsk.h b/src/dusk/asset/assetdsk.h new file mode 100644 index 00000000..055a005f --- /dev/null +++ b/src/dusk/asset/assetdsk.h @@ -0,0 +1,101 @@ +/** + * Copyright (c) 2026 Dominic Masters + * + * This software is released under the MIT License. + * https://opensource.org/licenses/MIT + */ + +#pragma once +#include "error/error.h" +#include + +// "DSK2" - distinct from a plain zip's "PK\x03\x04" so a stray plain zip +// never gets misread as a valid dusk.dsk. +#define ASSET_DSK_MAGIC_SIZE 4 +#define ASSET_DSK_MAGIC "DSK2" +#define ASSET_DSK_VERSION 1 + +// magic(4) + version(4) + compressedOffset(4) + compressedSize(4) + +// compressedChecksum(4) + storedOffset(4) + storedSize(4) + +// storedChecksum(4), all little-endian regardless of host - see +// assetDskParseHeader. +#define ASSET_DSK_HEADER_SIZE 32 + +/** + * Parsed dusk.dsk (DSK2 format) header. dusk.dsk is two independent, back + * to back zip archives (see tools/asset/pack) rather than a single plain + * zip: a "compressed" one (DEFLATE, expected to hold the bulk of a game's + * binary assets, opened lazily/on-demand) and a "stored" one (uncompressed, + * expected to hold small files - locale strings, config - that need + * reliable repeated seeking/re-opening, which libzip only supports for + * uncompressed entries). + */ +typedef struct { + uint32_t compressedOffset; + uint32_t compressedSize; + uint32_t compressedChecksum; + uint32_t storedOffset; + uint32_t storedSize; + uint32_t storedChecksum; +} assetdskheader_t; + +/** + * Parses a DSK2 header from a raw byte buffer (at least + * ASSET_DSK_HEADER_SIZE bytes), validating the magic/version and + * byte-swapping the little-endian fields to host order. + * + * @param bytes Buffer containing the header (and beyond). + * @param bytesSize Number of bytes available at `bytes`. + * @param outHeader Filled with the parsed header on success. + * @return OK on success, error if too short, bad magic, or unsupported version. + */ +errorret_t assetDskParseHeader( + const uint8_t *bytes, + const size_t bytesSize, + assetdskheader_t *outHeader +); + +/** + * Opens both archives of a dusk.dsk file given a filesystem path, using + * lazy/windowed file-backed zip sources - no more memory used than the + * existing per-platform small read buffers, matching the memory + * characteristics of a plain zip_open() on the whole file. Verifies the + * (small, by convention) stored archive's checksum; the compressed + * archive's checksum is intentionally not verified here since doing so + * would require reading the bulk of the game's assets just to compute it. + * + * @param path Filesystem path to the dusk.dsk file. + * @param outCompressed Set to the opened compressed archive on success. + * @param outStored Set to the opened stored archive on success. + * @return OK on success, error if the file is missing, too short, has a + * bad header, or either archive fails to open/verify. + */ +errorret_t assetDskOpenFromPath( + const char_t *path, + zip_t **outCompressed, + zip_t **outStored +); + +/** + * Opens both archives of a dusk.dsk file already fully resident in memory + * (e.g. a PSAR embedded in an EBOOT.PBP, or an ISO-embedded file already + * read via DVD_ReadAbs - platforms that already buffer the whole file for + * reasons unrelated to this format). Neither archive takes ownership of + * `buffer` (both are opened as non-owning sub-ranges of it) - the caller + * remains responsible for freeing it, and must keep it alive for as long + * as either archive stays open. Verifies both archives' checksums, since + * the bytes are already resident. + * + * @param buffer The whole dusk.dsk file's bytes. + * @param bufferSize Number of bytes at `buffer`. + * @param outCompressed Set to the opened compressed archive on success. + * @param outStored Set to the opened stored archive on success. + * @return OK on success, error if too short, has a bad header/checksum, or + * either archive fails to open. + */ +errorret_t assetDskOpenFromBuffer( + uint8_t *buffer, + const size_t bufferSize, + zip_t **outCompressed, + zip_t **outStored +); diff --git a/src/dusk/asset/assetfile.c b/src/dusk/asset/assetfile.c index c6dd77a6..19adfa3d 100644 --- a/src/dusk/asset/assetfile.c +++ b/src/dusk/asset/assetfile.c @@ -24,9 +24,15 @@ errorret_t assetFileInit( file->params = params; file->output = output; - // Stat the file + // Stat the file, trying the compressed archive first and falling back to + // the stored one - remember which matched so assetFileOpen opens it from + // the right archive. zip_stat_init(&file->stat); - if(!zip_stat(ASSET.zip, filename, 0, &file->stat) == 0) { + if(zip_stat(ASSET.zip, filename, 0, &file->stat) == 0) { + file->sourceZip = ASSET.zip; + } else if(zip_stat(ASSET.zipStored, filename, 0, &file->stat) == 0) { + file->sourceZip = ASSET.zipStored; + } else { errorThrow("Failed to stat asset file: %s", filename); } @@ -47,6 +53,21 @@ errorret_t assetFileRewind(assetfile_t *file) { errorOk(); } + // Prefer seeking within the still-open handle over closing and + // re-opening it. Repeatedly closing/re-opening the same compressed zip + // entry (once per rewind - e.g. once per locale string lookup) was + // confirmed unreliable on at least one platform's zip backend, silently + // skipping ranges of the decompressed content on some re-opens after + // the first. A real seek avoids that failure mode entirely, with no + // extra memory cost over the close+reopen fallback. + if(zip_file_is_seekable(file->zipFile)) { + if(zip_fseek(file->zipFile, 0, SEEK_SET) != 0) { + errorThrow("Failed to seek asset file: %s", file->filename); + } + file->position = 0; + errorOk(); + } + errorChain(assetFileClose(file)); errorChain(assetFileOpen(file)); errorOk(); @@ -55,10 +76,10 @@ errorret_t assetFileRewind(assetfile_t *file) { errorret_t assetFileOpen(assetfile_t *file) { assertNotNull(file, "Asset file cannot be NULL."); assertNotNull(file->filename, "Asset file filename cannot be NULL."); - assertNotNull(ASSET.zip, "Asset zip cannot be NULL."); + assertNotNull(file->sourceZip, "Asset file must be inited before opening."); assertNull(file->zipFile, "Asset file already open."); - file->zipFile = zip_fopen(ASSET.zip, file->filename, 0); + file->zipFile = zip_fopen(file->sourceZip, file->filename, 0); if(file->zipFile == NULL) { errorThrow("Failed to open asset file: %s", file->filename); } diff --git a/src/dusk/asset/assetfile.h b/src/dusk/asset/assetfile.h index a64e0aa6..c013521b 100644 --- a/src/dusk/asset/assetfile.h +++ b/src/dusk/asset/assetfile.h @@ -32,6 +32,11 @@ typedef struct assetfile_s { zip_int64_t position; zip_int64_t lastRead; zip_file_t *zipFile; + + // The archive this file was found in (ASSET.zip or ASSET.zipStored), + // set by assetFileInit and used by assetFileOpen so lookups fall back + // correctly between the two dusk.dsk archives. + zip_t *sourceZip; } assetfile_t; /** diff --git a/src/duskdolphin/asset/assetdolphindvd.c b/src/duskdolphin/asset/assetdolphindvd.c index 2670cf79..19d6ea67 100644 --- a/src/duskdolphin/asset/assetdolphindvd.c +++ b/src/duskdolphin/asset/assetdolphindvd.c @@ -7,6 +7,7 @@ #include "assetdolphindvd.h" #include "asset/asset.h" +#include "asset/assetdsk.h" #include "util/string.h" #include "util/memory.h" @@ -90,19 +91,15 @@ errorret_t assetInitDolphinDVD(void) { ); if(!data) errorThrow("Failed to read asset file from ISO."); - zip_error_t zerr; - zip_source_t *src = zip_source_buffer_create(data, fileSize, 1, &zerr); - if(!src) { + errorret_t ret = assetDskOpenFromBuffer( + data, fileSize, &ASSET.zip, &ASSET.zipStored + ); + if(errorIsNotOk(ret)) { memoryFree(data); - errorThrow("Failed to create zip source from DVD buffer."); - } - - ASSET.zip = zip_open_from_source(src, ZIP_RDONLY, &zerr); - if(!ASSET.zip) { - zip_source_free(src); - errorThrow("Failed to open asset zip from DVD."); + errorChain(ret); } + ASSET.platform.dskData = data; errorOk(); } @@ -124,5 +121,9 @@ u32 assetDolphinDVDReadBigEndian32(const u8 *p) { } errorret_t assetDisposeDolphinDVD(void) { + if(ASSET.platform.dskData != NULL) { + memoryFree(ASSET.platform.dskData); + ASSET.platform.dskData = NULL; + } errorOk(); } diff --git a/src/duskdolphin/asset/assetdolphindvd.h b/src/duskdolphin/asset/assetdolphindvd.h index 883a290b..9e5154eb 100644 --- a/src/duskdolphin/asset/assetdolphindvd.h +++ b/src/duskdolphin/asset/assetdolphindvd.h @@ -24,7 +24,10 @@ (((u32)(n) + ASSET_DOLPHIN_DVD_ALIGN - 1u) & ~(ASSET_DOLPHIN_DVD_ALIGN - 1u)) typedef struct { - uint8_t nothing; + // Whole dusk.dsk blob read from the ISO, kept alive for as long as + // ASSET.zip/ASSET.zipStored are open since they're non-owning windows + // into it (see assetDskOpenFromBuffer). Freed in assetDisposeDolphinDVD. + uint8_t *dskData; } assetdolphindvd_t; /** diff --git a/src/duskdolphin/asset/assetdolphinfat.c b/src/duskdolphin/asset/assetdolphinfat.c index f41002fd..577597dc 100644 --- a/src/duskdolphin/asset/assetdolphinfat.c +++ b/src/duskdolphin/asset/assetdolphinfat.c @@ -7,6 +7,7 @@ #include "assetdolphinfat.h" #include "asset/asset.h" +#include "asset/assetdsk.h" #include "util/string.h" #include #include @@ -50,10 +51,7 @@ errorret_t assetInitDolphinFAT(void) { if(foundPath[0] == '\0') errorThrow("Failed to find asset file on FAT filesystem."); - ASSET.zip = zip_open(foundPath, ZIP_RDONLY, NULL); - if(ASSET.zip == NULL) - errorThrow("Failed to open asset file on FAT filesystem."); - + errorChain(assetDskOpenFromPath(foundPath, &ASSET.zip, &ASSET.zipStored)); errorOk(); } diff --git a/src/dusklinux/asset/assetlinux.c b/src/dusklinux/asset/assetlinux.c index 57b790e7..1f761d14 100644 --- a/src/dusklinux/asset/assetlinux.c +++ b/src/dusklinux/asset/assetlinux.c @@ -6,6 +6,7 @@ */ #include "asset/asset.h" +#include "asset/assetdsk.h" #include "engine/engine.h" #include "util/string.h" #include "assert/assert.h" @@ -45,10 +46,10 @@ errorret_t assetInitLinux(void) { stringCopy(ASSET.platform.systemPath, ".", ASSET_SYSTEM_PATH_MAX); } - // Open zip file + // Open dusk.dsk char_t searchPath[ASSET_SYSTEM_PATH_MAX]; const char_t **path = ASSET_LINUX_SEARCH_PATHS; - int32_t error; + bool_t opened = false; do { char_t temp[ASSET_SYSTEM_PATH_MAX]; snprintf( @@ -75,17 +76,18 @@ errorret_t assetInitLinux(void) { printf("Try open asset file: %s\n", searchPath); // Try open - error = 0; - ASSET.zip = zip_open(searchPath, ZIP_RDONLY, &error); - if(ASSET.zip == NULL) { - printf("Opened asset file with non-zero error code: %d\n", error); + if(errorIsNotOk( + assetDskOpenFromPath(searchPath, &ASSET.zip, &ASSET.zipStored) + )) { + printf("Failed to open asset file: %s\n", searchPath); continue; } + opened = true; break;// Found! } while(*(++path) != NULL); // Did we open the asset? - if(ASSET.zip == NULL) { + if(!opened) { errorThrow("Failed to open asset file."); } diff --git a/src/duskpsp/asset/assetpbp.c b/src/duskpsp/asset/assetpbp.c index 1aeb3669..cb472d58 100644 --- a/src/duskpsp/asset/assetpbp.c +++ b/src/duskpsp/asset/assetpbp.c @@ -6,6 +6,7 @@ */ #include "asset/asset.h" +#include "asset/assetdsk.h" #include "assert/assert.h" #include "util/memory.h" #include "util/math.h" @@ -109,24 +110,15 @@ errorret_t assetInitPBP(const char_t *pbpPath) { fclose(ASSET.platform.pbpFile); ASSET.platform.pbpFile = NULL; - zip_source_t *psarSource = zip_source_buffer_create( - psarData, (zip_uint64_t)psarSize, 1, NULL + errorret_t ret = assetDskOpenFromBuffer( + psarData, psarSize, &ASSET.zip, &ASSET.zipStored ); - if(psarSource == NULL) { + if(errorIsNotOk(ret)) { free(psarData); - errorThrow("Failed to create zip source in PBP file: %s", pbpPath); - } - - ASSET.zip = zip_open_from_source( - psarSource, - ZIP_RDONLY, - NULL - ); - if(ASSET.zip == NULL) { - zip_source_free(psarSource); - errorThrow("Failed to open zip from PBP file: %s", pbpPath); + errorChain(ret); } + ASSET.platform.dskData = psarData; errorOk(); } @@ -136,5 +128,10 @@ errorret_t assetDisposePBP(void) { ASSET.platform.pbpFile = NULL; } + if(ASSET.platform.dskData != NULL) { + free(ASSET.platform.dskData); + ASSET.platform.dskData = NULL; + } + errorOk(); } \ No newline at end of file diff --git a/src/duskpsp/asset/assetpbp.h b/src/duskpsp/asset/assetpbp.h index f9ac848d..a6e7b819 100644 --- a/src/duskpsp/asset/assetpbp.h +++ b/src/duskpsp/asset/assetpbp.h @@ -30,6 +30,11 @@ typedef struct { typedef struct { FILE *pbpFile; assetpbpheader_t pbpHeader; + + // Whole dusk.dsk (PSAR) blob, kept alive for as long as ASSET.zip/ + // ASSET.zipStored are open since they're non-owning windows into it (see + // assetDskOpenFromBuffer). Freed in assetDisposePBP. + uint8_t *dskData; } assetpbp_t; /** diff --git a/src/duskvita/asset/assetvita.c b/src/duskvita/asset/assetvita.c index 667b2445..4dcd1d6a 100644 --- a/src/duskvita/asset/assetvita.c +++ b/src/duskvita/asset/assetvita.c @@ -6,14 +6,13 @@ */ #include "asset/asset.h" +#include "asset/assetdsk.h" #include "assert/assert.h" errorret_t assetInitVita(void) { - int32_t error; - ASSET.zip = zip_open(ASSET_VITA_DSK_PATH, ZIP_RDONLY, &error); - if(ASSET.zip == NULL) { - errorThrow("Failed to open asset file: " ASSET_VITA_DSK_PATH); - } + errorChain( + assetDskOpenFromPath(ASSET_VITA_DSK_PATH, &ASSET.zip, &ASSET.zipStored) + ); errorOk(); } diff --git a/tools/asset/pack/__main__.py b/tools/asset/pack/__main__.py new file mode 100644 index 00000000..cf2275be --- /dev/null +++ b/tools/asset/pack/__main__.py @@ -0,0 +1,132 @@ +# Copyright (c) 2026 Dominic Masters +# +# This software is released under the MIT License. +# https://opensource.org/licenses/MIT + +""" +Packs the assets directory into a dusk.dsk (DSK2) archive. + +Replaces a single plain zip with two independent zip archives back to +back, so specific files can be stored uncompressed instead of DEFLATEd - +useful for small files (locale strings, config) accessed at runtime via +repeated seeks/re-opens, which is only reliably supported by libzip for +uncompressed (STORED) entries. Reading a whole compressed archive into +memory just to get reliable access isn't viable once that archive holds +the bulk of a game's binary assets, so the two archives are kept +separate and only the (small, by convention) stored one is expected to +ever need to be buffered whole. + +DSK2 format: + Bytes 0-3: "DSK2" (magic) + Bytes 4-7: uint32_t version = 1 (little-endian) + Bytes 8-11: uint32_t compressedOffset + Bytes 12-15: uint32_t compressedSize + Bytes 16-19: uint32_t compressedChecksum (CRC32 of the compressed blob) + Bytes 20-23: uint32_t storedOffset + Bytes 24-27: uint32_t storedSize + Bytes 28-31: uint32_t storedChecksum (CRC32 of the stored blob) + Bytes 32+: compressed zip blob, then stored zip blob (each a + complete, independently valid zip archive) + +Usage: + python3 -m tools.asset.pack --input --output + [--stored ]... + Every file under is added, using its path relative to + as the zip entry name, to the stored archive if its + relative path matches any --stored pattern (fnmatch, default: + "locale/*"), otherwise to the compressed archive. +""" + +import argparse +import os +import struct +import zipfile +import zlib +import fnmatch +import io + +MAGIC = b'DSK2' +VERSION = 1 +HEADER_FORMAT = '<4sIIIIIII' +HEADER_SIZE = struct.calcsize(HEADER_FORMAT) +DEFAULT_STORED_PATTERNS = ['locale/*'] + + +def build_zip_blob(root, relative_paths, compression): + buf = io.BytesIO() + with zipfile.ZipFile(buf, 'w', compression=compression) as zf: + for relative_path in sorted(relative_paths): + zf.write(os.path.join(root, relative_path), arcname=relative_path) + return buf.getvalue() + + +def pack(input_dir, output_path, stored_patterns): + relative_paths = [] + for dirpath, _dirnames, filenames in os.walk(input_dir): + for filename in filenames: + full_path = os.path.join(dirpath, filename) + relative_paths.append( + os.path.relpath(full_path, input_dir).replace(os.sep, '/') + ) + + stored_paths = [ + path for path in relative_paths + if any(fnmatch.fnmatch(path, pattern) for pattern in stored_patterns) + ] + compressed_paths = [ + path for path in relative_paths if path not in stored_paths + ] + + compressed_blob = build_zip_blob( + input_dir, compressed_paths, zipfile.ZIP_DEFLATED + ) + stored_blob = build_zip_blob(input_dir, stored_paths, zipfile.ZIP_STORED) + + compressed_offset = HEADER_SIZE + stored_offset = compressed_offset + len(compressed_blob) + + header = struct.pack( + HEADER_FORMAT, + MAGIC, + VERSION, + compressed_offset, + len(compressed_blob), + zlib.crc32(compressed_blob) & 0xFFFFFFFF, + stored_offset, + len(stored_blob), + zlib.crc32(stored_blob) & 0xFFFFFFFF, + ) + + with open(output_path, 'wb') as f: + f.write(header) + f.write(compressed_blob) + f.write(stored_blob) + + print( + f'Wrote {output_path}: {len(compressed_paths)} compressed file(s) ' + f'({len(compressed_blob)} bytes), {len(stored_paths)} stored file(s) ' + f'({len(stored_blob)} bytes), {len(header) + len(compressed_blob) + len(stored_blob)} bytes total' + ) + + +def main(): + parser = argparse.ArgumentParser( + description='Pack the assets directory into a dusk.dsk (DSK2) archive.' + ) + parser.add_argument('--input', required=True, help='Assets directory to pack') + parser.add_argument('--output', required=True, help='Path to write dusk.dsk to') + parser.add_argument( + '--stored', + action='append', + dest='stored_patterns', + help='fnmatch pattern (relative to --input) of files to store ' + 'uncompressed instead of compressing. May be given multiple ' + 'times. Defaults to "locale/*" if never given.' + ) + args = parser.parse_args() + + pack(args.input, args.output, args.stored_patterns or DEFAULT_STORED_PATTERNS) + + +if __name__ == '__main__': + main()