YourWishes/dusk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Chunks as json

Browse Source
This commit is contained in:
Dominic Masters
2026-07-11 17:43:26 -05:00
parent 60dfb89b53
commit 9551e222dc
39 changed files with 1949 additions and 13252 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/asset/chunk/__main__.py
+55 -151
View File
@@ -4,8 +4,7 @@
# https://opensource.org/licenses/MIT
"""
Generates DCF + companion DMF files from raw chunk JSON files, or upgrades
legacy DCF files (version 1 or 2) to the current version.
Generates chunk JSON + companion DMF mesh files from raw chunk JSON files.
JSON input (assetsraw/<map>/chunks/chunk_X_Y_Z.json, one subdirectory per
map - e.g. assetsraw/overworld/chunks/):
@@ -21,22 +20,28 @@ map - e.g. assetsraw/overworld/chunks/):
Tiles absent from the array default to TILE_SHAPE_NULL. Each (x, y)
column may only have ONE real tile - chunks store one tile per column,
not one per (x, y, z), and that tile records its own local z (see the
version 4 tile format below). If two entries share the same (x, y) with
chunk JSON format below). If two entries share the same (x, y) with
different z, the later one in the array wins and a warning is printed.
Mesh files are located by searching under assets/meshes/ and referenced by
path from the assets root in the DCF.
path from the assets root in the output chunk JSON.
Output DCF is derived automatically, preserving the map subdirectory:
assetsraw/<map>/chunks/chunk_X_Y_Z.json -> assets/<map>/chunks/X_Y_Z.dcf
Output chunk JSON is derived automatically, preserving the map
subdirectory:
assetsraw/<map>/chunks/chunk_X_Y_Z.json -> assets/<map>/chunks/X_Y_Z.json
Version 4 DCF format (after 8-byte header):
tile_t tiles[CHUNK_WIDTH * CHUNK_HEIGHT] (one per x/y column)
each tile: uint32_t shape, uint8_t z, 3 padding bytes (8 bytes total,
matching the C tile_t struct's layout: { tileshape_t shape; uint8_t z; })
uint8_t meshCount
for each model:
null-terminated string (relative asset path to .json model)
float32[3] (x, y, z offset)
Chunk JSON format (loaded at runtime by assetChunkLoaderSync):
{
"tiles": [ [shape, z], ... ] // exactly CHUNK_WIDTH * CHUNK_HEIGHT
// entries, one per (x, y) column,
// x-major (matches tile_t: { shape, z })
"meshes": [
{ "model": "<path to .json model, relative to assets/>",
"offset": [x, y, z] }
]
}
By convention mesh index 0 (when present) is the chunk's auto-generated
terrain and the rest are props (see sceneOverworldDrawChunksBase/Props
on the C side).
DMF format:
Bytes 0-3: DMF\\x00
@@ -48,7 +53,6 @@ DMF format:
Usage:
python3 -m tools.asset.chunk # process all assetsraw/*/chunks/*.json
python3 -m tools.asset.chunk <file.json> # process one JSON file
python3 -m tools.asset.chunk <file.dcf> # upgrade legacy (v1/v2) DCF in-place
"""
import json
@@ -76,20 +80,13 @@ WORLD_LAYER_HEIGHT = 1.0 / math.sqrt(2)
CHUNK_MESH_COUNT_MAX = 10
CHUNK_MESH_NAME_MAX = 64
# Matches sizeof(tile_t) on the C side: uint32_t shape + uint8_t z, padded
# to 8 bytes ({ tileshape_t shape; uint8_t z; } with 4-byte enum alignment).
# Internal working format for a column's (shape, z) pair, packed/unpacked
# with struct purely so build_terrain_verts can index into it uniformly -
# unrelated to the on-disk format, which is JSON.
TILE_STRUCT_FORMAT = '<IB3x'
TILE_SIZE = struct.calcsize(TILE_STRUCT_FORMAT)
VERTEX_SIZE = 20
# Legacy (v1/v2) DCFs stored one uint32 shape per (x, y, z) triple across
# the full 3D grid, at whatever CHUNK_DEPTH was in effect when they were
# written (4). Frozen here, independent of the live CHUNK_DEPTH above, so
# a future depth change can't corrupt parsing of genuinely old files.
_LEGACY_CHUNK_DEPTH = 4
_LEGACY_TILE_SIZE = 4
_LEGACY_CHUNK_TILE_COUNT = CHUNK_WIDTH * CHUNK_HEIGHT * _LEGACY_CHUNK_DEPTH
TILE_SHAPE_NULL = 0
TILE_SHAPE_GROUND = 1
TILE_SHAPE_RAMP_NORTH = 2
@@ -105,9 +102,7 @@ TILE_SHAPE_RAMP_NORTHWEST_INNER = 11
TILE_SHAPE_RAMP_SOUTHEAST_INNER = 12
TILE_SHAPE_RAMP_SOUTHWEST_INNER = 13
FILE_MAGIC = b'DCF'
DMF_MAGIC = b'DMF\x00'
VERSION_OUT = 4
DMF_VERSION = 1
@@ -143,13 +138,13 @@ def write_model_json(path, mesh_rel, texture_rel, color):
print(f' Wrote model JSON {path}')
def derive_dcf_path(json_path):
"""assetsraw/<map>/chunks/chunk_X_Y_Z.json -> assets/<map>/chunks/X_Y_Z.dcf"""
def derive_chunk_json_path(json_path):
"""assetsraw/<map>/chunks/chunk_X_Y_Z.json -> assets/<map>/chunks/X_Y_Z.json"""
base = os.path.splitext(os.path.basename(json_path))[0]
if base.startswith('chunk_'):
base = base[len('chunk_'):]
rel_dir = os.path.relpath(os.path.dirname(os.path.abspath(json_path)), ASSETSRAW_DIR)
return os.path.join(ASSETS_DIR, rel_dir, base + '.dcf')
return os.path.join(ASSETS_DIR, rel_dir, base + '.json')
def write_dmf(path, vertex_bytes):
@@ -164,36 +159,36 @@ def write_dmf(path, vertex_bytes):
print(f' Wrote DMF {path}: {vert_count} vertices, {len(buf)} bytes')
def write_dcf(dcf_path, tiles, mesh_names, mesh_offsets=None):
"""Write a current-version DCF referencing the given DMF asset paths."""
def write_chunk_json(chunk_json_path, tiles, mesh_names, mesh_offsets=None):
"""Write a chunk JSON file referencing the given model asset paths."""
mesh_count = len(mesh_names)
if mesh_offsets is None:
mesh_offsets = [(0.0, 0.0, 0.0)] * mesh_count
buf = bytearray()
buf += FILE_MAGIC
buf += b'\x00'
buf += struct.pack('<I', VERSION_OUT)
buf += tiles
buf += struct.pack('<B', mesh_count)
tile_list = []
for i in range(CHUNK_TILE_COUNT):
shape, z = struct.unpack_from(TILE_STRUCT_FORMAT, tiles, i * TILE_SIZE)
tile_list.append([shape, z])
meshes = []
for name, offset in zip(mesh_names, mesh_offsets):
encoded = name.encode('ascii')
if len(encoded) >= CHUNK_MESH_NAME_MAX:
if len(name) >= CHUNK_MESH_NAME_MAX:
raise ValueError(
f"Mesh name too long (>= {CHUNK_MESH_NAME_MAX}): {name}"
)
buf += encoded + b'\x00'
buf += struct.pack('<3f', offset[0], offset[1], offset[2])
with open(dcf_path, 'wb') as f:
f.write(buf)
meshes.append({'model': name, 'offset': list(offset)})
obj = {'tiles': tile_list, 'meshes': meshes}
os.makedirs(os.path.dirname(chunk_json_path), exist_ok=True)
with open(chunk_json_path, 'w') as f:
json.dump(obj, f)
print(
f' Wrote DCF {dcf_path}: '
f'version {VERSION_OUT}, {mesh_count} mesh(es), {len(buf)} bytes'
f' Wrote chunk JSON {chunk_json_path}: {mesh_count} mesh(es)'
)
# ---------------------------------------------------------------------------
# JSON -> DCF + DMF
# JSON -> chunk JSON + DMF
# ---------------------------------------------------------------------------
def _tile_quad(u0, u1, v0, v1, fx, fy, sw_z, se_z, ne_z, nw_z):
@@ -264,7 +259,7 @@ def build_terrain_verts(tiles_bytes):
return bytes(buf)
def from_json(json_path, dcf_path):
def from_json(json_path, chunk_json_path):
with open(json_path) as f:
data = json.load(f)
@@ -289,7 +284,7 @@ def from_json(json_path, dcf_path):
model_names = []
mesh_offsets = []
dcf_base = os.path.splitext(os.path.basename(dcf_path))[0]
chunk_base = os.path.splitext(os.path.basename(chunk_json_path))[0]
# Terrain mesh + model (only written if non-empty)
terrain_dir = os.path.join(ASSETS_DIR, 'meshes', 'chunks')
@@ -297,10 +292,10 @@ def from_json(json_path, dcf_path):
os.makedirs(terrain_dir, exist_ok=True)
os.makedirs(models_dir, exist_ok=True)
if terrain_verts:
dmf_name = f'chunk_{dcf_base}_0.dmf'
dmf_name = f'chunk_{chunk_base}_0.dmf'
write_dmf(os.path.join(terrain_dir, dmf_name), terrain_verts)
mesh_rel = f'meshes/chunks/{dmf_name}'
json_name = f'chunk_{dcf_base}_0.json'
json_name = f'chunk_{chunk_base}_0.json'
write_model_json(
os.path.join(models_dir, json_name),
mesh_rel,
@@ -324,105 +319,14 @@ def from_json(json_path, dcf_path):
mesh_offsets.append((float(pos[0]), float(pos[1]), float(pos[2])))
print(f' Resolved {filename} -> {rel}')
write_dcf(dcf_path, bytes(tiles), model_names, mesh_offsets)
write_chunk_json(chunk_json_path, bytes(tiles), model_names, mesh_offsets)
def process_json(json_path):
dcf_path = derive_dcf_path(json_path)
os.makedirs(os.path.dirname(dcf_path), exist_ok=True)
print(f"{json_path} -> {dcf_path}")
from_json(json_path, dcf_path)
# ---------------------------------------------------------------------------
# Legacy DCF (v1/v2) -> current version
# ---------------------------------------------------------------------------
def collapse_legacy_tiles(legacy_tiles, path):
"""Collapse the old one-tile-per-(x,y,z) grid into the current
one-tile-per-(x,y) format. If a column has more than one non-null tile
across its Z layers, the highest Z wins and a warning is printed."""
tiles = bytearray(CHUNK_TILE_COUNT * TILE_SIZE)
for y in range(CHUNK_HEIGHT):
for x in range(CHUNK_WIDTH):
found = []
for z in range(_LEGACY_CHUNK_DEPTH):
legacy_idx = x + y * CHUNK_WIDTH + z * CHUNK_WIDTH * CHUNK_HEIGHT
shape = struct.unpack_from(
'<I', legacy_tiles, legacy_idx * _LEGACY_TILE_SIZE
)[0]
if shape != TILE_SHAPE_NULL:
found.append((z, shape))
if not found:
continue
if len(found) > 1:
print(
f' WARNING: {path}: column ({x}, {y}) has tiles at '
f'z={[z for z, _ in found]} - only one tile per column '
f'is kept, z={found[-1][0]} wins (highest Z)'
)
z, shape = found[-1]
struct.pack_into(
TILE_STRUCT_FORMAT, tiles, tile_index(x, y) * TILE_SIZE,
shape, z
)
return bytes(tiles)
def read_legacy_dcf(path):
with open(path, 'rb') as f:
data = f.read()
if data[:3] != FILE_MAGIC:
raise ValueError(f"{path}: not a DCF file")
version = struct.unpack_from('<I', data, 4)[0]
if version not in (1, 2):
raise ValueError(f"{path}: expected version 1 or 2, got {version}")
offset = 8
tiles_size = _LEGACY_CHUNK_TILE_COUNT * _LEGACY_TILE_SIZE
tiles = collapse_legacy_tiles(data[offset:offset + tiles_size], path)
offset += tiles_size
meshes = []
if version == 1:
vert_count = struct.unpack_from('<I', data, offset)[0]
offset += 4
verts = data[offset:offset + vert_count * VERTEX_SIZE]
if len(verts) != vert_count * VERTEX_SIZE:
raise ValueError(f"{path}: truncated vertex data")
if vert_count > 0:
meshes.append(verts)
else:
mesh_count = data[offset]
offset += 1
for _ in range(mesh_count):
vert_count = struct.unpack_from('<I', data, offset)[0]
offset += 4
verts = data[offset:offset + vert_count * VERTEX_SIZE]
if len(verts) != vert_count * VERTEX_SIZE:
raise ValueError(f"{path}: truncated vertex data")
offset += vert_count * VERTEX_SIZE
if vert_count > 0:
meshes.append(verts)
return tiles, meshes
def upgrade_dcf(path):
print(f"Upgrading legacy DCF {path} ...")
tiles, meshes = read_legacy_dcf(path)
print(
f" tiles={CHUNK_TILE_COUNT}, meshes={len(meshes)}, "
f"total_verts={sum(len(m) // VERTEX_SIZE for m in meshes)}"
)
base = os.path.splitext(os.path.basename(path))[0]
terrain_dir = os.path.join(ASSETS_DIR, 'meshes', 'chunks')
os.makedirs(terrain_dir, exist_ok=True)
mesh_names = []
for idx, verts in enumerate(meshes):
dmf_name = f'chunk_{base}_{idx}.dmf'
write_dmf(os.path.join(terrain_dir, dmf_name), verts)
mesh_names.append(f'meshes/chunks/{dmf_name}')
write_dcf(path, tiles, mesh_names)
chunk_json_path = derive_chunk_json_path(json_path)
os.makedirs(os.path.dirname(chunk_json_path), exist_ok=True)
print(f"{json_path} -> {chunk_json_path}")
from_json(json_path, chunk_json_path)
# ---------------------------------------------------------------------------
@@ -448,10 +352,10 @@ def main():
return
src = args[0]
if os.path.splitext(src)[1].lower() == '.json':
process_json(src)
else:
upgrade_dcf(src)
if os.path.splitext(src)[1].lower() != '.json':
print(f"Expected a .json input file, got: {src}")
sys.exit(1)
process_json(src)
if __name__ == '__main__':