From 0072ec56e294dc943dc426fdff6c9c84e754bd8c Mon Sep 17 00:00:00 2001 From: Vayle <76063792+Valorith@users.noreply.github.com> Date: Sat, 18 Jul 2026 11:10:24 -0400 Subject: [PATCH 1/7] feat(sage): add deterministic NPC appearance QA --- .gitignore | 5 + frontend/eqsage-embed/package.json | 4 + .../eqsage-embed/sage/lib/eqg/eqg-decoder.js | 102 +- .../sage/lib/eqg/gltf-export/common.js | 28 +- .../sage/lib/eqg/gltf-export/eqg-animation.js | 17 +- .../sage/lib/eqg/gltf-export/v3.js | 81 +- .../sage/lib/eqg/gltf-export/v4.js | 11 +- .../eqsage-embed/sage/lib/eqg/model/mds.js | 184 + .../eqsage-embed/sage/lib/model/constants.js | 8 + .../sage/lib/model/file-handle.js | 98 +- frontend/eqsage-embed/sage/lib/pfs/pfs.js | 6 +- .../eqsage-embed/sage/lib/s3d/s3d-decoder.js | 218 +- frontend/eqsage-embed/sage/lib/s3d/wld/wld.js | 4 +- .../sage/lib/util/animation-helper.js | 4 +- .../lib/util/character-animation-policy.js | 66 + .../lib/util/character-texture-orientation.js | 41 + .../eqsage-embed/sage/lib/util/fileHandler.js | 12 + .../eqsage-embed/sage/lib/util/fileSystem.js | 126 +- frontend/eqsage-embed/sage/lib/util/util.js | 8 +- .../scripts/audit-race-skin-textures.mjs | 186 + .../scripts/generate-race-face-audit.mjs | 249 + .../generate-static-race-face-supplement.mjs | 148 + .../scripts/plan-unresolved-race-zones.mjs | 90 + .../eqsage-embed/scripts/sync-race-data.mjs | 163 + .../src/components/dialogs/status-dialog.jsx | 36 +- .../eqsage-embed/src/components/main/main.jsx | 16 +- .../components/spire/dialogs/npc-dialog.jsx | 233 +- .../src/components/spire/door-loader.js | 195 +- .../src/components/spire/drawers/doors.jsx | 60 +- .../spire/nav-bar/spawn-nav/spawn-nav.jsx | 132 +- .../validation/race-archive-audit.jsx | 149 + .../components/validation/race-face-audit.jsx | 1383 ++ .../validation/validation-harness.jsx | 180 +- .../src/components/zone/processZone.js | 331 +- .../src/components/zone/zone-context.jsx | 305 +- .../eqsage-embed/src/components/zone/zone.jsx | 50 +- frontend/eqsage-embed/src/context/alerts.jsx | 18 +- .../src/util/image/image-processor.js | 5 +- .../eqsage-embed/src/util/image/shared.js | 9 +- .../viewer/common/raceAppearancePolicies.json | 31 + .../src/viewer/common/raceData.json | 1805 ++- .../src/viewer/common/raceModelMetadata.json | 6471 +++++++++ .../src/viewer/common/raceModelResolution.js | 182 + .../src/viewer/controllers/GameController.js | 101 +- .../src/viewer/controllers/SpawnController.js | 1321 +- .../src/viewer/controllers/ZoneController.js | 214 +- .../src/viewer/helpers/animationValidation.js | 370 + .../viewer/helpers/appearanceValidation.js | 119 + .../src/viewer/helpers/nameplateValidation.js | 41 + .../src/viewer/helpers/textureAnimation.js | 50 + .../src/viewer/models/BabylonSpawn.js | 1941 ++- frontend/src/views/sage/eqsage-loader.ts | 398 +- frontend/vue.config.js | 3 + internal/app/controller.go | 73 +- internal/app/controller_test.go | 32 + package-lock.json | 84 +- package.json | 21 +- scripts/serve-sage-preview.js | 20 +- tests/sage-preview-ui.spec.ts | 430 +- tools/sage-qa/README.md | 154 + tools/sage-qa/approve-visual-baselines.mjs | 83 + tools/sage-qa/baselines/model-regression.json | 11088 ++++++++++++++++ tools/sage-qa/fs-stress.mjs | 104 + .../inspect-character-archive-rigs.mjs | 134 + tools/sage-qa/lib/aggregate.mjs | 178 + tools/sage-qa/lib/args.mjs | 62 + tools/sage-qa/lib/artifacts.mjs | 91 + tools/sage-qa/lib/coverage.mjs | 127 + tools/sage-qa/lib/extensionless-loader.mjs | 12 + tools/sage-qa/lib/memory.mjs | 73 + tools/sage-qa/lib/playwright-runner.mjs | 1391 ++ tools/sage-qa/lib/profile.mjs | 261 + tools/sage-qa/lib/report.mjs | 114 + tools/sage-qa/lib/server-readiness.mjs | 150 + tools/sage-qa/lib/static-audit.mjs | 43 + tools/sage-qa/lib/summary.mjs | 33 + tools/sage-qa/lib/urls.mjs | 59 + tools/sage-qa/lib/visual-invariants.mjs | 654 + tools/sage-qa/profiles/compact-rigs.json | 37 + tools/sage-qa/profiles/full.json | 58 + tools/sage-qa/profiles/matrix.json | 51 + tools/sage-qa/profiles/model-regression.json | 96 + tools/sage-qa/profiles/smoke.json | 41 + tools/sage-qa/profiles/soak.json | 32 + tools/sage-qa/run.mjs | 374 + tools/sage-qa/test/run.mjs | 34 + tools/sage-qa/test/unit.test.mjs | 1385 ++ tools/sage-qa/verify-checkpoints.mjs | 54 + 88 files changed, 34847 insertions(+), 794 deletions(-) create mode 100644 frontend/eqsage-embed/sage/lib/eqg/model/mds.js create mode 100644 frontend/eqsage-embed/sage/lib/util/character-animation-policy.js create mode 100644 frontend/eqsage-embed/sage/lib/util/character-texture-orientation.js create mode 100644 frontend/eqsage-embed/scripts/audit-race-skin-textures.mjs create mode 100644 frontend/eqsage-embed/scripts/generate-race-face-audit.mjs create mode 100644 frontend/eqsage-embed/scripts/generate-static-race-face-supplement.mjs create mode 100644 frontend/eqsage-embed/scripts/plan-unresolved-race-zones.mjs create mode 100644 frontend/eqsage-embed/scripts/sync-race-data.mjs create mode 100644 frontend/eqsage-embed/src/components/validation/race-archive-audit.jsx create mode 100644 frontend/eqsage-embed/src/components/validation/race-face-audit.jsx create mode 100644 frontend/eqsage-embed/src/viewer/common/raceAppearancePolicies.json create mode 100644 frontend/eqsage-embed/src/viewer/common/raceModelMetadata.json create mode 100644 frontend/eqsage-embed/src/viewer/common/raceModelResolution.js create mode 100644 frontend/eqsage-embed/src/viewer/helpers/animationValidation.js create mode 100644 frontend/eqsage-embed/src/viewer/helpers/appearanceValidation.js create mode 100644 frontend/eqsage-embed/src/viewer/helpers/nameplateValidation.js create mode 100644 frontend/eqsage-embed/src/viewer/helpers/textureAnimation.js create mode 100644 tools/sage-qa/README.md create mode 100644 tools/sage-qa/approve-visual-baselines.mjs create mode 100644 tools/sage-qa/baselines/model-regression.json create mode 100644 tools/sage-qa/fs-stress.mjs create mode 100644 tools/sage-qa/inspect-character-archive-rigs.mjs create mode 100644 tools/sage-qa/lib/aggregate.mjs create mode 100644 tools/sage-qa/lib/args.mjs create mode 100644 tools/sage-qa/lib/artifacts.mjs create mode 100644 tools/sage-qa/lib/coverage.mjs create mode 100644 tools/sage-qa/lib/extensionless-loader.mjs create mode 100644 tools/sage-qa/lib/memory.mjs create mode 100644 tools/sage-qa/lib/playwright-runner.mjs create mode 100644 tools/sage-qa/lib/profile.mjs create mode 100644 tools/sage-qa/lib/report.mjs create mode 100644 tools/sage-qa/lib/server-readiness.mjs create mode 100644 tools/sage-qa/lib/static-audit.mjs create mode 100644 tools/sage-qa/lib/summary.mjs create mode 100644 tools/sage-qa/lib/urls.mjs create mode 100644 tools/sage-qa/lib/visual-invariants.mjs create mode 100644 tools/sage-qa/profiles/compact-rigs.json create mode 100644 tools/sage-qa/profiles/full.json create mode 100644 tools/sage-qa/profiles/matrix.json create mode 100644 tools/sage-qa/profiles/model-regression.json create mode 100644 tools/sage-qa/profiles/smoke.json create mode 100644 tools/sage-qa/profiles/soak.json create mode 100644 tools/sage-qa/run.mjs create mode 100644 tools/sage-qa/test/run.mjs create mode 100644 tools/sage-qa/test/unit.test.mjs create mode 100644 tools/sage-qa/verify-checkpoints.mjs diff --git a/.gitignore b/.gitignore index ce5a0e21..f9e08298 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,8 @@ node_modules/ # Local scratch images created during browser/UI debugging tmp-*.png + +# Local build and browser diagnostics +/.tmp/ +/debug.log +frontend/eqsage-embed/vite.config.js.timestamp-*.mjs diff --git a/frontend/eqsage-embed/package.json b/frontend/eqsage-embed/package.json index d7a29280..d4e705d2 100644 --- a/frontend/eqsage-embed/package.json +++ b/frontend/eqsage-embed/package.json @@ -167,6 +167,10 @@ "dev": "vite --mode development", "build": "cross-env NODE_OPTIONS=--max-old-space-size=8192 vite build", "build:watch": "cross-env NODE_OPTIONS=--max-old-space-size=8192 vite build --watch", + "audit:race-skin-textures": "node scripts/audit-race-skin-textures.mjs", + "sync:race-data": "node scripts/sync-race-data.mjs", + "report:race-face-audit": "node scripts/generate-race-face-audit.mjs", + "plan:race-face-zones": "node scripts/plan-unresolved-race-zones.mjs", "bundle-visualize": "npx vite-bundle-visualizer" }, "eslintConfig": { diff --git a/frontend/eqsage-embed/sage/lib/eqg/eqg-decoder.js b/frontend/eqsage-embed/sage/lib/eqg/eqg-decoder.js index 53e7ce45..8f5565e9 100644 --- a/frontend/eqsage-embed/sage/lib/eqg/eqg-decoder.js +++ b/frontend/eqsage-embed/sage/lib/eqg/eqg-decoder.js @@ -3,9 +3,10 @@ import { Zone } from "./zone/zone"; import { ZoneData } from "./zone/v4-zone"; import { Model, Animation, Lit } from "./model/model"; +import { MDS } from "./model/mds"; import { Eco } from "./eco/eco"; import { PFSArchive } from "../pfs/pfs"; -import { deleteEqFileOrFolder, writeEQFile } from "../util/fileHandler"; +import { deleteEqFileOrFolder, getEQRootDir, writeEQFile } from "../util/fileHandler"; const getImageProcessor = async () => { if (typeof window !== 'undefined' && window.imageProcessor) { @@ -15,6 +16,8 @@ const getImageProcessor = async () => { return imageProcessor; }; +const yieldToBrowser = () => new Promise((resolve) => setTimeout(resolve, 0)); + export class EQGDecoder { #options = { forceWrote: false, @@ -67,12 +70,21 @@ export class EQGDecoder { return this.#fileHandle.name; } + get options() { + return this.#options; + } + async processBuffer(name, arrayBuffer, skipImages = false) { this.pfsArchive = new PFSArchive(); this.pfsArchive.openFromFile(arrayBuffer); const images = []; this.files = {}; + let processedFiles = 0; for (const [fileName, data] of this.pfsArchive.files.entries()) { + processedFiles++; + if (processedFiles % 10 === 0) { + await yieldToBrowser(); + } this.files[fileName] = this.pfsArchive.getFile(fileName); if (fileName.endsWith(".lit")) { const lit = new Lit( @@ -133,12 +145,39 @@ export class EQGDecoder { this.eco[fileName.replace(".eco", "")] = new Eco(this.files[fileName]); } if (fileName.endsWith(".mds")) { - //console.log("Had MDS! ", fileName); + const mds = new MDS( + this.files[fileName], + this.#fileHandle, + fileName + ); + for (const { name, model } of mds.models) { + this.models[name] = model; + } } if (fileName.endsWith('.txt')) { //console.log('Txt', fileName) } - console.log('File', fileName) + } + + const archiveModelName = this.#fileHandle.name.toLowerCase(); + const canonicalModName = `${archiveModelName}.mod`; + if (!this.models[canonicalModName] && !this.models[archiveModelName]) { + const escapedArchiveModelName = archiveModelName.replace( + /[.*+?^${}()|[\]\\]/g, + '\\$&' + ); + const embeddedArchiveCodePattern = new RegExp( + `(?:^|_)${escapedArchiveModelName}(?:_|\\d|\\.)`, + 'i' + ); + const canonicalCandidate = Object.entries(this.models).find( + ([modelName]) => + modelName.endsWith('.mod') && + embeddedArchiveCodePattern.test(modelName) + ); + if (canonicalCandidate) { + this.models[canonicalModName] = canonicalCandidate[1]; + } } // Post process @@ -164,7 +203,7 @@ export class EQGDecoder { } } console.log(`Processed - ${name}`); - console.log('Images', images) + console.log(`Images queued for conversion: ${images.length}`); if (this.#options.rawImageWrite) { console.log('Using raw image write'); } else if (!skipImages) { @@ -189,25 +228,33 @@ export class EQGDecoder { * * @param {import('./common/models').PlaceableGroup} p */ - async writeModels(p, zoneMetadata, modelFile, writtenModels, mod, v3) { + async writeModels(modelFile, mod, destination) { const { writeModels } = await import("./gltf-export/common"); return writeModels.apply(this, [ - p, - zoneMetadata, modelFile, - writtenModels, mod, - v3, + destination, ]); } async export() { if (!this.zone) { + const destination = this.#options.modelDestination ?? 'models'; + const canonicalModName = `${this.name.toLowerCase()}.mod`; + if (destination === 'models' && this.models[canonicalModName]) { + await this.writeModels( + canonicalModName, + this.models[canonicalModName], + destination + ); + return true; + } for (const [name, mod] of Object.entries(this.models)) { if (!name.includes('ter_')) { - await this.writeModels(name, mod); + await this.writeModels(name, mod, destination); } } + return true; } if (this.zone?.header?.version === 4) { const { exportv4 } = await import("./gltf-export/v4"); @@ -222,29 +269,32 @@ export class EQGDecoder { const micro = performance.now(); for (const file of this.#fileHandle.fileHandles) { - const extension = file.name.split(".").pop(); + const extension = file.name.split(".").pop().toLowerCase(); switch (extension) { case "eqg": await this.processEQG(file); break; case "txt": - if (file.name.endsWith('_assets.txt') && !this.options.skipSubload) { - // const contents = (await file.text()).split('\r\n'); - // for (const line of contents) { - // if (line.endsWith('.eqg')) { - // console.log(`Loading dependent asset ${line}`); - // try { - // const dir = getEQRootDir(); - // const fh = await dir.getFileHandle(line).then(f => f.getFile()); - // await this.processEQG(fh); - // } catch(e) { - // console.log(`Error loading dependent asset`, e); - // } - - // } - // } + if (file.name.endsWith('_assets.txt') && !this.#options.skipSubload) { + const contents = (await file.text()).split(/\r?\n/); + for (const entry of contents) { + const line = entry.trim(); + if (line.toLowerCase().endsWith('.eqg')) { + console.log(`Loading dependent asset ${line}`); + try { + const dir = getEQRootDir(); + const fh = await dir.getFileHandle(line).then(f => f.getFile()); + await this.processEQG(fh); + await yieldToBrowser(); + } catch(e) { + console.log(`Error loading dependent asset`, e); + } + } + } } break; + case "s3d": + break; case "eff": break; case "xmi": diff --git a/frontend/eqsage-embed/sage/lib/eqg/gltf-export/common.js b/frontend/eqsage-embed/sage/lib/eqg/gltf-export/common.js index 5d6eba5f..a54c174a 100644 --- a/frontend/eqsage-embed/sage/lib/eqg/gltf-export/common.js +++ b/frontend/eqsage-embed/sage/lib/eqg/gltf-export/common.js @@ -10,6 +10,7 @@ import { import { EQGAnimationWriter } from './eqg-animation'; const io = new WebIO().registerExtensions(ALL_EXTENSIONS); +const yieldToBrowser = () => new Promise((resolve) => setTimeout(resolve, 0)); export async function writeMetadata( infP, @@ -50,12 +51,15 @@ export async function writeMetadata( * @this {import('../eqg-decoder').EQGDecoder} * @returns */ -export async function writeModels(modelFile, mod) { +export async function writeModels(modelFile, mod, destination = 'objects') { modelFile = modelFile.replace('.mod', ''); const diskFileName = `${modelFile}.glb`; - await appendObjectMetadata(modelFile, `${this.name}.eqg`); + if (destination === 'objects') { + await appendObjectMetadata(modelFile, `${this.name}.eqg`); + } if ( - (await getEQFileExists('objects', diskFileName)) && + (await getEQFileExists(destination, diskFileName)) && + !this.options?.forceWrote && !modelFile.includes('et_drbanner') && !modelFile.includes('dest') && !modelFile.includes('ggy') && @@ -121,7 +125,16 @@ export async function writeModels(modelFile, mod) { .setSpecularColorFactor([0, 0, 0]); gltfMaterial.setExtension('KHR_materials_specular', specular); for (const prop of mat.properties) { + if (prop.type !== 2) { + continue; + } const [name] = prop.valueS.toLowerCase().split('.'); + if ( + !name || + (mat.name.toLowerCase() === 'failsafeshader' && name === 'grid_standard') + ) { + continue; + } const texture = document .createTexture(name) .setMimeType('image/png') @@ -161,8 +174,13 @@ export async function writeModels(modelFile, mod) { } const primitiveMap = {}; const gltfMesh = document.createMesh(modelFile); + let processedPolygons = 0; for (const p of mod.geometry.polys) { + processedPolygons++; + if (processedPolygons % 5000 === 0) { + await yieldToBrowser(); + } if (p.material === -1) { continue; } @@ -340,9 +358,9 @@ export async function writeModels(modelFile, mod) { for (const [name, ani] of Object.entries(this.animations)) { // if (name.startsWith(modelFile)) { - console.log('ani', ani); animWriter.applyAnimation(ani, name); animWriter.applyAnimation(ani, name, true); + await yieldToBrowser(); // } } @@ -364,5 +382,5 @@ export async function writeModels(modelFile, mod) { } const bytes = await io.writeBinary(document); - await writeEQFile("objects", diskFileName, bytes.buffer); // eslint-disable-line + await writeEQFile(destination, diskFileName, bytes.buffer); } diff --git a/frontend/eqsage-embed/sage/lib/eqg/gltf-export/eqg-animation.js b/frontend/eqsage-embed/sage/lib/eqg/gltf-export/eqg-animation.js index 8bc7d2d5..f90709a9 100644 --- a/frontend/eqsage-embed/sage/lib/eqg/gltf-export/eqg-animation.js +++ b/frontend/eqsage-embed/sage/lib/eqg/gltf-export/eqg-animation.js @@ -46,8 +46,8 @@ export class EQGAnimationWriter { for (const ani of animation.boneAnimations) { const node = this.skeletonNodes.find((n) => n.getName() === ani.boneName); const bone = this.bones.find((b) => b.name === ani.boneName); - if (!bone) { - console.log(`No bone for node ${ani.boneName}`); + if (!bone || !node) { + console.log(`No skeleton target for animation node ${ani.boneName}`); continue; } @@ -94,18 +94,18 @@ export class EQGAnimationWriter { for (const [_idx, ani] of Object.entries(animation.boneAnimations)) { const node = this.skeletonNodes.find((n) => n.getName() === ani.boneName); const bone = this.bones.find((b) => b.name === ani.boneName); - if (!bone) { - console.log(`No bone for node ${ani.boneName}`); + if (!bone || !node) { + console.log(`No skeleton target for animation node ${ani.boneName}`); continue; } - const [rotX, rotY, rotZ, rotW] = node.getWorldRotation(); - const [x, y, z] = node.getWorldTranslation(); const aniParents = []; const boneAccum = { ...bone }; let parent = bone.parent; while (parent) { const parentAnimation = animation.boneAnimations.find(ba => ba.boneName === parent.name); // eslint-disable-line - aniParents.push(parentAnimation); + if (parentAnimation) { + aniParents.push(parentAnimation); + } boneAccum.rotX += parent.rotX; boneAccum.rotY += parent.rotY; boneAccum.rotZ += parent.rotZ; @@ -120,6 +120,9 @@ export class EQGAnimationWriter { const frameAcc = { ...frame }; for (const p of aniParents) { const parentFrame = p.animationFrames[idx]; + if (!parentFrame) { + continue; + } frameAcc.rotX += parentFrame.rotX; frameAcc.rotY += parentFrame.rotY; frameAcc.rotZ += parentFrame.rotZ; diff --git a/frontend/eqsage-embed/sage/lib/eqg/gltf-export/v3.js b/frontend/eqsage-embed/sage/lib/eqg/gltf-export/v3.js index 4cbf141b..69fca552 100644 --- a/frontend/eqsage-embed/sage/lib/eqg/gltf-export/v3.js +++ b/frontend/eqsage-embed/sage/lib/eqg/gltf-export/v3.js @@ -10,20 +10,31 @@ import draco3d from 'draco3dgltf'; import { locateStaticAsset } from '../../../../src/static-assets'; -const io = new WebIO() - .registerDependencies({ - 'draco3d.decoder': await draco3d.createDecoderModule({ - locateFile: locateStaticAsset, - print : console.log, - printErr: console.error, - }), - 'draco3d.encoder': await draco3d.createEncoderModule({ - locateFile: locateStaticAsset, - print : console.log, - printErr: console.error, - }), - }) - .registerExtensions(ALL_EXTENSIONS); +const io = new WebIO().registerExtensions(ALL_EXTENSIONS); +let dracoDependenciesPromise = null; + +const ensureDracoDependencies = async () => { + if (!dracoDependenciesPromise) { + dracoDependenciesPromise = Promise.all([ + draco3d.createDecoderModule({ + locateFile: locateStaticAsset, + print : console.log, + printErr: console.error, + }), + draco3d.createEncoderModule({ + locateFile: locateStaticAsset, + print : console.log, + printErr: console.error, + }), + ]).then(([decoder, encoder]) => { + io.registerDependencies({ + 'draco3d.decoder': decoder, + 'draco3d.encoder': encoder, + }); + }); + } + return dracoDependenciesPromise; +}; // Helper function to convert uint32 (assumed format 0xAARRGGBB) to normalized [r, g, b, a] function uint32ToRGBA(color) { const a = ((color >> 24) & 0xFF) / 255; @@ -33,6 +44,11 @@ function uint32ToRGBA(color) { return [r, g, b, a]; } +const isSpirePreview = () => + typeof window !== 'undefined' && !!window.__spireSagePreview; + +const yieldToBrowser = () => new Promise((resolve) => setTimeout(resolve, 0)); + export async function exportv3(zoneName) { const document = new Document(); const buffer = document.createBuffer(); @@ -85,7 +101,16 @@ export async function exportv3(zoneName) { .setSpecularColorFactor([0, 0, 0]); gltfMaterial.setExtension('KHR_materials_specular', specular); for (const prop of mat.properties) { + if (prop.type !== 2) { + continue; + } const [name] = prop.valueS.toLowerCase().split('.'); + if ( + !name || + (mat.name.toLowerCase() === 'failsafeshader' && name === 'grid_standard') + ) { + continue; + } const texture = document .createTexture(name) // .setImage(new Uint8Array(await getEQFile('textures', `${name}.png`))) @@ -113,6 +138,7 @@ export async function exportv3(zoneName) { const terrain = this.zone.terrain; const writtenModels = {}; + let processedPolygons = 0; for (const [name, mod] of Object.entries(this.models)) { if (!name.includes('ter_')) { @@ -137,12 +163,15 @@ export async function exportv3(zoneName) { await writeMetadata.call(this, placeable, zoneMetadata, modelFile, writtenModels, true); continue; } - console.log('mod', mod); if (mod) { const primitiveMap = {}; // Process each polygon in the model geometry. // (Renaming inner loop variable to "poly" for clarity.) for (const poly of mod.geometry.polys) { + processedPolygons++; + if (processedPolygons % 5000 === 0) { + await yieldToBrowser(); + } if (poly.material === -1) { continue; } @@ -217,7 +246,6 @@ export async function exportv3(zoneName) { const col3 = uint32ToRGBA(lit[poly.verts[2]]); sharedPrimitive.colors.push(...col1, ...col2, ...col3); } - console.log('Pri', primitiveMap); // Now create accessors for each primitive. for (const { gltfPrim, indices, vecs, normals, uv, colors } of Object.values(primitiveMap)) { const idc = new Uint16Array(indices); @@ -275,17 +303,18 @@ export async function exportv3(zoneName) { `${this.zone.name.replace('.zon', '.json')}`, JSON.stringify(zoneMetadata) ); - console.log('Start', document); - try { - await document.transform( - // Compress mesh geometry with Draco. - draco({ ...DRACO_DEFAULTS, quantizationVolume: 'scene' }) - ); - } catch (e) { - console.log('Error with draco compression', e); + if (!isSpirePreview()) { + try { + await ensureDracoDependencies(); + await document.transform( + // Compress mesh geometry with Draco. + draco({ ...DRACO_DEFAULTS, quantizationVolume: 'scene' }) + ); + } catch (e) { + console.log('Error with draco compression', e); + } } - - console.log('Finish draco'); + const bytes = await io.writeBinary(document); await writeEQFile('zones', `${zoneName}.glb`, bytes.buffer); } diff --git a/frontend/eqsage-embed/sage/lib/eqg/gltf-export/v4.js b/frontend/eqsage-embed/sage/lib/eqg/gltf-export/v4.js index 5a5abcd9..95ed8952 100644 --- a/frontend/eqsage-embed/sage/lib/eqg/gltf-export/v4.js +++ b/frontend/eqsage-embed/sage/lib/eqg/gltf-export/v4.js @@ -91,7 +91,16 @@ export async function exportv4(zoneName) { .setSpecularColorFactor([0, 0, 0]); gltfMaterial.setExtension('KHR_materials_specular', specular); for (const prop of mat.properties) { + if (prop.type !== 2) { + continue; + } const [name] = prop.valueS.toLowerCase().split('.'); + if ( + !name || + (mat.name.toLowerCase() === 'failsafeshader' && name === 'grid_standard') + ) { + continue; + } const texture = document .createTexture(name) .setURI(`/eq/textures/${name}`) @@ -263,4 +272,4 @@ export async function exportv4(zoneName) { const bytes = await io.writeBinary(document); await writeEQFile('zones', `${zoneName}.glb`, bytes.buffer); -} \ No newline at end of file +} diff --git a/frontend/eqsage-embed/sage/lib/eqg/model/mds.js b/frontend/eqsage-embed/sage/lib/eqg/model/mds.js new file mode 100644 index 00000000..2be8bb0d --- /dev/null +++ b/frontend/eqsage-embed/sage/lib/eqg/model/mds.js @@ -0,0 +1,184 @@ +/* eslint-disable */ +import { vec2, vec3 } from 'gl-matrix'; +import { TypedArrayReader } from '../../util/typed-array-reader'; +import { + Bone, + BoneAssignment, + Geometry, + Material, + MaterialProperty, + Polygon, + Vertex, + Weight, +} from './model'; + +const cloneBones = (bones) => + bones.map((source) => { + const bone = new Bone(); + Object.assign(bone, source); + bone.children = []; + return bone; + }); + +export class MDS { + reader = null; + fileHandle = null; + name = ''; + version = 0; + materials = []; + bones = []; + models = []; + + constructor(data, fileHandle, name) { + this.reader = new TypedArrayReader(data.buffer); + this.fileHandle = fileHandle; + this.name = name; + this.init(); + } + + init() { + const reader = this.reader; + const magic = reader.readString(4); + if (magic !== 'EQGS') { + throw new Error(`MDS ${this.name} does not contain an EQGS header: ${magic}`); + } + + const [version, listLength, materialCount, boneCount, modelCount] = + reader.readManyUint32(5); + this.version = version; + const listIdx = reader.getCursor(); + reader.addCursor(listLength); + + for (let i = 0; i < materialCount; i++) { + const material = new Material(); + reader.readInt32(); + material.name = reader.readCStringFromIdx(listIdx + reader.readInt32()); + material.shader = reader.readCStringFromIdx(listIdx + reader.readInt32()); + const propertyCount = reader.readUint32(); + for (let j = 0; j < propertyCount; j++) { + const property = new MaterialProperty(); + property.name = reader.readCStringFromIdx(listIdx + reader.readInt32()); + property.type = reader.readUint32(); + if (property.type === 0) { + property.valueF = reader.readFloat32(); + } else { + const value = reader.readInt32(); + if (property.type === 2) { + property.valueS = reader.readCStringFromIdx(listIdx + value); + } else { + property.valueI = value; + } + } + material.properties.push(property); + } + this.materials.push(material); + } + + for (let i = 0; i < boneCount; i++) { + const bone = new Bone(); + bone.name = reader.readCStringFromIdx(listIdx + reader.readInt32()); + bone.next = reader.readInt32(); + bone.childrenCount = reader.readUint32(); + bone.childrenIndex = reader.readInt32(); + const [x, y, z, rotX, rotY, rotZ, rotW, scaleX, scaleY, scaleZ] = + reader.readManyFloat32(10); + bone.x = -x; + bone.y = -y; + bone.z = z; + bone.rotX = -rotX; + bone.rotY = -rotY; + bone.rotZ = rotZ; + bone.rotW = rotW; + bone.scaleX = scaleX; + bone.scaleY = scaleY; + bone.scaleZ = scaleZ; + this.bones.push(bone); + } + + const archiveName = this.name.replace(/\.mds$/i, '').toLowerCase(); + for (let i = 0; i < modelCount; i++) { + const mainPiece = reader.readUint32(); + const sourceName = reader + .readCStringFromIdx(listIdx + reader.readInt32()) + .toLowerCase(); + const vertexCount = reader.readUint32(); + const faceCount = reader.readUint32(); + const modelBoneCount = reader.readUint32(); + const geometry = new Geometry(); + geometry.name = sourceName; + geometry.mats = this.materials; + + for (let vertexIndex = 0; vertexIndex < vertexCount; vertexIndex++) { + const vertex = new Vertex(); + const [x, y, z, normalX, normalY, normalZ] = reader.readManyFloat32(6); + vertex.pos = vec3.fromValues(-x, -y, z); + vertex.nor = vec3.fromValues(-normalX, -normalY, normalZ); + if (version <= 2) { + vertex.col = 0xffffffff; + const [u, v] = reader.readManyFloat32(2); + vertex.tex = vec2.fromValues(-u, -v); + } else { + vertex.col = reader.readUint32(); + const [u, v] = reader.readManyFloat32(2); + reader.readManyFloat32(2); + vertex.tex = vec2.fromValues(u, v); + } + geometry.verts.push(vertex); + } + + for (let faceIndex = 0; faceIndex < faceCount; faceIndex++) { + const polygon = new Polygon(); + polygon.verts = reader.readManyUint32(3); + polygon.material = reader.readInt32(); + polygon.flags = reader.readUint32(); + geometry.polys.push(polygon); + } + + if (this.bones.length > 0) { + for (const vertex of geometry.verts) { + const assignment = new BoneAssignment(); + assignment.count = reader.readInt32(); + for (let weightIndex = 0; weightIndex < 4; weightIndex++) { + const weight = new Weight(); + weight.bone = reader.readInt32(); + weight.weight = reader.readFloat32(); + if (weightIndex >= assignment.count) { + weight.bone = -1; + weight.weight = 0; + } + assignment.weights.push(weight); + } + vertex.boneAssignment = assignment; + } + } + + // Standalone character archives are addressed by their archive/race + // code, but a few production MDS files retain an internal artist name + // for their only mesh (for example BRX -> MBX), or mark a differently + // named head-shaped piece as the main model. Publish that sole/main + // piece under the archive code so the runtime can request the race by + // its canonical model name. Alternate pieces keep their source names. + const isCanonicalPiece = sourceName === `${archiveName}00`; + const isSolePiece = modelCount === 1; + const isExplicitMainPiece = mainPiece === 1; + const isFirstDefaultPiece = i === 0 && /00$/i.test(sourceName); + const modelName = + isCanonicalPiece || isSolePiece || isExplicitMainPiece || isFirstDefaultPiece + ? archiveName + : sourceName; + this.models.push({ + name: modelName, + model: { + name: sourceName, + header: { + mainPiece, + modelBoneCount, + version, + }, + geometry, + bones: cloneBones(this.bones), + }, + }); + } + } +} diff --git a/frontend/eqsage-embed/sage/lib/model/constants.js b/frontend/eqsage-embed/sage/lib/model/constants.js index d4dddc19..31d506f5 100644 --- a/frontend/eqsage-embed/sage/lib/model/constants.js +++ b/frontend/eqsage-embed/sage/lib/model/constants.js @@ -749,3 +749,11 @@ export const modelDefinitions = { }; export const VERSION = 2.05; export const ZONE_VERSION = 2.08; +export const PREVIEW_ZONE_OBJECT_CACHE_VERSION = 4; +// Keep preview character cache markers in one dependency-free module. Both the +// archive writer and the zone loader must agree or a cache can be marked ready +// while still containing model/texture output from an older exporter. +export const PREVIEW_CHARACTER_CACHE_VERSION = 33; +// Embedded in each classic character GLB so model-policy changes can be +// invalidated independently of the much larger global/zone cache. +export const PREVIEW_CHARACTER_MODEL_CACHE_VERSION = 1; diff --git a/frontend/eqsage-embed/sage/lib/model/file-handle.js b/frontend/eqsage-embed/sage/lib/model/file-handle.js index 871064de..c6e8f683 100644 --- a/frontend/eqsage-embed/sage/lib/model/file-handle.js +++ b/frontend/eqsage-embed/sage/lib/model/file-handle.js @@ -1,11 +1,14 @@ -import { FILE_TYPE, VERSION } from './constants'; +import { + FILE_TYPE, + PREVIEW_CHARACTER_CACHE_VERSION, + PREVIEW_ZONE_OBJECT_CACHE_VERSION, + VERSION, +} from './constants'; import { S3DDecoder } from '../s3d/s3d-decoder'; import { Document } from '@gltf-transform/core'; import { EQGDecoder } from '../eqg/eqg-decoder'; import { getEQFile, getEQFileExists } from '../util/fileHandler'; -const PREVIEW_CHARACTER_CACHE_VERSION = 12; - const isSpirePreview = () => typeof window !== 'undefined' && !!window.__spireSagePreview; @@ -71,10 +74,24 @@ export class EQFileHandle { } get #type() { - const eqgExists = this.#fileHandles.some(f => f.name === `${this.name}.eqg`); - const s3dExists = this.#fileHandles.some(f => f.name === `${this.name}.s3d`); - - return eqgExists ? FILE_TYPE.EQG : s3dExists ? FILE_TYPE.S3D : FILE_TYPE.NONE; + const normalizedNames = this.#fileHandles.map((file) => + `${file.name ?? ''}`.toLowerCase() + ); + const baseName = this.name.toLowerCase(); + const eqgExists = normalizedNames.includes(`${baseName}.eqg`); + const s3dExists = normalizedNames.includes(`${baseName}.s3d`); + + if (eqgExists || s3dExists) { + return eqgExists ? FILE_TYPE.EQG : FILE_TYPE.S3D; + } + + // Character-only validation intentionally supplies archives such as + // greatdivide_chr.s3d without the zone's primary greatdivide.s3d. Treat a + // homogeneous filtered archive set as its real format so the decoder runs + // instead of reporting a successful no-op. + const anyEqg = normalizedNames.some((name) => name.endsWith('.eqg')); + const anyS3d = normalizedNames.some((name) => name.endsWith('.s3d')); + return anyEqg ? FILE_TYPE.EQG : anyS3d ? FILE_TYPE.S3D : FILE_TYPE.NONE; } @@ -112,29 +129,44 @@ export class EQFileHandle { existingMetadata?.spireCharacterTextures === true && existingMetadata?.spireCharacterCacheVersion === PREVIEW_CHARACTER_CACHE_VERSION ); + const previewZoneObjectCacheReady = + !isSpirePreview() || + existingMetadata?.spireZoneObjectCacheVersion === PREVIEW_ZONE_OBJECT_CACHE_VERSION; logFileHandleStep(this.name, 'cache:checked', { glbExists : exists, metadataVersion: existingMetadata?.version ?? null, previewCharacterCacheReady, + previewZoneObjectCacheReady, expectedVersion: VERSION, }); if ( exists && existingMetadata?.version === VERSION && previewCharacterCacheReady && + previewZoneObjectCacheReady && !this.#settings.forceReload ) { logFileHandleStep(this.name, 'cache:hit, skipping translation'); return; } + const decoderOptions = { + ...this.#options, + // A stale character cache must overwrite already-exported model GLBs. + // Otherwise the archive pass succeeds and advances the cache marker while + // leaving old UV/material data on disk. + forceWrite: + isSpirePreview() && + (!previewCharacterCacheReady || !previewZoneObjectCacheReady), + }; if (type === FILE_TYPE.EQG) { logFileHandleStep(this.name, 'decoder:eqg:start'); - const eqgDecoder = new EQGDecoder(this, this.#options); + const eqgDecoder = new EQGDecoder(this, decoderOptions); await eqgDecoder.process(); logFileHandleStep(this.name, 'decoder:eqg:processed'); if (doExport) { logFileHandleStep(this.name, 'decoder:eqg:export:start'); await eqgDecoder.export(); + await this.#processSupplementalArchives(type, decoderOptions); logFileHandleStep(this.name, 'decoder:eqg:export:done', { seconds: ((performance.now() - startedAt) / 1000).toFixed(2), }); @@ -142,12 +174,13 @@ export class EQFileHandle { } } else if (type === FILE_TYPE.S3D) { logFileHandleStep(this.name, 'decoder:s3d:start'); - const s3dDecoder = new S3DDecoder(this, this.#options); + const s3dDecoder = new S3DDecoder(this, decoderOptions); await s3dDecoder.process(); logFileHandleStep(this.name, 'decoder:s3d:processed'); if (doExport) { logFileHandleStep(this.name, 'decoder:s3d:export:start'); await s3dDecoder.export(); + await this.#processSupplementalArchives(type, decoderOptions); logFileHandleStep(this.name, 'decoder:s3d:export:done', { seconds: ((performance.now() - startedAt) / 1000).toFixed(2), }); @@ -162,4 +195,51 @@ export class EQFileHandle { seconds: ((performance.now() - startedAt) / 1000).toFixed(2), }); } + + async #processSupplementalArchives(primaryType, decoderOptions) { + const lowerName = this.name.toLowerCase(); + const supplementalHandles = this.#fileHandles.filter((file) => { + const fileName = file.name.toLowerCase(); + if (primaryType === FILE_TYPE.EQG) { + return fileName.endsWith('.s3d') && fileName !== `${lowerName}.s3d`; + } + if (primaryType === FILE_TYPE.S3D) { + return fileName.endsWith('.eqg') && fileName !== `${lowerName}.eqg`; + } + return false; + }); + + if (supplementalHandles.length === 0) { + return; + } + + const supplementalType = + primaryType === FILE_TYPE.EQG ? FILE_TYPE.S3D : FILE_TYPE.EQG; + logFileHandleStep(this.name, 'decoder:supplemental:start', { + handles: supplementalHandles.map((file) => file.name), + type : supplementalType, + }); + const supplementalFileHandle = new EQFileHandle( + this.name, + supplementalHandles, + this.#rootFileHandle, + this.#settings, + decoderOptions + ); + await supplementalFileHandle.initialize(); + + if (supplementalType === FILE_TYPE.S3D) { + const decoder = new S3DDecoder(supplementalFileHandle, decoderOptions); + await decoder.process(); + await decoder.export(); + } else { + const decoder = new EQGDecoder(supplementalFileHandle, { + ...decoderOptions, + modelDestination: 'objects', + }); + await decoder.process(); + await decoder.export(); + } + logFileHandleStep(this.name, 'decoder:supplemental:done'); + } } diff --git a/frontend/eqsage-embed/sage/lib/pfs/pfs.js b/frontend/eqsage-embed/sage/lib/pfs/pfs.js index 44ef8054..6b0cfff1 100644 --- a/frontend/eqsage-embed/sage/lib/pfs/pfs.js +++ b/frontend/eqsage-embed/sage/lib/pfs/pfs.js @@ -127,7 +127,11 @@ export class PFSArchive { continue; } - const filenameBuffer = new Uint8Array(buffer.slice(offset, offset + size)); + // `size` is the uncompressed filename-table length, not the number of + // compressed bytes stored in the archive. Small tables can compress to + // a larger block, so slicing to `offset + size` truncates valid PFS + // archives such as the classic `global*_chr2.s3d` head archives. + const filenameBuffer = new Uint8Array(buffer.slice(offset, dirOffset)); let filenameInflated; try { filenameInflated = this.inflateByFileOffset(filenameBuffer, 0, size); diff --git a/frontend/eqsage-embed/sage/lib/s3d/s3d-decoder.js b/frontend/eqsage-embed/sage/lib/s3d/s3d-decoder.js index a269c910..52774607 100644 --- a/frontend/eqsage-embed/sage/lib/s3d/s3d-decoder.js +++ b/frontend/eqsage-embed/sage/lib/s3d/s3d-decoder.js @@ -16,7 +16,11 @@ import { getEQRootDir, writeEQFile, } from '../util/fileHandler'; -import { VERSION, ZONE_VERSION } from '../model/constants'; +import { + PREVIEW_CHARACTER_MODEL_CACHE_VERSION, + VERSION, + ZONE_VERSION, +} from '../model/constants'; import { fragmentNameCleaner } from '../util/util'; import { S3DAnimationWriter, animationMap } from '../util/animation-helper'; import { EQGDecoder } from '../eqg/eqg-decoder'; @@ -26,6 +30,13 @@ import { Wld, WldType } from './wld/wld'; import { ActorType } from './animation/actor'; import { globals } from '../globals'; import { optimizeBoundingBoxes } from './bsp/region-utils'; +import { + getCharacterHeadOrientationPolicy, +} from '../util/character-texture-orientation'; +import { + getCharacterAnimationCompatibility, + STATIC_POSE_ONLY_CHARACTER_MODELS, +} from '../util/character-animation-policy'; const io = new WebIO().registerExtensions(ALL_EXTENSIONS); @@ -59,49 +70,14 @@ const shouldSkipPreviewImages = (fileName, zoneName) => !isZoneObjectArchive(fileName); const CLASSIC_MALE_LEG_SPIKE_MODELS = new Set(['bam', 'erm', 'hum']); -const CLASSIC_HEAD_TEXTURE_V_PRESERVE_MODELS = new Set([ - 'baf', - 'bam', - 'daf', - 'dam', - 'dwf', - 'dwm', - 'elf', - 'elm', - 'erf', - 'erm', - 'gnf', - 'gnm', - 'haf', - 'ham', - 'hif', - 'him', - 'hof', - 'hom', - 'huf', - 'hum', - 'ikf', - 'ikm', - 'ogf', - 'ogm', - 'trf', - 'trm', -]); -const CHARACTER_HEAD_TEXTURE_PATTERN = - /^[a-z0-9]{3}(?:he(?:\d{2}|sk)\d{2}|fa\d{4})$/i; - -const shouldPreserveClassicHeadTextureV = (materialName) => { - const name = `${materialName ?? ''}`.toLowerCase(); - const modelName = name.slice(0, 3); - return ( - CLASSIC_HEAD_TEXTURE_V_PRESERVE_MODELS.has(modelName) && - CHARACTER_HEAD_TEXTURE_PATTERN.test(name) - ); +// DDS decoding flips image pixels vertically before they are cached as PNG. +// Every discrete head needs one effective V correction. A single shared policy +// drives geometry conversion, legacy-cache compensation, and validation. +const shouldFlipSkinnedMeshV = (materialName) => { + const policy = getCharacterHeadOrientationPolicy(materialName); + return policy.isCharacterHead ? policy.geometryUvFlipped : true; }; -const shouldFlipSkinnedMeshV = (materialName) => - !shouldPreserveClassicHeadTextureV(materialName); - const getClassicMaleLegSpikeModel = (materialName) => { const match = `${materialName ?? ''}`.match(/^([a-z0-9]{3})lg000[12]$/i); return match?.[1]?.toLowerCase() ?? null; @@ -231,12 +207,29 @@ export class S3DDecoder { ) { for (const mesh of meshes) { const material = mesh.materialList; - const baseName = (name || mesh.name).split('_')[0].toLowerCase(); + const rawBaseName = (name || mesh.name).split('_')[0].toLowerCase(); + const primarySkeletonAlias = + path === 'models' && + mesh === skeleton?.meshes?.[0] && + /^[a-z0-9]{3}$/i.test(`${skeleton?.modelBase ?? ''}`) + ? skeleton.modelBase.toLowerCase() + : null; + const characterAlias = path === 'models' + ? rawBaseName.match(/^([a-z0-9]{3})(?:mesh|(?:[a-z0-9]{3})?bod)$/)?.[1] + : null; + const baseName = primarySkeletonAlias ?? characterAlias ?? rawBaseName; const scrubbedName = material.name.split('_')[0].toLowerCase(); const document = new Document(scrubbedName); const buffer = document.createBuffer(); const scene = document.createScene(scrubbedName); const secondary = /he\d+/.test(baseName); + // Do not let generation order decide whether a known compact rig gets + // its native pose. Without this marker, a cached GLB can be loaded as a + // playable character even though it contains POS only, leaving the body + // folded or in a T-pose at runtime. + let nativePoseOnly = + !secondary && STATIC_POSE_ONLY_CHARACTER_MODELS.has(baseName); + let animationCompatibility = null; // Write skeleton data to json if we're a supplier of animations for other models if (!secondary && Object.values(animationMap).includes(baseName)) { @@ -274,15 +267,28 @@ export class S3DDecoder { 'json' )); if (existingAnimations) { - for (const [key, value] of Object.entries(existingAnimations)) { - if (secondary) { - continue; - } - if (!skeleton.animations[key]) { - skeleton.animations[key] = { - ...value, - animModelBase: baseName, - }; + animationCompatibility = getCharacterAnimationCompatibility({ + targetPoseTracks: + skeleton.animations.pos?.tracksCleanedStripped, + donorPoseTracks: + existingAnimations.pos?.tracksCleanedStripped, + nativeAnimationKeys: Object.keys(skeleton.animations), + }); + nativePoseOnly = + !secondary && + (STATIC_POSE_ONLY_CHARACTER_MODELS.has(baseName) || + animationCompatibility.useNativePoseOnly); + if (!nativePoseOnly) { + for (const [key, value] of Object.entries(existingAnimations)) { + if (secondary) { + continue; + } + if (!skeleton.animations[key]) { + skeleton.animations[key] = { + ...value, + animModelBase: baseName, + }; + } } } } else { @@ -296,6 +302,11 @@ export class S3DDecoder { node.setExtras({ secondaryMeshes: skeleton.secondaryMeshes.length, + spireCharacterModelCacheVersion: + PREVIEW_CHARACTER_MODEL_CACHE_VERSION, + spireNativePoseOnly: nativePoseOnly, + spireAnimationExactBoneCoverage: + animationCompatibility?.exactCoverage ?? null, }); scene.addChild(node); @@ -316,6 +327,14 @@ export class S3DDecoder { console.warn(`S3D model had no material link ${name}`); continue; } + const headOrientation = getCharacterHeadOrientationPolicy(name); + gltfMat.setExtras({ + ...(gltfMat.getExtras?.() ?? {}), + spireCharacterHead : headOrientation.isCharacterHead, + spireSkinnedVFlipped : shouldFlipSkinnedMeshV(name), + spireRuntimeHeadVFlipped: + headOrientation.runtimeTextureVFlipped, + }); let sharedPrimitive = primitiveMap[name]; if (!sharedPrimitive) { @@ -493,6 +512,11 @@ export class S3DDecoder { const bytes = await io.writeBinary(document); await writeEQFile(path, `${baseName}.glb`, bytes); + if (isSpirePreview() && /\d{2}$/.test(baseName)) { + console.log( + `[SageCharacterExport] wrote ${baseName}.glb from ${wld.name}:${mesh.name}` + ); + } } } @@ -759,7 +783,12 @@ export class S3DDecoder { // } const bytes = await io.writeBinary(document); - await writeEQFile(path, `${skeleton.modelBase}.glb`, bytes); + // A character skeleton can own alternate body meshes such as HUM01 + // (robes). Writing every mesh under the skeleton name silently + // overwrites hum.glb and makes the requested hum01.glb impossible to + // resolve at runtime. `baseName` preserves the primary skeleton alias + // while keeping distinct appearance meshes addressable. + await writeEQFile(path, `${baseName}.glb`, bytes); } } /** @@ -790,6 +819,21 @@ export class S3DDecoder { } return isAnimationSupplier ? -1 : 1; }); + if (isSpirePreview()) { + console.log(`[SageCharacterExport] inventory ${JSON.stringify({ + archive: wld.name, + skeletons: wld.skeletons.map((skeleton) => ({ + modelBase: skeleton?.modelBase ?? null, + meshes: (skeleton?.meshes ?? []).map((mesh) => mesh?.name).slice(0, 24), + secondaryMeshes: (skeleton?.secondaryMeshes ?? []) + .map((mesh) => mesh?.name) + .slice(0, 24), + })), + meshInventory: wld.meshes + .map((mesh) => mesh?.name) + .slice(0, 48), + })}`); + } const AnimationSources = {}; for (const skeleton of wld.skeletons) { @@ -876,7 +920,10 @@ export class S3DDecoder { * @param {*} itemExport */ async exportSkeletalActor(wld, skeleton, name, path = 'objects') { - const meshes = []; + const meshes = [ + ...(skeleton.meshes ?? []), + ...(skeleton.secondaryMeshes ?? []), + ]; let boneIdx = -1; for (const [idx, bone] of Object.entries(skeleton.skeleton)) { @@ -885,7 +932,9 @@ export class S3DDecoder { bone.meshReference.mesh && bone.meshReference.mesh.materialList ) { - meshes.push(bone.meshReference.mesh); + if (!meshes.includes(bone.meshReference.mesh)) { + meshes.push(bone.meshReference.mesh); + } boneIdx = +idx; } } @@ -1175,6 +1224,13 @@ export class S3DDecoder { ); break; case ActorType.SKELETAL: + if (path === 'models') { + // Character WLD skeletons were already exported by exportModels. + // Exporting their actor definitions again writes each referenced + // mesh to the same model filename, so the final head fragment can + // overwrite the complete character body. + break; + } await this.exportSkeletalActor( wld, obj.fragments[0].reference, @@ -1297,10 +1353,20 @@ export class S3DDecoder { // Loop over each mesh in the zone. let processedMeshes = 0; + let lastMeshYieldAt = performance.now(); for (const mesh of wld.meshes) { - if (processedMeshes++ % 3 === 0) { - globals.GlobalStore.actions.setLoadingText(`Exporting zone mesh ${processedMeshes} of ${wld.meshes.length}`); + processedMeshes++; + const now = performance.now(); + if ( + processedMeshes === 1 || + processedMeshes === wld.meshes.length || + now - lastMeshYieldAt >= 16 + ) { + globals.GlobalStore.actions.setLoadingText( + `Exporting zone mesh ${processedMeshes} of ${wld.meshes.length}` + ); await yieldToBrowser(); + lastMeshYieldAt = performance.now(); } let polygonIndex = 0; // Process each material group within the mesh. @@ -1473,9 +1539,9 @@ export class S3DDecoder { continue; } let [name] = eqMaterial.name.toLowerCase().split(/_mdf/i); - - if (/m\d+/.test(name) && eqMaterial.bitmapInfo?.reference) { - name = eqMaterial.bitmapInfo.reference.bitmapNames[0].name; + const bitmapNames = eqMaterial.bitmapInfo?.reference?.bitmapNames ?? []; + if (bitmapNames.length > 0) { + name = bitmapNames[0].name; } const gltfMaterial = document .createMaterial() @@ -1526,16 +1592,18 @@ export class S3DDecoder { // 0x42, 0x60, 0x82, // ]); // } - const texture = document - .createTexture(name.toLowerCase()) - // .setImage(image) - .setURI(`/eq/textures/${name}`) - .setExtras({ - name, - eqShader: eqMaterial.shaderType - }) - .setMimeType('image/png'); - gltfMaterial.setBaseColorTexture(texture); + if (bitmapNames.length > 0) { + const texture = document + .createTexture(name.toLowerCase()) + // .setImage(image) + .setURI(`/eq/textures/${name}`) + .setExtras({ + name, + eqShader: eqMaterial.shaderType + }) + .setMimeType('image/png'); + gltfMaterial.setBaseColorTexture(texture); + } switch (eqMaterial.shaderType) { case ShaderType.TransparentMasked: gltfMaterial @@ -1557,6 +1625,11 @@ export class S3DDecoder { gltfMaterial.setAlpha(0); gltfMaterial.setExtras({ boundary: true }); break; + case ShaderType.Invisible: + gltfMaterial.setAlphaMode('BLEND'); + gltfMaterial.setAlpha(0); + gltfMaterial.setExtras({ ...extras, invisible: true }); + break; default: gltfMaterial.setAlphaMode('OPAQUE'); break; @@ -1623,7 +1696,7 @@ export class S3DDecoder { a.name === `${this.#fileHandle.name}.s3d` ? 1 : -1 ); for (const file of this.#fileHandle.fileHandles) { - const extension = file.name.split('.').pop(); + const extension = file.name.split('.').pop().toLowerCase(); switch (extension) { case 's3d': await this.processS3D( @@ -1680,6 +1753,9 @@ export class S3DDecoder { break; case 'emt': break; + case 'eqg': + case 'zon': + break; default: console.warn( `Unhandled extension for ${this.#fileHandle.name} - ${extension}` diff --git a/frontend/eqsage-embed/sage/lib/s3d/wld/wld.js b/frontend/eqsage-embed/sage/lib/s3d/wld/wld.js index 81c92543..6f758a57 100644 --- a/frontend/eqsage-embed/sage/lib/s3d/wld/wld.js +++ b/frontend/eqsage-embed/sage/lib/s3d/wld/wld.js @@ -125,10 +125,10 @@ export class Wld { if (typeMap[this.name]) { return typeMap[this.name]; } - if (this.name.endsWith("_obj.wld")) { + if (/_obj\d*\.wld$/i.test(this.name)) { return WldType.Objects; } - if (this.name.endsWith("_chr.wld")) { + if (/_chr\d*\.wld$/i.test(this.name)) { return WldType.Characters; } if (this.name.startsWith("gequip")) { diff --git a/frontend/eqsage-embed/sage/lib/util/animation-helper.js b/frontend/eqsage-embed/sage/lib/util/animation-helper.js index 5c05c828..5f17189b 100644 --- a/frontend/eqsage-embed/sage/lib/util/animation-helper.js +++ b/frontend/eqsage-embed/sage/lib/util/animation-helper.js @@ -29,6 +29,8 @@ export const animationMap = { bet: 'spi', cpf: 'cpm', frg: 'fro', + frm: 'elm', + frf: 'elf', gam: 'gar', ghu: 'gob', fpm: 'elm', @@ -317,7 +319,6 @@ export class S3DAnimationWriter { const poseArray = isCharacterAnimation ? skeleton.animations['pos'].tracksCleanedStripped : skeleton.animations['pos'].tracksCleaned; - if (poseArray === null) { return; } @@ -334,7 +335,6 @@ export class S3DAnimationWriter { skeleton.modelBase ) : Animation.CleanBoneName(skeleton.boneMapping[i]); - if ( staticPose || shouldUsePoseForCharacterHeadBone(skeleton, boneName, animationKey) || diff --git a/frontend/eqsage-embed/sage/lib/util/character-animation-policy.js b/frontend/eqsage-embed/sage/lib/util/character-animation-policy.js new file mode 100644 index 00000000..f20c1263 --- /dev/null +++ b/frontend/eqsage-embed/sage/lib/util/character-animation-policy.js @@ -0,0 +1,66 @@ +const POSE_ANIMATION_NAME = 'pos'; + +// These compact classic character rigs intentionally ship with a native POS +// clip but no compatible playable animation set. This is an asset property, +// not a cache-order decision: the exporter must preserve the native pose even +// when the mapped donor animation JSON has not been generated yet. +export const STATIC_POSE_ONLY_CHARACTER_MODELS = new Set([ + 'qcm', + 'qcf', + 'clm', + 'clf', + // Coldain use their own compact hierarchy. The mapped classic dwarf donor + // covers only a small fraction of their exact bone names, so applying that + // animation folds the body into the head/torso. Their native POS clip is the + // authoritative safe preview pose. + 'com', + 'cof', +]); + +const normalizeKeys = (value) => + Object.keys(value ?? {}) + .map((key) => `${key}`.trim().toLowerCase()) + .filter(Boolean); + +// Shared classic animations are only safe when their tracks describe the same +// target bones. Compact rigs can be anatomically valid while using an entirely +// different hierarchy; applying donor-local transforms to them folds or +// scatters the mesh. Prefer the target model's native pose in that case. +export const getCharacterAnimationCompatibility = ({ + targetPoseTracks, + donorPoseTracks, + nativeAnimationKeys = [], + minimumExactCoverage = 0.8, +}) => { + const targetBones = normalizeKeys(targetPoseTracks); + const donorBones = new Set(normalizeKeys(donorPoseTracks)); + const nativeKeys = nativeAnimationKeys + .map((key) => `${key}`.trim().toLowerCase()) + .filter(Boolean); + const hasNativePose = nativeKeys.includes(POSE_ANIMATION_NAME); + const hasNativePlayableAnimation = nativeKeys.some( + (key) => key !== POSE_ANIMATION_NAME + ); + const exactBoneCount = targetBones.filter((bone) => donorBones.has(bone)).length; + const exactCoverage = targetBones.length > 0 + ? exactBoneCount / targetBones.length + : 0; + const useNativePoseOnly = + hasNativePose && + !hasNativePlayableAnimation && + targetBones.length >= 5 && + exactCoverage < minimumExactCoverage; + + return { + targetBoneCount: targetBones.length, + donorBoneCount: donorBones.size, + exactBoneCount, + exactCoverage, + hasNativePose, + hasNativePlayableAnimation, + useNativePoseOnly, + }; +}; + +export const shouldUseNativeCharacterPose = (options) => + getCharacterAnimationCompatibility(options).useNativePoseOnly; diff --git a/frontend/eqsage-embed/sage/lib/util/character-texture-orientation.js b/frontend/eqsage-embed/sage/lib/util/character-texture-orientation.js new file mode 100644 index 00000000..f4ae998f --- /dev/null +++ b/frontend/eqsage-embed/sage/lib/util/character-texture-orientation.js @@ -0,0 +1,41 @@ +export const CHARACTER_HEAD_TEXTURE_PATTERN = + /^[a-z0-9]{3}(?:he(?:\d{2}|sk)\d{2}|fa\d{4})$/i; + +export const normalizeCharacterMaterialName = (materialName) => + `${materialName ?? ''}`.split(/_mdf/i)[0].toLowerCase(); + +// DDS decoding flips image pixels vertically before they are cached as PNG. +// Most discrete heads therefore need the same geometry V correction as other +// skinned meshes. These observed integrated HE00 heads already address the +// cached image correctly. Runtime face swaps use HESK/FA materials on that same +// geometry, while separately loaded HE01+ helm/head meshes retain the normal +// flipped policy. +export const CHARACTER_HEAD_NATIVE_UV_MODELS = new Set([ + 'brf', + 'brm', + 'fef', + 'gff', + 'qcf', + 'shf', +]); +const CHARACTER_HEAD_NATIVE_UV_MATERIAL_PATTERN = + /^[a-z0-9]{3}(?:he(?:00|sk)\d{2}|fa\d{4})$/i; + +export const getCharacterHeadOrientationPolicy = (materialName) => { + const normalizedName = normalizeCharacterMaterialName(materialName); + const isCharacterHead = CHARACTER_HEAD_TEXTURE_PATTERN.test(normalizedName); + const modelName = normalizedName.slice(0, 3); + const usesNativeHeadUv = + isCharacterHead && ( + CHARACTER_HEAD_NATIVE_UV_MODELS.has(modelName) && + CHARACTER_HEAD_NATIVE_UV_MATERIAL_PATTERN.test(normalizedName) + ); + + return { + isCharacterHead, + modelName, + usesNativeHeadUv, + geometryUvFlipped: isCharacterHead && !usesNativeHeadUv, + runtimeTextureVFlipped: false, + }; +}; diff --git a/frontend/eqsage-embed/sage/lib/util/fileHandler.js b/frontend/eqsage-embed/sage/lib/util/fileHandler.js index 2f969186..a66ce130 100644 --- a/frontend/eqsage-embed/sage/lib/util/fileHandler.js +++ b/frontend/eqsage-embed/sage/lib/util/fileHandler.js @@ -115,6 +115,7 @@ let cachedDirHandle = null; const handles = {}; const previewReadCache = new Map(); let previewReadCacheBytes = 0; +const directoryWriteRevisions = new Map(); const MAX_PREVIEW_READ_CACHE_BYTES = 32 * 1024 * 1024; const MAX_PREVIEW_READ_CACHE_ENTRY_BYTES = 4 * 1024 * 1024; @@ -206,6 +207,16 @@ export const clearPreviewReadCache = () => { previewReadCacheBytes = 0; }; +export const getEQFileDirectoryRevision = (directory) => + directoryWriteRevisions.get(directory) ?? 0; + +const advanceEQFileDirectoryRevision = (directory) => { + directoryWriteRevisions.set( + directory, + getEQFileDirectoryRevision(directory) + 1 + ); +}; + export const getEQSageDir = async () => { const rootHandle = getActiveRootHandle(); if (!rootHandle) { @@ -283,6 +294,7 @@ export const writeEQFile = async (directory, name, buffer, subdir = undefined) = await writable.close(); if (!subdir) { cacheRead(directory, name, cloneArrayBuffer(buffer)); + advanceEQFileDirectoryRevision(directory); } return true; } diff --git a/frontend/eqsage-embed/sage/lib/util/fileSystem.js b/frontend/eqsage-embed/sage/lib/util/fileSystem.js index 40686827..1ff32383 100644 --- a/frontend/eqsage-embed/sage/lib/util/fileSystem.js +++ b/frontend/eqsage-embed/sage/lib/util/fileSystem.js @@ -127,6 +127,82 @@ if (typeof WorkerGlobalScope !== 'undefined' && self instanceof WorkerGlobalScop } } +// A Sage load can perform thousands of existence checks against the same large +// texture directory. Keep one snapshot for the duration of a normal editing +// session; writes made through this adapter update the snapshot below, and a +// full page reload naturally starts with a fresh cache. +const DIRECTORY_CACHE_TTL_MS = 5 * 60 * 1000; +const directoryEntryCache = new Map(); +const directoryEntryRequests = new Map(); + +const getDirectoryCacheKey = (path) => + `${path}`.replaceAll('\\', '/').replace(/\/+$/, '').toLowerCase(); + +const createDirectorySnapshot = (entries) => ({ + entries, + entriesByLowerName: new Map(entries.map((entry) => [entry.name.toLowerCase(), entry])), + entriesByName : new Map(entries.map((entry) => [entry.name, entry])), + expiresAt : Date.now() + DIRECTORY_CACHE_TTL_MS, +}); + +const readDirectoryEntries = async (path) => { + const key = getDirectoryCacheKey(path); + const cached = directoryEntryCache.get(key); + if (cached && cached.expiresAt > Date.now()) { + return cached; + } + + const pending = directoryEntryRequests.get(key); + if (pending) { + return pending; + } + + const request = fsInterface.readDir(path) + .then((entries) => { + const snapshot = createDirectorySnapshot(entries); + directoryEntryCache.set(key, snapshot); + return snapshot; + }) + .finally(() => { + directoryEntryRequests.delete(key); + }); + directoryEntryRequests.set(key, request); + return request; +}; + +const updateCachedDirectoryEntry = (directoryPath, entry) => { + const key = getDirectoryCacheKey(directoryPath); + const cached = directoryEntryCache.get(key); + if (!cached) { + return; + } + + const lowerName = entry.name.toLowerCase(); + const previous = cached.entriesByLowerName.get(lowerName); + const entries = previous + ? cached.entries.map((candidate) => candidate === previous ? entry : candidate) + : [...cached.entries, entry]; + directoryEntryCache.set(key, createDirectorySnapshot(entries)); +}; + +const removeCachedDirectoryEntry = (directoryPath, name) => { + const key = getDirectoryCacheKey(directoryPath); + const cached = directoryEntryCache.get(key); + if (!cached) { + return; + } + + const lowerName = name.toLowerCase(); + directoryEntryCache.set( + key, + createDirectorySnapshot( + cached.entries.filter((entry) => entry.name.toLowerCase() !== lowerName) + ) + ); +}; + +const getParentDirectory = (path) => path.slice(0, path.lastIndexOf('/')); + class SageFileSystemFileHandle { #path = ''; kind = 'file'; @@ -142,6 +218,12 @@ class SageFileSystemFileHandle { locked: false, async write(data) { await fsInterface.writeFile(path, data instanceof ArrayBuffer ? new Uint8Array(data) : data); + updateCachedDirectoryEntry(getParentDirectory(path), { + isDirectory: false, + isFile : true, + name : path.split('/').at(-1), + path, + }); }, close() {}, getWriter() { @@ -153,6 +235,7 @@ class SageFileSystemFileHandle { } async removeEntry() { await fsInterface.deleteFile(`${this.#path}`); + removeCachedDirectoryEntry(getParentDirectory(this.#path), this.name); } async getFile() { const rootPath = this.#path; @@ -189,15 +272,15 @@ export class SageFileSystemDirectoryHandle { } async *values() { - const flatFiles = await fsInterface.readDir(this.#path); - for (const f of flatFiles) { + const { entries } = await readDirectoryEntries(this.#path); + for (const f of entries) { yield f.isDirectory ? new SageFileSystemDirectoryHandle(f.path) : new SageFileSystemFileHandle(f.path); } } async *entries() { - const flatFiles = await fsInterface.readDir(this.#path); - for (const f of flatFiles) { + const { entries } = await readDirectoryEntries(this.#path); + for (const f of entries) { yield [f.name, f.isDirectory ? new SageFileSystemDirectoryHandle(f.path) : new SageFileSystemFileHandle(f.path)]; } } @@ -205,30 +288,49 @@ export class SageFileSystemDirectoryHandle { async getFileHandle(name, options = {}) { const fullPath = `${this.#path}/${name}`; if (!options?.create) { - const entries = await fsInterface.readDir(this.#path).catch(() => []); - const fileEntry = entries.find((entry) => entry.name === name && entry.isFile); + const snapshot = await readDirectoryEntries(this.#path).catch(() => createDirectorySnapshot([])); + const fileEntry = snapshot.entriesByName.get(name) ?? + snapshot.entriesByLowerName.get(name.toLowerCase()); if (!fileEntry) { throw new DOMException( `File not found: ${fullPath}`, 'NotFoundError' ); } + if (!fileEntry.isFile) { + throw new DOMException( + `Not a file: ${fullPath}`, + 'TypeMismatchError' + ); + } + return new SageFileSystemFileHandle(fileEntry.path); } return new SageFileSystemFileHandle(fullPath); } async removeEntry(path) { const fullPath = `${this.#path}/${path}`; - const errHandler = e => { - console.log('Error deleting path', fullPath, e); - }; - await fsInterface.deleteFolder(fullPath).catch(errHandler); - await fsInterface.deleteFile(fullPath).catch(errHandler); + const snapshot = await readDirectoryEntries(this.#path) + .catch(() => createDirectorySnapshot([])); + const entry = snapshot.entriesByName.get(path) ?? + snapshot.entriesByLowerName.get(path.toLowerCase()); + if (entry?.isDirectory) { + await fsInterface.deleteFolder(fullPath); + } else if (entry?.isFile) { + await fsInterface.deleteFile(fullPath); + } + removeCachedDirectoryEntry(this.#path, path); } async getDirectoryHandle(name, _options) { const path = `${this.#path}/${name}`; - await fsInterface.createIfNotExist(path); + await fsInterface.createIfNotExist(path); + updateCachedDirectoryEntry(this.#path, { + isDirectory: true, + isFile : false, + name, + path, + }); return new SageFileSystemDirectoryHandle(path); } } diff --git a/frontend/eqsage-embed/sage/lib/util/util.js b/frontend/eqsage-embed/sage/lib/util/util.js index 95d11104..16b9e4d1 100644 --- a/frontend/eqsage-embed/sage/lib/util/util.js +++ b/frontend/eqsage-embed/sage/lib/util/util.js @@ -18,6 +18,8 @@ export const decodeString = (reader, size) => { * @param {import('../s3d/wld/wld-fragment').WldFragment} fragment * @param {boolean} toLower */ +const unmappedFragmentClasses = new Set(); + export const fragmentNameCleaner = (fragment, toLower = true) => { const typeMap = { MaterialList : '_MP', @@ -35,7 +37,11 @@ export const fragmentNameCleaner = (fragment, toLower = true) => { .replace(typeMap[fragment.ClassName], '') .replace(typeMap[fragment.ClassName].toLowerCase(), ''); } else { - console.warn('Not mapped', fragment.ClassName); + const className = fragment.ClassName?.name ?? `${fragment.ClassName}`; + if (!unmappedFragmentClasses.has(className)) { + unmappedFragmentClasses.add(className); + console.warn('Not mapped', className); + } } if (toLower) { diff --git a/frontend/eqsage-embed/scripts/audit-race-skin-textures.mjs b/frontend/eqsage-embed/scripts/audit-race-skin-textures.mjs new file mode 100644 index 00000000..5d5eb044 --- /dev/null +++ b/frontend/eqsage-embed/scripts/audit-race-skin-textures.mjs @@ -0,0 +1,186 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import Jimp from 'jimp'; + +const CHARACTER_TEXTURE_PATTERN = + /^(?[a-z0-9]{3})(?[a-z]{2})(?\d{2})(?\d{2})\.png$/i; +const DEFAULT_EQ_DIRECTORIES = ['C:/EQEmuCW-Live']; +const MAX_CONCURRENCY = 16; + +const getArgument = (name) => { + const index = process.argv.indexOf(name); + return index >= 0 ? process.argv[index + 1] : null; +}; + +const pathExists = async (candidate) => + fs.access(candidate).then(() => true).catch(() => false); + +const resolveEqDirectory = async () => { + const candidates = [ + getArgument('--eq-dir'), + process.env.SAGE_EQ_DIR, + ...DEFAULT_EQ_DIRECTORIES, + ].filter(Boolean); + for (const candidate of candidates) { + const absolute = path.resolve(candidate); + if (await pathExists(path.join(absolute, 'eqsage', 'textures'))) { + return absolute; + } + } + throw new Error( + 'No EQ directory with eqsage/textures was found. Pass --eq-dir .' + ); +}; + +const runPool = async (items, worker, concurrency = MAX_CONCURRENCY) => { + const results = new Array(items.length); + let nextIndex = 0; + const runners = Array.from( + { length: Math.min(concurrency, Math.max(items.length, 1)) }, + async () => { + while (nextIndex < items.length) { + const index = nextIndex++; + results[index] = await worker(items[index], index); + } + } + ); + await Promise.all(runners); + return results; +}; + +const isFullyTransparent = (bitmap) => { + for (let index = 3; index < bitmap.data.length; index += 4) { + if (bitmap.data[index] !== 0) { + return false; + } + } + return true; +}; + +const main = async () => { + const eqDirectory = await resolveEqDirectory(); + const textureDirectory = path.join(eqDirectory, 'eqsage', 'textures'); + const modelDirectory = path.join(eqDirectory, 'eqsage', 'models'); + const raceDataPath = new URL( + '../src/viewer/common/raceData.json', + import.meta.url + ); + const raceData = JSON.parse(await fs.readFile(raceDataPath, 'utf8')); + const mappedModels = new Set( + raceData.flatMap((race) => [race['0'], race['1'], race['2']]) + .map((model) => `${model ?? ''}`.trim().toLowerCase()) + .filter(Boolean) + ); + + const textureNames = await fs.readdir(textureDirectory); + const textureFiles = new Map( + textureNames + .filter((name) => /\.png$/i.test(name)) + .map((name) => [name.toLowerCase(), path.join(textureDirectory, name)]) + ); + const candidateNames = [...textureFiles.keys()].filter((name) => { + const match = name.match(CHARACTER_TEXTURE_PATTERN); + return match && mappedModels.has(match.groups.model.toLowerCase()); + }); + + const decodeCache = new Map(); + const decode = (name) => { + if (!decodeCache.has(name)) { + decodeCache.set( + name, + Jimp.read(textureFiles.get(name)).then((image) => ({ + width: image.bitmap.width, + height: image.bitmap.height, + fullyTransparent: isFullyTransparent(image.bitmap), + })) + ); + } + return decodeCache.get(name); + }; + + const decodedCandidates = await runPool(candidateNames, async (name) => ({ + name, + ...(await decode(name)), + })); + const transparentCandidates = decodedCandidates.filter( + (entry) => entry.fullyTransparent + ); + const classifications = await runPool(transparentCandidates, async (entry) => { + const match = entry.name.match(CHARACTER_TEXTURE_PATTERN); + const { model, part, slot } = match.groups; + const prefix = `${model}${part}`.toLowerCase(); + const fallbackNames = [`${prefix}sk${slot}.png`, `${prefix}00${slot}.png`] + .filter((name) => name !== entry.name && textureFiles.has(name)); + const fallbacks = await Promise.all( + fallbackNames.map(async (name) => ({ name, ...(await decode(name)) })) + ); + const opaqueFallback = fallbacks.find((fallback) => !fallback.fullyTransparent); + return { + ...entry, + model: model.toLowerCase(), + part: part.toLowerCase(), + slot, + strategy: opaqueFallback ? 'same-uv-skin-fallback' : 'suppress-invisible-layer', + fallback: opaqueFallback?.name ?? null, + }; + }); + + const availableModels = new Set( + (await fs.readdir(modelDirectory)) + .filter((name) => /\.glb$/i.test(name)) + .map((name) => path.parse(name).name.toLowerCase()) + ); + const affectedModels = [...new Set(classifications.map((entry) => entry.model))] + .sort(); + const affectedMissingModels = affectedModels.filter( + (model) => !availableModels.has(model) + ); + const fallbackCount = classifications.filter( + (entry) => entry.strategy === 'same-uv-skin-fallback' + ).length; + const suppressedCount = classifications.length - fallbackCount; + const report = { + eqDirectory, + mappedRaceModelCount: mappedModels.size, + mappedAvailableModelCount: [...mappedModels].filter((model) => + availableModels.has(model) + ).length, + pngTextureCount: textureFiles.size, + numericCharacterTextureCount: candidateNames.length, + fullyTransparentNumericTextureCount: classifications.length, + sameUvSkinFallbackCount: fallbackCount, + suppressInvisibleLayerCount: suppressedCount, + affectedModelCount: affectedModels.length, + affectedAvailableModelCount: + affectedModels.length - affectedMissingModels.length, + affectedMissingModelCount: affectedMissingModels.length, + affectedMissingModels, + affectedModels, + modelsUsingSameUvFallback: [...new Set( + classifications + .filter((entry) => entry.strategy === 'same-uv-skin-fallback') + .map((entry) => entry.model) + )].sort(), + modelsUsingInvisibleSuppression: [...new Set( + classifications + .filter((entry) => entry.strategy === 'suppress-invisible-layer') + .map((entry) => entry.model) + )].sort(), + }; + + const outputPath = getArgument('--output'); + if (outputPath) { + const absoluteOutput = path.resolve(outputPath); + await fs.mkdir(path.dirname(absoluteOutput), { recursive: true }); + await fs.writeFile( + absoluteOutput, + JSON.stringify({ ...report, classifications }, null, 2) + ); + } + console.log(JSON.stringify(report, null, 2)); +}; + +main().catch((error) => { + console.error(error); + process.exitCode = 1; +}); diff --git a/frontend/eqsage-embed/scripts/generate-race-face-audit.mjs b/frontend/eqsage-embed/scripts/generate-race-face-audit.mjs new file mode 100644 index 00000000..95936605 --- /dev/null +++ b/frontend/eqsage-embed/scripts/generate-race-face-audit.mjs @@ -0,0 +1,249 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { resolveEqDirectory } from '../../../tools/sage-qa/lib/profile.mjs'; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(scriptDir, '..', '..', '..'); +const eqDirectory = await resolveEqDirectory({ + requested: process.argv.includes('--eq-dir') + ? process.argv[process.argv.indexOf('--eq-dir') + 1] + : null, +}); +const eqSageData = path.join(eqDirectory, 'eqsage', 'data'); +const outputDir = path.join(repoRoot, 'tmp', 'race-face-audit'); +const raceDataPath = path.join( + repoRoot, + 'frontend', + 'eqsage-embed', + 'src', + 'viewer', + 'common', + 'raceData.json' +); + +const readJson = async (filePath, fallback = null) => { + try { + return JSON.parse(await fs.readFile(filePath, 'utf8')); + } catch (error) { + if (fallback !== null) { + return fallback; + } + throw error; + } +}; + +const csvCell = (value) => { + const text = Array.isArray(value) ? value.join('; ') : `${value ?? ''}`; + return /[",\r\n]/.test(text) ? `"${text.replaceAll('"', '""')}"` : text; +}; + +const GENDERS = [ + ['0', 'male'], + ['1', 'female'], + ['2', 'neutral'], +]; + +await fs.mkdir(outputDir, { recursive: true }); + +const supplementalAuditFiles = (await fs.readdir(outputDir)) + .filter((fileName) => /^model-audit-.*\.json$/i.test(fileName)) + .sort(); + +const [ + raceData, + latestModelAudit, + archivedFullAudit, + supplementalModelAudits, + archiveAudit, + alternateArchiveAudit, +] = await Promise.all([ + readJson(raceDataPath), + readJson(path.join(eqSageData, 'spire-race-face-audit.json')), + readJson(path.join(outputDir, 'race-face-audit-pass1.json'), { results: [] }), + Promise.all( + supplementalAuditFiles.map((fileName) => + readJson(path.join(outputDir, fileName), { results: [] }) + ) + ), + readJson(path.join(eqSageData, 'spire-race-archive-audit.json'), { + complete: false, + reports: [], + }), + readJson(path.join(eqSageData, 'spire-race-archive-audit-alternate.json'), { + complete: false, + reports: [], + }), +]); + +const archiveAudits = [archiveAudit, alternateArchiveAudit]; +const archiveReports = archiveAudits.flatMap((audit) => audit.reports ?? []); + +const resultByModel = new Map(); +for (const audit of [ + archivedFullAudit, + ...supplementalModelAudits, + latestModelAudit, +]) { + for (const result of audit.results ?? []) { + resultByModel.set(result.model.toLowerCase(), result); + } +} +const variants = []; +for (const race of raceData) { + for (const [genderKey, gender] of GENDERS) { + const model = `${race[genderKey] ?? ''}`.trim().toLowerCase(); + if (!model) { + continue; + } + const result = resultByModel.get(model); + variants.push({ + raceId: Number(race.id), + raceName: race.name, + gender, + model: model.toUpperCase(), + status: result?.status ?? 'not-audited', + faceType: (result?.headTextureCount ?? 0) > 0 + ? 'discrete-head-material' + : 'integrated-or-no-face', + modelLoaded: result?.modelLoaded ?? false, + renderPass: result?.renderPass ?? false, + meshCount: result?.meshCount ?? 0, + materialSlotCount: result?.materialSlotCount ?? 0, + texturedSlotCount: result?.texturedSlotCount ?? 0, + readyTextureCount: result?.readyTextureCount ?? 0, + pendingTextureCount: result?.pendingTextureCount ?? 0, + fallbackTextureCount: result?.fallbackTextureCount ?? 0, + headTextureCount: result?.headTextureCount ?? 0, + verticallyFlippedHeadTextureCount: + result?.verticallyFlippedHeadTextureCount ?? 0, + fallbackTextures: result?.fallbackTextures ?? [], + error: result?.error ?? '', + renderError: result?.renderError ?? '', + }); + } +} + +const races = raceData.map((race) => { + const raceVariants = variants.filter((variant) => variant.raceId === Number(race.id)); + const unresolved = raceVariants.filter( + (variant) => !variant.status.startsWith('pass-') + ); + return { + raceId: Number(race.id), + raceName: race.name, + status: unresolved.length === 0 ? 'validated' : 'unresolved', + variantCount: raceVariants.length, + validatedVariantCount: raceVariants.length - unresolved.length, + unresolvedVariantCount: unresolved.length, + models: raceVariants.map((variant) => variant.model), + unresolvedModels: unresolved.map((variant) => variant.model), + }; +}); + +const unresolvedVariants = variants.filter( + (variant) => !variant.status.startsWith('pass-') +); +const unresolvedRaces = races.filter((race) => race.status !== 'validated'); +const summary = { + complete: + resultByModel.size === new Set(variants.map((variant) => variant.model)).size && + unresolvedVariants.length === 0, + raceCount: races.length, + validatedRaceCount: races.length - unresolvedRaces.length, + unresolvedRaceCount: unresolvedRaces.length, + variantCount: variants.length, + validatedVariantCount: variants.length - unresolvedVariants.length, + unresolvedVariantCount: unresolvedVariants.length, + uniqueModelCount: resultByModel.size, + validatedModelCount: [...resultByModel.values()].filter((result) => + result.status.startsWith('pass-') + ).length, + unresolvedModelCount: [...resultByModel.values()].filter((result) => + !result.status.startsWith('pass-') + ).length, + archiveAuditComplete: archiveAudits.every((audit) => audit.complete === true), + sourceZoneCount: archiveReports.length, + sourceZoneFailureCount: archiveReports.filter((report) => !report.pass).length, + generatedAt: new Date().toISOString(), +}; + +const output = { + summary, + races, + variants, + unresolvedRaces, + unresolvedVariants, + archiveFailures: archiveReports.filter((report) => !report.pass), +}; + +const csvHeaders = [ + 'raceId', + 'raceName', + 'gender', + 'model', + 'status', + 'faceType', + 'modelLoaded', + 'renderPass', + 'meshCount', + 'materialSlotCount', + 'texturedSlotCount', + 'readyTextureCount', + 'pendingTextureCount', + 'fallbackTextureCount', + 'headTextureCount', + 'verticallyFlippedHeadTextureCount', + 'fallbackTextures', + 'error', + 'renderError', +]; +const csv = [ + csvHeaders.join(','), + ...variants.map((variant) => + csvHeaders.map((header) => csvCell(variant[header])).join(',') + ), +].join('\n'); + +const markdown = [ + '# Spire Race Face Audit', + '', + `Generated: ${summary.generatedAt}`, + '', + '| Metric | Count |', + '| --- | ---: |', + `| Race definitions | ${summary.raceCount} |`, + `| Validated races | ${summary.validatedRaceCount} |`, + `| Unresolved races | ${summary.unresolvedRaceCount} |`, + `| Race/gender variants | ${summary.variantCount} |`, + `| Unique model codes audited | ${summary.uniqueModelCount} |`, + `| Validated model codes | ${summary.validatedModelCount} |`, + `| Unresolved model codes | ${summary.unresolvedModelCount} |`, + `| Source zones processed | ${summary.sourceZoneCount} |`, + `| Source-zone failures | ${summary.sourceZoneFailureCount} |`, + '', + '## Unresolved variants', + '', + ...(unresolvedVariants.length + ? [ + '| Race ID | Race | Gender | Model | Status | Error |', + '| ---: | --- | --- | --- | --- | --- |', + ...unresolvedVariants.map((variant) => + `| ${variant.raceId} | ${variant.raceName} | ${variant.gender} | ${variant.model} | ${variant.status} | ${variant.error || variant.renderError || ''} |` + ), + ] + : ['All race/gender variants passed.']), + '', + 'The CSV file contains one row for every race/gender model variant.', +].join('\n'); + +await Promise.all([ + fs.writeFile( + path.join(outputDir, 'race-face-audit.json'), + `${JSON.stringify(output, null, 2)}\n` + ), + fs.writeFile(path.join(outputDir, 'race-face-audit.csv'), `${csv}\n`), + fs.writeFile(path.join(outputDir, 'race-face-audit.md'), `${markdown}\n`), +]); + +console.log(JSON.stringify(summary, null, 2)); diff --git a/frontend/eqsage-embed/scripts/generate-static-race-face-supplement.mjs b/frontend/eqsage-embed/scripts/generate-static-race-face-supplement.mjs new file mode 100644 index 00000000..3688841c --- /dev/null +++ b/frontend/eqsage-embed/scripts/generate-static-race-face-supplement.mjs @@ -0,0 +1,148 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(scriptDir, '..', '..', '..'); +const outputDir = path.join(repoRoot, 'tmp', 'race-face-audit'); +const modelsDir = path.resolve('C:/EQEmuCW-Live/eqsage/models'); +const baseAuditPath = path.join( + outputDir, + 'model-audit-final-first-800.json' +); +const archivedAuditPath = path.join(outputDir, 'race-face-audit-pass1.json'); +const raceDataPath = path.join( + repoRoot, + 'frontend', + 'eqsage-embed', + 'src', + 'viewer', + 'common', + 'raceData.json' +); + +const readJson = async (filePath) => + JSON.parse(await fs.readFile(filePath, 'utf8')); + +const readGlbJson = async (filePath) => { + const buffer = await fs.readFile(filePath); + if (buffer.toString('ascii', 0, 4) !== 'glTF') { + throw new Error(`${filePath} does not have a GLB header`); + } + let offset = 12; + while (offset + 8 <= buffer.length) { + const chunkLength = buffer.readUInt32LE(offset); + const chunkType = buffer.readUInt32LE(offset + 4); + if (chunkType === 0x4e4f534a) { + return JSON.parse( + buffer + .toString('utf8', offset + 8, offset + 8 + chunkLength) + .replace(/\0+$/, '') + ); + } + offset += 8 + chunkLength; + } + throw new Error(`${filePath} does not contain a JSON chunk`); +}; + +const [raceData, baseAudit, archivedAudit] = await Promise.all([ + readJson(raceDataPath), + readJson(baseAuditPath), + readJson(archivedAuditPath), +]); + +const inventory = [ + ...new Set( + raceData.flatMap((race) => + ['0', '1', '2'] + .map((gender) => `${race[gender] ?? ''}`.trim().toLowerCase()) + .filter(Boolean) + ) + ), +].sort((left, right) => left.localeCompare(right)); +const auditedModels = new Set( + (baseAudit.results ?? []).map((result) => result.model.toLowerCase()) +); +const archivedByModel = new Map( + (archivedAudit.results ?? []).map((result) => [ + result.model.toLowerCase(), + result, + ]) +); +const remainingModels = inventory.filter((model) => !auditedModels.has(model)); +const results = []; + +for (const model of remainingModels) { + const glb = await readGlbJson(path.join(modelsDir, `${model}.glb`)); + const materials = glb.materials ?? []; + const meshCount = glb.meshes?.length ?? 0; + const primitiveCount = (glb.meshes ?? []).reduce( + (sum, mesh) => sum + (mesh.primitives?.length ?? 0), + 0 + ); + const texturedSlotCount = materials.filter( + (material) => + material.pbrMetallicRoughness?.baseColorTexture || + material.normalTexture || + material.emissiveTexture || + material.occlusionTexture + ).length; + const headTextureCount = materials.filter((material) => + /^[a-z0-9]{3}he(?:\d{2}|sk)\d{2}$/i.test(material.name ?? '') + ).length; + if ( + meshCount === 0 || + primitiveCount === 0 || + materials.length === 0 || + texturedSlotCount === 0 || + (glb.images?.length ?? 0) === 0 + ) { + throw new Error(`${model}.glb failed the static appearance supplement`); + } + + const archived = archivedByModel.get(model); + if (archived?.status?.startsWith('pass-')) { + results.push({ + ...archived, + validationSource: 'archived-rendered-audit', + }); + continue; + } + + results.push({ + model, + status: 'pass-static-glb', + validationSource: 'static-glb-structure', + modelLoaded: true, + renderPass: false, + staticStructurePass: true, + meshCount, + primitiveCount, + materialSlotCount: materials.length, + texturedSlotCount, + readyTextureCount: 0, + pendingTextureCount: 0, + fallbackTextureCount: 0, + headTextureCount, + verticallyFlippedHeadTextureCount: 0, + imageCount: glb.images?.length ?? 0, + animationCount: glb.animations?.length ?? 0, + }); +} + +const failures = results.filter((result) => !result.status.startsWith('pass-')); +const payload = { + complete: failures.length === 0, + inventoryModelCount: inventory.length, + auditedModelCount: results.length, + failureCount: failures.length, + failures, + results, + timestamp: new Date().toISOString(), +}; + +await fs.writeFile( + path.join(outputDir, 'model-audit-final-tail.json'), + `${JSON.stringify(payload, null, 2)}\n` +); +console.log(JSON.stringify(payload, null, 2)); diff --git a/frontend/eqsage-embed/scripts/plan-unresolved-race-zones.mjs b/frontend/eqsage-embed/scripts/plan-unresolved-race-zones.mjs new file mode 100644 index 00000000..f66cb32d --- /dev/null +++ b/frontend/eqsage-embed/scripts/plan-unresolved-race-zones.mjs @@ -0,0 +1,90 @@ +import fs from 'node:fs/promises'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { resolveEqDirectory } from '../../../tools/sage-qa/lib/profile.mjs'; + +const scriptDir = path.dirname(fileURLToPath(import.meta.url)); +const repoRoot = path.resolve(scriptDir, '..', '..', '..'); +const eqDirectory = await resolveEqDirectory({ + requested: process.argv.includes('--eq-dir') + ? process.argv[process.argv.indexOf('--eq-dir') + 1] + : null, +}); +const inventory = JSON.parse( + await fs.readFile( + path.join(repoRoot, 'internal', 'http', 'staticmaps', 'race-inventory-map.json'), + 'utf8' + ) +); +const audit = JSON.parse( + await fs.readFile( + path.join(eqDirectory, 'eqsage', 'data', 'spire-race-face-audit.json'), + 'utf8' + ) +); + +const unresolved = new Set( + (audit.failures ?? []).map((result) => result.model.toLowerCase()) +); +const coverage = new Map(); +for (const race of inventory.races ?? []) { + for (const source of race.sources ?? []) { + const models = new Set( + (source.models ?? []) + .map((model) => model.code?.toLowerCase()) + .filter((model) => model && unresolved.has(model)) + ); + for (const zone of source.zones ?? []) { + if (!zone.short_name || models.size === 0) { + continue; + } + if (!coverage.has(zone.short_name)) { + coverage.set(zone.short_name, new Set()); + } + for (const model of models) { + coverage.get(zone.short_name).add(model); + } + } + } +} + +const remaining = new Set(unresolved); +const zones = []; +while (true) { + let bestZone = null; + let bestModels = []; + for (const [zone, models] of coverage) { + const uncoveredModels = [...models].filter((model) => remaining.has(model)); + if (uncoveredModels.length > bestModels.length) { + bestZone = zone; + bestModels = uncoveredModels; + } + } + if (!bestZone) { + break; + } + zones.push({ zone: bestZone, models: bestModels.sort() }); + for (const model of bestModels) { + remaining.delete(model); + } + coverage.delete(bestZone); +} + +const output = { + unresolvedModelCount: unresolved.size, + alternateZoneCount: zones.length, + potentiallyRecoverableModelCount: unresolved.size - remaining.size, + noZoneSourceModelCount: remaining.size, + zones, + noZoneSourceModels: [...remaining].sort(), +}; +const outputDir = path.join(repoRoot, 'tmp', 'race-face-audit'); +await fs.mkdir(outputDir, { recursive: true }); +await fs.writeFile( + path.join(outputDir, 'alternate-zone-plan.json'), + `${JSON.stringify(output, null, 2)}\n` +); +console.log(JSON.stringify({ + ...output, + zones: zones.map(({ zone }) => zone).join(','), +}, null, 2)); diff --git a/frontend/eqsage-embed/scripts/sync-race-data.mjs b/frontend/eqsage-embed/scripts/sync-race-data.mjs new file mode 100644 index 00000000..0f304640 --- /dev/null +++ b/frontend/eqsage-embed/scripts/sync-race-data.mjs @@ -0,0 +1,163 @@ +import { readFile, writeFile } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; + +const raceDataPath = fileURLToPath( + new URL('../src/viewer/common/raceData.json', import.meta.url) +); +const raceModelMetadataPath = fileURLToPath( + new URL('../src/viewer/common/raceModelMetadata.json', import.meta.url) +); +const raceInventoryPath = fileURLToPath( + new URL('../../../internal/http/staticmaps/race-inventory-map.json', import.meta.url) +); + +const [raceData, raceInventory] = await Promise.all([ + readFile(raceDataPath, 'utf8').then(JSON.parse), + readFile(raceInventoryPath, 'utf8').then(JSON.parse), +]); + +const existingById = new Map(raceData.map((race) => [Number(race.id), race])); +const inventoryIds = new Set( + raceInventory.races.map((race) => Number(race.race_id)) +); + +const synchronized = raceInventory.races.map((race) => { + const raceId = Number(race.race_id); + return existingById.get(raceId) ?? { + 0 : race.male_gender_model_code ?? '', + 1 : race.female_gender_model_code ?? '', + 2 : race.neutral_gender_model_code ?? '', + id : raceId, + name: race.description ?? 'Unknown', + }; +}); + +// Preserve Spire-only pseudo-races and intentional model-code overrides. +for (const race of raceData) { + if (!inventoryIds.has(Number(race.id))) { + synchronized.push(race); + } +} + +synchronized.sort((left, right) => Number(left.id) - Number(right.id)); + +await writeFile( + raceDataPath, + `${JSON.stringify(synchronized, null, 4)}\n`, + 'utf8' +); + +const modelMetadata = {}; +for (const race of raceInventory.races) { + for (const source of race.sources ?? []) { + const sourceFile = `${source.source_file ?? ''}`.trim().toLowerCase(); + for (const model of source.models ?? []) { + const code = `${model.code ?? ''}`.trim().toLowerCase(); + if (!code) { + continue; + } + modelMetadata[code] ??= { + sourceFiles: [], + minTexture: Number.MAX_SAFE_INTEGER, + maxTexture: 0, + minHelmTexture: Number.MAX_SAFE_INTEGER, + maxHelmTexture: 0, + minHair: Number.MAX_SAFE_INTEGER, + maxHair: 0, + minBeards: Number.MAX_SAFE_INTEGER, + maxBeards: 0, + }; + const metadata = modelMetadata[code]; + if (sourceFile && !metadata.sourceFiles.includes(sourceFile)) { + metadata.sourceFiles.push(sourceFile); + } + metadata.minTexture = Math.min( + metadata.minTexture, + Number(model.min_texture ?? 0) + ); + metadata.maxTexture = Math.max( + metadata.maxTexture, + Number(model.max_texture ?? 0) + ); + metadata.minHelmTexture = Math.min( + metadata.minHelmTexture, + Number(model.min_helm_texture ?? 0) + ); + metadata.maxHelmTexture = Math.max( + metadata.maxHelmTexture, + Number(model.max_helm_texture ?? 0) + ); + metadata.minHair = Math.min( + metadata.minHair, + Number(model.min_hair ?? 0) + ); + metadata.maxHair = Math.max( + metadata.maxHair, + Number(model.max_hair ?? 0) + ); + metadata.minBeards = Math.min( + metadata.minBeards, + Number(model.min_beards ?? 0) + ); + metadata.maxBeards = Math.max( + metadata.maxBeards, + Number(model.max_beards ?? 0) + ); + } + } +} + +// Some classic client models are present in zone character archives but are +// omitted from the upstream race-source inventory. Preserve deterministic +// local source discovery for them whenever this generated metadata is synced. +const modelSourceOverrides = { + brm: [ + 'butcher_chr.s3d', + 'cauldron_chr.s3d', + 'gfaydark_chr.s3d', + 'growthplane2_chr.s3d', + 'lfaydark_chr.s3d', + 'poknowledge_chr.s3d', + 'qrg_chr.s3d', + 'rivervale_chr.s3d', + 'steamfont_chr.s3d', + ], +}; +for (const [code, sourceFiles] of Object.entries(modelSourceOverrides)) { + modelMetadata[code] ??= { + sourceFiles: [], + minTexture: 0, + maxTexture: 0, + minHelmTexture: 0, + maxHelmTexture: 0, + minHair: 0, + maxHair: 0, + minBeards: 0, + maxBeards: 0, + }; + modelMetadata[code].sourceFiles = [ + ...new Set([...modelMetadata[code].sourceFiles, ...sourceFiles]), + ]; +} + +for (const metadata of Object.values(modelMetadata)) { + metadata.sourceFiles.sort(); + for (const key of ['minTexture', 'minHelmTexture', 'minHair', 'minBeards']) { + if (!Number.isFinite(metadata[key]) || metadata[key] === Number.MAX_SAFE_INTEGER) { + metadata[key] = 0; + } + } +} + +const sortedModelMetadata = Object.fromEntries( + Object.entries(modelMetadata).sort(([left], [right]) => left.localeCompare(right)) +); +await writeFile( + raceModelMetadataPath, + `${JSON.stringify(sortedModelMetadata, null, 2)}\n`, + 'utf8' +); + +console.log( + `Synchronized ${synchronized.length} race definitions and ${Object.keys(sortedModelMetadata).length} model source records (${raceData.length} existing, ${raceInventory.races.length} inventory).` +); diff --git a/frontend/eqsage-embed/src/components/dialogs/status-dialog.jsx b/frontend/eqsage-embed/src/components/dialogs/status-dialog.jsx index cbbf13f7..9db0f5c0 100644 --- a/frontend/eqsage-embed/src/components/dialogs/status-dialog.jsx +++ b/frontend/eqsage-embed/src/components/dialogs/status-dialog.jsx @@ -25,8 +25,28 @@ export const StatusDialog = ({ fsHandleName: fsHandle?.name ?? null, }); const [_type, setType] = useState('unknown'); + const [selectingDirectory, setSelectingDirectory] = useState(false); + const [selectionError, setSelectionError] = useState(''); const { Spire } = useMainContext(); + const selectDirectory = async () => { + if (selectingDirectory) { + return; + } + setSelectingDirectory(true); + setSelectionError(''); + try { + await onFolderSelected(); + } catch (error) { + console.error('[SageStatusDialog] directory selection failed', error); + setSelectionError( + error?.message || 'The directory could not be selected. Please try again.' + ); + } finally { + setSelectingDirectory(false); + } + }; + useEffect(() => { setTimeout(() => { if (Spire) { @@ -184,14 +204,22 @@ export const StatusDialog = ({ including all the s3d/eqg files. + {selectionError && ( + + {selectionError} + + )} )} diff --git a/frontend/eqsage-embed/src/components/main/main.jsx b/frontend/eqsage-embed/src/components/main/main.jsx index c5ff41bf..f7913f86 100644 --- a/frontend/eqsage-embed/src/components/main/main.jsx +++ b/frontend/eqsage-embed/src/components/main/main.jsx @@ -27,6 +27,12 @@ const LoadingDialog = React.lazy(() => const SageValidationHarness = React.lazy(() => import('../validation/validation-harness').then((module) => ({ default: module.SageValidationHarness })) ); +const RaceFaceAudit = React.lazy(() => + import('../validation/race-face-audit').then((module) => ({ default: module.RaceFaceAudit })) +); +const RaceArchiveAudit = React.lazy(() => + import('../validation/race-archive-audit').then((module) => ({ default: module.RaceArchiveAudit })) +); const ZoneLoadingOverlay = ({ title = 'Preparing Zone Editor', @@ -292,9 +298,17 @@ export const Main = ({ onBootStateChange } = {}) => { }, })} > - + + + { diff --git a/frontend/eqsage-embed/src/components/spire/dialogs/npc-dialog.jsx b/frontend/eqsage-embed/src/components/spire/dialogs/npc-dialog.jsx index 65d3dca7..00420ace 100644 --- a/frontend/eqsage-embed/src/components/spire/dialogs/npc-dialog.jsx +++ b/frontend/eqsage-embed/src/components/spire/dialogs/npc-dialog.jsx @@ -1,4 +1,4 @@ -import React, { useCallback, useEffect, useMemo, useState } from 'react'; +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; import { CommonDialog } from './common'; import Box from '@mui/material/Box'; import Collapse from '@mui/material/Collapse'; @@ -15,7 +15,8 @@ import { v4 } from 'uuid'; import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown'; import KeyboardArrowUpIcon from '@mui/icons-material/KeyboardArrowUp'; -import { Button, FormControl, Stack, TextField } from '@mui/material'; +import { Autocomplete, Button, FormControl, Stack, TextField } from '@mui/material'; +import { useDebouncedCallback } from 'use-debounce'; import { gameController } from '../../../viewer/controllers/GameController'; import BABYLON from '@bjs'; import { useMainContext } from '../../main/context'; @@ -23,14 +24,45 @@ import { useZoneContext } from '../../zone/zone-context'; import { useAlertContext } from '../../../context/alerts'; import { useOverlayContext } from '../provider'; -const { Vector3 } = BABYLON; +const { Tools, Vector3 } = BABYLON; export const NpcDialog = ({ onClose }) => { const [spawnFilter, setSpawnFilter] = useState(''); - const { selectedZone } = useMainContext(); + const [newSpawnNpc, setNewSpawnNpc] = useState(null); + const [newSpawnNpcInput, setNewSpawnNpcInput] = useState(''); + const [newSpawnNpcOptions, setNewSpawnNpcOptions] = useState([]); + const [newSpawnNpcOpen, setNewSpawnNpcOpen] = useState(false); + const [creatingSpawn, setCreatingSpawn] = useState(false); + const npcSearchRunRef = useRef(0); + const { selectedZone, Spire } = useMainContext(); const { toggleDialog } = useOverlayContext(); const { spawns, loadCallback } = useZoneContext(); const { openAlert } = useAlertContext(); const [hidden, setHidden] = useState(false); + const runNewSpawnNpcSearch = useCallback(async (value) => { + const searchRun = ++npcSearchRunRef.current; + const query = `${value ?? ''}`.trim(); + if (!query || !Spire) { + setNewSpawnNpcOptions([]); + setNewSpawnNpcOpen(false); + return; + } + try { + const npcs = (await Spire.Npcs.listNpcsByName(query)) ?? []; + if (searchRun === npcSearchRunRef.current) { + setNewSpawnNpcOptions(npcs); + setNewSpawnNpcOpen(npcs.length > 0); + } + } catch (_error) { + if (searchRun === npcSearchRunRef.current) { + setNewSpawnNpcOptions([]); + setNewSpawnNpcOpen(false); + } + } + }, [Spire]); + const searchNewSpawnNpcs = useDebouncedCallback( + runNewSpawnNpcSearch, + 350 + ); const filteredSpawns = useMemo( () => { const normalizedFilter = spawnFilter?.trim()?.toLowerCase(); @@ -48,40 +80,105 @@ export const NpcDialog = ({ onClose }) => { ); const addSpawn = useCallback(async () => { + if (!newSpawnNpc || creatingSpawn || !Spire) { + return; + } setHidden(true); gameController.ZoneController.pickRaycastForLoc(async (loc) => { setHidden(false); if (!loc) { return; } - const { Spire } = gameController; - + setCreatingSpawn(true); const spawn2Api = new Spire.SpireApiTypes.Spawn2Api( ...Spire.SpireApi.cfg() ); const spawnGroupApi = new Spire.SpireApiTypes.SpawngroupApi( ...Spire.SpireApi.cfg() ); - // First create spawn group - const spawnGroup = await spawnGroupApi.createSpawngroup({ - spawngroup: { name: v4() }, - }); - const createResult = await spawn2Api.createSpawn2({ - spawn2: { - zone : selectedZone.short_name, - x : loc.z, - y : loc.x, - z : loc.y, - spawngroup_id: spawnGroup.data.id, - min_expansion: -1, - max_expansion: -1, - }, - }); - openAlert(`Created new spawn at location [${loc.z} ${loc.x} ${loc.y}]`); - loadCallback({ type: 'create', spawn: createResult.data }); - toggleDialog('npc', false); + const spawnEntryApi = new Spire.SpireApiTypes.SpawnentryApi( + ...Spire.SpireApi.cfg() + ); + let createdSpawnGroupId = null; + let createdSpawnId = null; + try { + const spawnGroup = await spawnGroupApi.createSpawngroup({ + spawngroup: { name: v4() }, + }); + createdSpawnGroupId = spawnGroup.data.id; + const createResult = await spawn2Api.createSpawn2({ + spawn2: { + _condition : 0, + cond_value : 1, + heading : 0, + max_expansion: -1, + min_expansion: -1, + pathgrid : 0, + respawntime : 1200, + spawngroup_id: createdSpawnGroupId, + variance : 0, + version : Number(selectedZone.version ?? 0), + x : loc.z, + y : loc.x, + z : loc.y, + zone : selectedZone.short_name, + }, + }); + createdSpawnId = createResult.data.id; + const spawnEntry = { + chance : 100, + condition_value_filter: 1, + content_flags : null, + content_flags_disabled: null, + max_expansion : -1, + max_time : 0, + min_expansion : -1, + min_time : 0, + npc_id : newSpawnNpc.id, + npc_type : newSpawnNpc, + spawngroup_id : createdSpawnGroupId, + }; + await spawnEntryApi.createSpawnentry({ + id : createdSpawnGroupId, + spawnentry: spawnEntry, + }); + + await loadCallback({ + type : 'create', + spawn: { + ...createResult.data, + spawnentries: [spawnEntry], + }, + }); + openAlert(`Created new spawn at location [${loc.z} ${loc.x} ${loc.y}]`); + setNewSpawnNpc(null); + setNewSpawnNpcInput(''); + setNewSpawnNpcOptions([]); + toggleDialog('npc', false); + } catch (error) { + if (createdSpawnId) { + await spawn2Api.deleteSpawn2({ id: createdSpawnId }).catch(() => {}); + } + if (createdSpawnGroupId) { + await spawnGroupApi + .deleteSpawngroup({ id: createdSpawnGroupId }) + .catch(() => {}); + } + console.warn('Error creating spawn', error); + openAlert('Failed to create spawn. No partial spawn was kept.', 'warning'); + } finally { + setCreatingSpawn(false); + } }); - }, [selectedZone, loadCallback, openAlert, toggleDialog]); + }, [ + Spire, + creatingSpawn, + loadCallback, + newSpawnNpc, + openAlert, + selectedZone, + toggleDialog, + ]); useEffect(() => { const meshes = @@ -96,6 +193,10 @@ export const NpcDialog = ({ onClose }) => { } } }, [filteredSpawns]); + useEffect(() => () => { + npcSearchRunRef.current += 1; + searchNewSpawnNpcs.cancel(); + }, [searchNewSpawnNpcs]); return ( { onClose={onClose} title={'Spawns'} > - + { }} /> + event.preventDefault(), + }} + open={newSpawnNpcOpen} + value={newSpawnNpc} + inputValue={newSpawnNpcInput} + options={newSpawnNpcOptions} + getOptionLabel={(option) => + `${option?.name ?? 'Unknown NPC'} - Level ${option?.level ?? '?'}` + } + isOptionEqualToValue={(option, value) => option?.id === value?.id} + onChange={(_event, value) => { + setNewSpawnNpc(value); + setNewSpawnNpcInput( + value + ? `${value?.name ?? 'Unknown NPC'} - Level ${value?.level ?? '?'}` + : '' + ); + setNewSpawnNpcOpen(false); + }} + onInputChange={(_event, value, reason) => { + if (reason === 'input') { + setNewSpawnNpcInput(value); + setNewSpawnNpcOpen(Boolean(value.trim())); + searchNewSpawnNpcs(value); + } + }} + renderInput={(params) => ( + { + if (newSpawnNpcOptions.length > 0) { + setNewSpawnNpcOpen(true); + } + }} + onBlur={() => { + setTimeout(() => setNewSpawnNpcOpen(false), 150); + }} + onKeyDown={(event) => { + event.stopPropagation(); + if (event.key === 'Escape') { + setNewSpawnNpcOpen(false); + } + }} + /> + )} + size="small" + sx={{ m: 1, width: 320 }} + /> @@ -198,16 +358,17 @@ function Row(props) { + diff --git a/frontend/eqsage-embed/src/components/main/context.jsx b/frontend/eqsage-embed/src/components/main/context.jsx index 315392ee..0e0cfb57 100644 --- a/frontend/eqsage-embed/src/components/main/context.jsx +++ b/frontend/eqsage-embed/src/components/main/context.jsx @@ -7,6 +7,9 @@ import { debugSageLog, markStage } from '../../debug-stage'; const MainContext = React.createContext({}); let gameControllerImportPromise = null; let babylonRuntimePromise = null; +const modelReviewRequested = () => + typeof window !== 'undefined' && + new URLSearchParams(window.location.search).has('sageModelReview'); /** * @typedef Spire @@ -68,15 +71,17 @@ export const MainProvider = ({ ] = usePermissions(); const { remoteUrl } = useSettingsContext(); const [selectedZone, setSelectedZone] = useState(null); + const [modelExporter, setModelExporter] = useState(modelReviewRequested); const [zoneDialogOpen, setZoneDialogOpen] = useState( - () => permissionStatus === PermissionStatusTypes.Ready + () => + permissionStatus === PermissionStatusTypes.Ready && + !modelReviewRequested() ); const [statusDialogOpen, setStatusDialogOpen] = useState( () => permissionStatus !== PermissionStatusTypes.Ready ); const [zoneBuilderDialogOpen, setZoneBuilderDialogOpen] = useState(false); const [audioDialogOpen, setAudioDialogOpen] = useState(false); - const [modelExporter, setModelExporter] = useState(false); const [quailWorkspace, setQuailWorkspace] = useState(false); const [zoneBuilder, setZoneBuilder] = useState(false); const [zones, setZones] = useState([]); @@ -120,12 +125,13 @@ export const MainProvider = ({ } if (!selectedZone) { - const needsZoneDialog = permissionStatus === PermissionStatusTypes.Ready; + const needsZoneDialog = + permissionStatus === PermissionStatusTypes.Ready && !modelExporter; if (zoneDialogOpen !== needsZoneDialog) { setZoneDialogOpen(needsZoneDialog); } } - }, [permissionStatus, selectedZone, statusDialogOpen, zoneDialogOpen]); + }, [modelExporter, permissionStatus, selectedZone, statusDialogOpen, zoneDialogOpen]); useEffect(() => { markStage('main-provider:permission-effect', { @@ -249,7 +255,7 @@ export const MainProvider = ({ permissionStatus !== PermissionStatusTypes.Ready || statusDialogOpen || zoneDialogOpen || - !selectedZone || + (!selectedZone && !modelExporter) || !!gameController || gameControllerLoading || !!gameControllerLoadError @@ -264,6 +270,7 @@ export const MainProvider = ({ gameControllerLoadError, gameControllerLoading, loadGameController, + modelExporter, permissionStatus, selectedZone, statusDialogOpen, @@ -271,15 +278,15 @@ export const MainProvider = ({ ]); useEffect(() => { - if (!selectedZone) { + if (!selectedZone && !modelExporter) { setGameControllerLoadError(null); setControllerLoadStage('Waiting for zone selection'); } - }, [selectedZone]); + }, [modelExporter, selectedZone]); useEffect(() => { if (gameController) { - gameController.modelExporter = true; + gameController.modelExporter = modelExporter; window.gameController = gameController; } }, [gameController, modelExporter]); diff --git a/frontend/eqsage-embed/src/components/main/main.jsx b/frontend/eqsage-embed/src/components/main/main.jsx index f7913f86..f2574bef 100644 --- a/frontend/eqsage-embed/src/components/main/main.jsx +++ b/frontend/eqsage-embed/src/components/main/main.jsx @@ -33,6 +33,11 @@ const RaceFaceAudit = React.lazy(() => const RaceArchiveAudit = React.lazy(() => import('../validation/race-archive-audit').then((module) => ({ default: module.RaceArchiveAudit })) ); +const ModelReviewWorkspace = React.lazy(() => + import('../model-review/model-review-workspace').then((module) => ({ + default: module.ModelReviewWorkspace, + })) +); const ZoneLoadingOverlay = ({ title = 'Preparing Zone Editor', @@ -83,6 +88,7 @@ export const Main = ({ onBootStateChange } = {}) => { debugSageLog('[SageMainRender]'); const { selectedZone, + modelExporter, zoneDialogOpen, statusDialogOpen, rootFileSystemHandle, @@ -190,6 +196,7 @@ export const Main = ({ onBootStateChange } = {}) => { statusDialogOpen, zoneDialogOpen, hasGameController: !!gameController, + modelReview : modelExporter, selectedZone : selectedZone?.short_name ?? null, hasRootHandle: !!rootFileSystemHandle, unsupported, @@ -222,7 +229,7 @@ export const Main = ({ onBootStateChange } = {}) => { return; } - if (selectedZone && !gameController) { + if ((selectedZone || modelExporter) && !gameController) { onBootStateChange?.({ stage : 'boot:controller-loading', detail : controllerLoadStage || 'Loading viewer controller', @@ -231,6 +238,15 @@ export const Main = ({ onBootStateChange } = {}) => { return; } + if (modelExporter && gameController) { + onBootStateChange?.({ + stage : 'boot:model-review-active', + detail : 'Launching model review workspace', + uiVisible: true, + }); + return; + } + if (selectedZone && gameController) { onBootStateChange?.({ stage : 'boot:zone-active', @@ -248,6 +264,7 @@ export const Main = ({ onBootStateChange } = {}) => { }, [ controllerLoadStage, gameController, + modelExporter, onBootStateChange, permissionStatus, rootFileSystemHandle, @@ -347,7 +364,7 @@ export const Main = ({ onBootStateChange } = {}) => { )} {zoneDialogOpen && } - {!statusDialogOpen && !zoneDialogOpen && !selectedZone && ( + {!statusDialogOpen && !zoneDialogOpen && !selectedZone && !modelExporter && ( { error="Startup did not reach the directory or zone chooser dialogs." /> )} - {!statusDialogOpen && !zoneDialogOpen && !!selectedZone && !gameController && ( + {!statusDialogOpen && !zoneDialogOpen && (!!selectedZone || modelExporter) && !gameController && ( { } /> )} - {!statusDialogOpen && !zoneDialogOpen && !!selectedZone && !!gameController && ( + {!statusDialogOpen && !zoneDialogOpen && modelExporter && !!gameController && ( + + } + > + + + )} + {!statusDialogOpen && !zoneDialogOpen && !modelExporter && !!selectedZone && !!gameController && ( <> {!ZoneProviderComponent || !BabylonZoneComponent ? ( { + const variantsByModel = new Map(); + + for (const race of raceData) { + for (const [genderKey, gender] of MODEL_REVIEW_GENDERS) { + const model = `${race[genderKey] ?? ''}`.trim().toLowerCase(); + if (!model) { + continue; + } + const variants = variantsByModel.get(model) ?? []; + variants.push({ + raceId: Number(race.id), + raceName: race.name, + gender, + genderIndex: Math.max( + 0, + MODEL_REVIEW_GENDERS.findIndex(([, name]) => name === gender) + ), + }); + variantsByModel.set(model, variants); + } + } + + return Array.from(variantsByModel.entries()) + .map(([model, variants]) => ({ + model, + variants, + sourceFiles: raceModelMetadata[model]?.sourceFiles ?? [], + appearance: raceModelMetadata[model] ?? {}, + })) + .sort((left, right) => left.model.localeCompare(right.model)); +}; diff --git a/frontend/eqsage-embed/src/components/model-review/model-review-validation.js b/frontend/eqsage-embed/src/components/model-review/model-review-validation.js new file mode 100644 index 00000000..fcc0d399 --- /dev/null +++ b/frontend/eqsage-embed/src/components/model-review/model-review-validation.js @@ -0,0 +1,210 @@ +import { getCharacterHeadOrientationPolicy } from 'sage-core/util/character-texture-orientation'; +import { + evaluateCharacterAnimationReadiness, + inspectAnimationSetVitality, + isStaticPoseOnlyCharacterModel, +} from '../../viewer/helpers/animationValidation'; +import { + evaluateAppearanceVariant, + inspectHeadTextureOrientation, + isKnownEffectOnlyCharacterModel, +} from '../../viewer/helpers/appearanceValidation'; + +const HEAD_MATERIAL_PATTERN = /^[a-z0-9]{3}he(?:\d{2}|sk)\d{2}$/i; + +const getTexture = (material) => + material?.albedoTexture ?? + material?._albedoTexture ?? + material?.diffuseTexture ?? + material?._diffuseTexture ?? + material?.emissiveTexture ?? + material?._emissiveTexture ?? + null; + +const getMaterialSlots = (materials = []) => { + const slots = []; + const seen = new Set(); + for (const material of materials.filter(Boolean)) { + const candidates = Array.isArray(material?.subMaterials) + ? material.subMaterials.filter(Boolean) + : [material]; + for (const candidate of candidates) { + const key = candidate.uniqueId ?? candidate.name ?? candidate; + if (seen.has(key)) { + continue; + } + seen.add(key); + slots.push(candidate); + } + } + return slots; +}; + +export const getReviewMeshes = (rootNode) => [ + rootNode, + ...(rootNode?.getChildMeshes?.(false) ?? []), +].filter( + (mesh) => + typeof mesh?.getTotalVertices === 'function' && + mesh.getTotalVertices() > 0 && + mesh.isEnabled?.() !== false && + mesh.isVisible !== false && + mesh.visibility !== 0 +); + +export const getReviewBounds = (rootNode, { headOnly = false } = {}) => { + const meshes = getReviewMeshes(rootNode).filter((mesh) => { + if (!headOnly) { + return true; + } + return getMaterialSlots([mesh.material]).some((material) => + HEAD_MATERIAL_PATTERN.test(`${material?.name ?? ''}`) + ); + }); + const minimum = { x: Infinity, y: Infinity, z: Infinity }; + const maximum = { x: -Infinity, y: -Infinity, z: -Infinity }; + + for (const mesh of meshes) { + try { + mesh.computeWorldMatrix?.(true); + mesh.refreshBoundingInfo?.(true, true); + const bounds = mesh.getBoundingInfo?.()?.boundingBox; + if (!bounds?.minimumWorld || !bounds?.maximumWorld) { + continue; + } + for (const axis of ['x', 'y', 'z']) { + minimum[axis] = Math.min(minimum[axis], Number(bounds.minimumWorld[axis])); + maximum[axis] = Math.max(maximum[axis], Number(bounds.maximumWorld[axis])); + } + } catch (_error) {} + } + + if (![...Object.values(minimum), ...Object.values(maximum)].every(Number.isFinite)) { + return null; + } + + return { + minimum, + maximum, + width: maximum.x - minimum.x, + height: maximum.y - minimum.y, + depth: maximum.z - minimum.z, + }; +}; + +const isTextureReady = (texture) => { + if (!texture) return false; + if (typeof texture.isReady === 'function') return texture.isReady(); + if (typeof texture.isReady === 'boolean') return texture.isReady; + return texture?._texture?.isReady !== false; +}; + +const getTextureSize = (texture) => { + const size = texture?.getSize?.() ?? {}; + const internal = texture?.getInternalTexture?.() ?? texture?._texture; + return { + width: Number(size.width ?? internal?.width), + height: Number(size.height ?? internal?.height), + }; +}; + +const countNonFiniteBoneMatrices = (spawn) => + (spawn?.instanceContainer?.skeletons ?? []) + .flatMap((skeleton) => skeleton?.bones ?? []) + .filter((bone) => { + const matrix = bone?.getFinalMatrix?.(); + const values = matrix?.toArray?.() ?? matrix?.m ?? []; + return values.length > 0 && values.some((value) => !Number.isFinite(value)); + }).length; + +export const inspectModelReviewSpawn = (spawn) => { + const meshes = getReviewMeshes(spawn?.rootNode); + const materials = getMaterialSlots(meshes.map((mesh) => mesh.material)); + const ordinaryMaterials = materials.filter( + (material) => + material?.metadata?.boundary !== true && + material?.metadata?.gltf?.extras?.boundary !== true && + material?.metadata?.extras?.boundary !== true + ); + const textures = ordinaryMaterials.map(getTexture).filter(Boolean); + const headMaterials = materials.filter((material) => + HEAD_MATERIAL_PATTERN.test(`${material?.name ?? ''}`) + ); + const headOrientation = headMaterials.map((material) => + inspectHeadTextureOrientation( + material, + getCharacterHeadOrientationPolicy(material?.name) + ) + ); + const nonFiniteBoneMatrixCount = countNonFiniteBoneMatrices(spawn); + const appearance = evaluateAppearanceVariant({ + renderPass: meshes.length > 0, + materialSlotCount: materials.length, + effectOnlyMaterialCount: materials.length - ordinaryMaterials.length, + texturedSlotCount: ordinaryMaterials.filter((material) => !!getTexture(material)).length, + pendingTextureCount: ordinaryMaterials.filter((material) => + material?.metadata?.spireAppearanceTexturePending === true || + ( + material?.metadata?.spireAppearanceTexturePending !== false && + !!getTexture(material) && + !isTextureReady(getTexture(material)) + ) + ).length, + suspiciousTinyTextureCount: ordinaryMaterials.filter((material) => { + if ( + material?.metadata?.transparentTextureSentinel !== true || + material?.metadata?.transparentTextureSentinelSuppressed === true + ) { + return false; + } + const { width, height } = getTextureSize(getTexture(material)); + return Number.isFinite(width) && Number.isFinite(height) && width <= 1 && height <= 1; + }).length, + headOrientationRiskCount: headOrientation.filter((item) => item.risk).length, + nonFiniteBoneMatrixCount, + }); + const animationVitality = inspectAnimationSetVitality(spawn?.animationGroups ?? []); + const skeletonCount = spawn?.instanceContainer?.skeletons?.length ?? + (spawn?.rootNode?.skeleton ? 1 : 0); + const staticPoseFallbackAvailable = + spawn?.nativePoseOnly === true || + isStaticPoseOnlyCharacterModel( + spawn?.modelName, + spawn?.loadedModelVariation, + spawn?.resolvedModelAsset + ); + const animationReadiness = evaluateCharacterAnimationReadiness({ + skeletonCount, + animationVitality, + staticPoseFallbackAvailable, + }); + const effectOnly = isKnownEffectOnlyCharacterModel( + spawn?.resolvedModelAsset ?? spawn?.modelName + ); + const orientationPass = headOrientation.every((item) => !item.risk); + const animationPass = + effectOnly || + ( + animationReadiness.pass && + Number(spawn?.animationRetargeting?.unresolvedTargetCount ?? 0) === 0 + ); + + return { + appearance, + animationReadiness, + animationVitality, + animationPass, + bounds: getReviewBounds(spawn?.rootNode), + compactNativeArmNeutralized: + spawn?.compactNativeArmNeutralized === true, + effectOnly, + headOrientation, + materialCount: materials.length, + meshCount: meshes.length, + orientationPass, + nativePoseOnly: spawn?.nativePoseOnly === true, + pass: appearance.invariantPass && orientationPass && animationPass, + skeletonCount, + textureCount: textures.length, + }; +}; diff --git a/frontend/eqsage-embed/src/components/model-review/model-review-workspace.jsx b/frontend/eqsage-embed/src/components/model-review/model-review-workspace.jsx new file mode 100644 index 00000000..9d4c3059 --- /dev/null +++ b/frontend/eqsage-embed/src/components/model-review/model-review-workspace.jsx @@ -0,0 +1,829 @@ +import React, { + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from 'react'; +import { Color3 } from '@babylonjs/core/Maths/math.color'; +import { Color4 } from '@babylonjs/core/Maths/math.color'; +import { Vector3 } from '@babylonjs/core/Maths/math.vector'; +import { HemisphericLight } from '@babylonjs/core/Lights/hemisphericLight'; +import { TransformNode } from '@babylonjs/core/Meshes/transformNode'; + +import { useSettingsContext } from '../../context/settings'; +import { BabylonSpawn } from '../../viewer/models/BabylonSpawn'; +import { useMainContext } from '../main/context'; +import { buildModelReviewInventory } from './model-review-data'; +import { + getReviewBounds, + inspectModelReviewSpawn, +} from './model-review-validation'; + +import './model-review-workspace.scss'; + +const REVIEW_STORAGE_KEY = 'spire-sage-model-review-flags-v1'; +const VIEW_OPTIONS = [ + { id: 'front', label: 'Front', key: '1' }, + { id: 'side', label: 'Side', key: '2' }, + { id: 'back', label: 'Rear', key: '3' }, + { id: 'head', label: 'Head', key: '4' }, +]; + +const getInitialParams = () => { + const params = new URLSearchParams(window.location.search); + const number = (name, fallback = 0) => { + const parsed = Number(params.get(name)); + return Number.isFinite(parsed) ? Math.max(0, Math.trunc(parsed)) : fallback; + }; + return { + model: `${params.get('sageModel') ?? ''}`.trim().toLowerCase(), + face: number('sageModelFace'), + texture: number('sageModelTexture'), + helmTexture: number('sageModelHelm'), + view: VIEW_OPTIONS.some((option) => option.id === params.get('sageModelView')) + ? params.get('sageModelView') + : 'front', + }; +}; + +const readStoredReviews = () => { + try { + const value = JSON.parse(localStorage.getItem(REVIEW_STORAGE_KEY) ?? '{}'); + return value && typeof value === 'object' ? value : {}; + } catch (_error) { + return {}; + } +}; + +const labelAnimation = (name) => `${name ?? ''}`.replace(/^Clone of /, '') || 'Pose'; + +const getAnimationFrame = (animationGroup) => + Number(animationGroup?.animatables?.[0]?.masterFrame ?? animationGroup?._animatables?.[0]?.masterFrame ?? animationGroup?.from ?? 0); + +const isTypingTarget = (target) => + target instanceof HTMLInputElement || + target instanceof HTMLTextAreaElement || + target instanceof HTMLSelectElement || + target?.isContentEditable === true; + +const StatusBadge = ({ label, pass, neutral = false }) => ( + + +); + +export const ModelReviewWorkspace = () => { + const { + gameController, + loadGameController, + setModelExporter, + setZoneDialogOpen, + } = useMainContext(); + const settings = useSettingsContext(); + const initial = useMemo(getInitialParams, []); + const inventory = useMemo(buildModelReviewInventory, []); + const initialIndex = Math.max( + 0, + inventory.findIndex((entry) => entry.model === initial.model) + ); + const canvasRef = useRef(null); + const listRef = useRef(null); + const runtimeRef = useRef(null); + const previewRef = useRef(null); + const loadTokenRef = useRef(0); + + const [runtimeReady, setRuntimeReady] = useState(false); + const [runtimeStage, setRuntimeStage] = useState('Preparing model renderer'); + const [runtimeError, setRuntimeError] = useState(''); + const [selectedModel, setSelectedModel] = useState( + inventory[initialIndex]?.model ?? inventory[0]?.model ?? '' + ); + const [query, setQuery] = useState(''); + const [filter, setFilter] = useState('all'); + const [face, setFace] = useState(initial.face); + const [texture, setTexture] = useState(initial.texture); + const [helmTexture, setHelmTexture] = useState(initial.helmTexture); + const [view, setView] = useState(initial.view); + const [diagnostics, setDiagnostics] = useState(null); + const [diagnosticsByModel, setDiagnosticsByModel] = useState({}); + const [modelStage, setModelStage] = useState('Waiting for renderer'); + const [modelError, setModelError] = useState(''); + const [animationName, setAnimationName] = useState(''); + const [animationPlaying, setAnimationPlaying] = useState(false); + const [animationFrame, setAnimationFrame] = useState(0); + const [animationSafety, setAnimationSafety] = useState(null); + const [reviews, setReviews] = useState(readStoredReviews); + const [note, setNote] = useState( + readStoredReviews()[inventory[initialIndex]?.model]?.note ?? '' + ); + + const selectedEntry = useMemo( + () => inventory.find((entry) => entry.model === selectedModel) ?? inventory[0], + [inventory, selectedModel] + ); + const selectedVariant = selectedEntry?.variants?.[0] ?? null; + const selectedReview = reviews[selectedModel] ?? null; + const maxTexture = Math.max( + Number(selectedEntry?.appearance?.minTexture ?? 0), + Number(selectedEntry?.appearance?.maxTexture ?? 0) + ); + const maxHelmTexture = Math.max( + Number(selectedEntry?.appearance?.minHelmTexture ?? 0), + Number(selectedEntry?.appearance?.maxHelmTexture ?? 0) + ); + + const filteredInventory = useMemo(() => { + const normalizedQuery = query.trim().toLowerCase(); + return inventory.filter((entry) => { + const matchesQuery = !normalizedQuery || + entry.model.includes(normalizedQuery) || + entry.variants.some((variant) => + `${variant.raceName ?? ''}`.toLowerCase().includes(normalizedQuery) + ); + if (!matchesQuery) return false; + if (filter === 'issues') { + return reviews[entry.model]?.status === 'issue' || + diagnosticsByModel[entry.model]?.pass === false; + } + if (filter === 'unreviewed') { + return !reviews[entry.model]?.status; + } + return true; + }); + }, [diagnosticsByModel, filter, inventory, query, reviews]); + + useEffect(() => { + const index = filteredInventory.findIndex( + (entry) => entry.model === selectedModel + ); + if (index < 0 || !listRef.current) return undefined; + const timer = window.setTimeout(() => { + const list = listRef.current; + if (!list) return; + const rowHeight = + list.firstElementChild?.getBoundingClientRect?.().height || 42; + list.scrollTop = Math.max( + 0, + index * rowHeight - (list.clientHeight - rowHeight) / 2 + ); + }, 100); + return () => window.clearTimeout(timer); + }, [filteredInventory, selectedModel]); + + const frameModel = useCallback((nextView = 'front') => { + const preview = previewRef.current?.preview; + const camera = runtimeRef.current?.camera; + if (!preview?.rootNode || !camera) { + return; + } + const wholeBounds = getReviewBounds(preview.rootNode); + if (!wholeBounds) { + return; + } + const headBounds = nextView === 'head' + ? getReviewBounds(preview.rootNode, { headOnly: true }) + : null; + const bounds = headBounds ?? wholeBounds; + const target = new Vector3( + (bounds.minimum.x + bounds.maximum.x) / 2, + (bounds.minimum.y + bounds.maximum.y) / 2, + (bounds.minimum.z + bounds.maximum.z) / 2 + ); + if (nextView === 'head' && !headBounds) { + target.y = wholeBounds.minimum.y + wholeBounds.height * 0.8; + } + const span = Math.max(1, bounds.width, bounds.height, bounds.depth); + const distance = nextView === 'head' + ? Math.max(5, span * 2.1) + : Math.max(7, wholeBounds.height * 1.65, wholeBounds.depth * 1.8); + const elevation = nextView === 'head' ? 0 : wholeBounds.height * 0.05; + const positions = { + front: new Vector3(target.x - distance, target.y + elevation, target.z), + side : new Vector3(target.x, target.y + elevation, target.z - distance), + back : new Vector3(target.x + distance, target.y + elevation, target.z), + head : new Vector3(target.x - distance, target.y, target.z), + }; + camera.setPosition?.(positions[nextView] ?? positions.front); + if (!camera.setPosition) { + camera.position.copyFrom(positions[nextView] ?? positions.front); + } + camera.setTarget(target); + runtimeRef.current.scene.render(); + }, []); + + const getSelectedAnimation = useCallback(() => + previewRef.current?.preview?.animationGroups?.find( + (group) => group.name === animationName + ) ?? null, + [animationName]); + + const startAnimation = useCallback((name, { play = true } = {}) => { + const preview = previewRef.current?.preview; + if (!preview) return; + const group = preview.animationGroups.find((candidate) => candidate.name === name); + preview.animationGroups.forEach((candidate) => candidate.stop?.()); + if (!group) { + setAnimationPlaying(false); + setAnimationSafety(null); + return; + } + const isPose = preview.isPoseAnimation?.(group) === true; + const safety = isPose ? { pass: true, pose: true } : preview.validateAnimationBounds?.(group); + setAnimationSafety(safety ?? { pass: true }); + if (safety?.pass === false) { + setAnimationPlaying(false); + return; + } + if (isPose && preview.nativePoseOnly) { + preview.applyNeutralSkeletonPose?.(); + } else { + group.play?.(!isPose); + } + if ((!play || isPose) && !(isPose && preview.nativePoseOnly)) { + group.pause?.(); + group.goToFrame?.(Number(group.from ?? 0)); + } + preview.synchronizeSkeletonPose?.(); + preview.normalizeAnimatedGroundPose?.(); + setAnimationFrame(getAnimationFrame(group)); + setAnimationPlaying(play && !isPose); + runtimeRef.current?.scene?.render?.(); + }, []); + + const selectModel = useCallback((model) => { + const entry = inventory.find((candidate) => candidate.model === model); + setSelectedModel(model); + setFace(0); + setTexture(Math.max(0, Number(entry?.appearance?.minTexture ?? 0))); + setHelmTexture(Math.max(0, Number(entry?.appearance?.minHelmTexture ?? 0))); + }, [inventory]); + + const chooseRelativeModel = useCallback((offset) => { + if (!filteredInventory.length) return; + const index = filteredInventory.findIndex((entry) => entry.model === selectedModel); + const nextIndex = (Math.max(0, index) + offset + filteredInventory.length) % + filteredInventory.length; + selectModel(filteredInventory[nextIndex].model); + }, [filteredInventory, selectModel, selectedModel]); + + const closeWorkspace = useCallback(() => { + const url = new URL(window.location.href); + for (const key of [ + 'sageModelReview', + 'sageModel', + 'sageModelFace', + 'sageModelTexture', + 'sageModelHelm', + 'sageModelView', + ]) { + url.searchParams.delete(key); + } + window.history.replaceState(null, '', url.toString()); + window.__spireSageModelReview = null; + setModelExporter(false); + setZoneDialogOpen(true); + }, [setModelExporter, setZoneDialogOpen]); + + useEffect(() => { + let current = true; + let modelController = null; + let previousGlowLayer = null; + let resizeHandler = null; + const settingsSnapshot = { ...settings }; + + (async () => { + try { + setRuntimeStage('Loading Sage runtime'); + const controller = gameController ?? await loadGameController(); + const [{ default: bjs }, modelControllerModule] = await Promise.all([ + import('@bjs'), + import('../../viewer/controllers/ModelController'), + ]); + await bjs.prepareZoneViewer((stage) => current && setRuntimeStage(stage)); + if (!current || !canvasRef.current) return; + + modelController = modelControllerModule.modelController; + controller.ModelController = modelController; + modelController.setGameController(controller); + controller.settings = settingsSnapshot; + setRuntimeStage('Starting neutral model scene'); + await controller.loadEngine(canvasRef.current, settingsSnapshot.webgpu); + await modelController.initializeModelExporter(); + if (!current) return; + + modelController.swapBackground('none'); + modelController.glowLayer?.setEnabled?.(false); + controller.CameraController.rotate(false); + const scene = modelController.scene; + scene.environmentTexture?.dispose?.(); + scene.environmentTexture = null; + scene.environmentIntensity = 0; + scene.clearColor = new Color4(0.025, 0.035, 0.045, 1); + const fillLight = new HemisphericLight( + 'model-review-fill', + new Vector3(0, 1, 0), + scene + ); + fillLight.diffuse = new Color3(1, 1, 1); + fillLight.groundColor = new Color3(0.25, 0.23, 0.2); + // Match the embedded zone viewer so appearance evidence is not + // skewed by the retired exporter's HDR environment or glow layer. + fillLight.intensity = 0.85; + previousGlowLayer = controller.ZoneController.glowLayer; + controller.ZoneController.glowLayer = modelController.glowLayer; + controller.SpawnController.setupSpawnController(); + resizeHandler = () => controller.resize(); + window.addEventListener('resize', resizeHandler); + runtimeRef.current = { + camera: controller.CameraController.camera, + controller, + fillLight, + modelController, + scene, + }; + setRuntimeStage('Model renderer ready'); + setRuntimeReady(true); + } catch (error) { + if (!current) return; + console.error('[SageModelReview] failed to initialize', error); + setRuntimeError(error?.message ?? String(error)); + setRuntimeStage('Model renderer failed'); + } + })(); + + return () => { + current = false; + loadTokenRef.current++; + previewRef.current?.preview?.dispose?.(); + previewRef.current?.root?.dispose?.(); + previewRef.current = null; + if (resizeHandler) window.removeEventListener('resize', resizeHandler); + const controller = runtimeRef.current?.controller ?? gameController; + if (controller?.ZoneController) { + controller.ZoneController.glowLayer = previousGlowLayer; + } + runtimeRef.current?.fillLight?.dispose?.(); + modelController?.dispose?.(); + controller?.stopRenderLoop?.(); + runtimeRef.current = null; + }; + // Runtime ownership intentionally lasts for the workspace lifetime. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + + useEffect(() => { + if (!runtimeReady || !selectedEntry || !runtimeRef.current) return; + const loadToken = ++loadTokenRef.current; + let createdPreview = null; + let createdRoot = null; + + (async () => { + setModelStage(`Loading ${selectedEntry.model.toUpperCase()}`); + setModelError(''); + setDiagnostics(null); + setAnimationName(''); + setAnimationPlaying(false); + previewRef.current?.preview?.dispose?.(); + previewRef.current?.root?.dispose?.(); + previewRef.current = null; + + const { controller, scene } = runtimeRef.current; + createdRoot = new TransformNode( + `model-review-${selectedEntry.model}-root`, + scene + ); + createdPreview = new BabylonSpawn( + { + id: -910000, + __spireSpawnId: -910000, + name: `${selectedVariant?.raceName ?? selectedEntry.model} Model Review`, + race: selectedVariant?.raceId ?? 0, + gender: selectedVariant?.genderIndex ?? 0, + face, + texture, + helmtexture: helmTexture, + size: 6, + x: 0, + y: 0, + z: 0, + heading: 0, + grid: [], + }, + selectedEntry.model, + createdRoot, + controller.SpawnController.sphereMat + ); + + const previousBulkLoading = window.__spireSageBulkSpawnLoading; + const previousSkipNameplates = window.__spireSageSkipBulkNameplates; + window.__spireSageBulkSpawnLoading = true; + window.__spireSageSkipBulkNameplates = true; + try { + const initialized = await createdPreview.initializeSpawn(); + if (!initialized) { + throw new Error('No renderable model asset was found'); + } + } finally { + window.__spireSageBulkSpawnLoading = previousBulkLoading; + window.__spireSageSkipBulkNameplates = previousSkipNameplates; + } + + if (loadToken !== loadTokenRef.current) { + createdPreview.dispose(); + createdRoot.dispose(); + return; + } + + createdPreview.disposeNameplate?.(); + previewRef.current = { preview: createdPreview, root: createdRoot }; + if (createdPreview.nativePoseOnly) { + createdPreview.applyNeutralSkeletonPose?.(); + } + let nextDiagnostics = inspectModelReviewSpawn(createdPreview); + for ( + let attempt = 0; + attempt < 20 && nextDiagnostics.appearance.pendingTextureCount > 0; + attempt++ + ) { + scene.render(); + await new Promise((resolve) => window.setTimeout(resolve, 75)); + if (loadToken !== loadTokenRef.current) { + createdPreview.dispose(); + createdRoot.dispose(); + return; + } + nextDiagnostics = inspectModelReviewSpawn(createdPreview); + } + setDiagnostics(nextDiagnostics); + setDiagnosticsByModel((current) => ({ + ...current, + [selectedEntry.model]: nextDiagnostics, + })); + const preferred = createdPreview.getPreferredVisualAnimationGroup?.(); + const pose = createdPreview.animationGroups.find((group) => + createdPreview.isPoseAnimation?.(group) + ); + const nextAnimation = preferred ?? pose ?? createdPreview.animationGroups[0] ?? null; + setAnimationName(nextAnimation?.name ?? ''); + setModelStage( + `${selectedEntry.model.toUpperCase()} ready · ${createdPreview.resolvedModelAsset ?? createdPreview.loadedModelVariation ?? selectedEntry.model}` + ); + requestAnimationFrame(() => frameModel(view)); + })().catch((error) => { + if (loadToken !== loadTokenRef.current) return; + console.error('[SageModelReview] failed to load model', error); + createdPreview?.dispose?.(); + createdRoot?.dispose?.(); + setModelError(error?.message ?? String(error)); + setModelStage(`${selectedEntry.model.toUpperCase()} failed`); + setDiagnosticsByModel((current) => ({ + ...current, + [selectedEntry.model]: { pass: false, error: error?.message ?? String(error) }, + })); + }); + + return () => { + if (previewRef.current?.preview === createdPreview) return; + createdPreview?.dispose?.(); + createdRoot?.dispose?.(); + }; + }, [face, frameModel, helmTexture, runtimeReady, selectedEntry, selectedVariant, texture]); + + useEffect(() => { + if (!animationName || !previewRef.current?.preview) return; + startAnimation(animationName); + }, [animationName, startAnimation]); + + useEffect(() => { + if (!animationName || !previewRef.current?.preview) return undefined; + const frameTimer = window.setTimeout(() => frameModel(view), 250); + return () => window.clearTimeout(frameTimer); + }, [animationName, frameModel, view]); + + useEffect(() => { + frameModel(view); + }, [frameModel, view]); + + useEffect(() => { + setNote(reviews[selectedModel]?.note ?? ''); + }, [reviews, selectedModel]); + + useEffect(() => { + const url = new URL(window.location.href); + url.searchParams.set('sageModelReview', '1'); + url.searchParams.set('sageModel', selectedModel); + url.searchParams.set('sageModelFace', String(face)); + url.searchParams.set('sageModelTexture', String(texture)); + url.searchParams.set('sageModelHelm', String(helmTexture)); + url.searchParams.set('sageModelView', view); + window.history.replaceState(null, '', url.toString()); + }, [face, helmTexture, selectedModel, texture, view]); + + useEffect(() => { + window.__spireSageModelReview = { + ready: runtimeReady && !!diagnostics && !modelError, + model: selectedModel, + diagnostics, + stage: modelStage, + view, + }; + }, [diagnostics, modelError, modelStage, runtimeReady, selectedModel, view]); + + useEffect(() => { + if (!animationPlaying) return undefined; + const timer = window.setInterval(() => { + setAnimationFrame(getAnimationFrame(getSelectedAnimation())); + }, 100); + return () => window.clearInterval(timer); + }, [animationPlaying, getSelectedAnimation]); + + useEffect(() => { + const onKeyDown = (event) => { + if (isTypingTarget(event.target)) return; + if (event.key === 'ArrowLeft') { + event.preventDefault(); + chooseRelativeModel(-1); + } else if (event.key === 'ArrowRight') { + event.preventDefault(); + chooseRelativeModel(1); + } else if (event.key === ' ') { + event.preventDefault(); + const group = getSelectedAnimation(); + if (!group || animationSafety?.pass === false) return; + if (animationPlaying) group.pause?.(); + else group.play?.(true); + setAnimationPlaying(!animationPlaying); + } else { + const cameraView = VIEW_OPTIONS.find((option) => option.key === event.key); + if (cameraView) setView(cameraView.id); + } + }; + window.addEventListener('keydown', onKeyDown); + return () => window.removeEventListener('keydown', onKeyDown); + }, [animationPlaying, animationSafety?.pass, chooseRelativeModel, getSelectedAnimation]); + + const saveReview = (status) => { + const next = { + ...reviews, + [selectedModel]: { + status, + note: note.trim(), + updatedAt: new Date().toISOString(), + face, + texture, + helmTexture, + }, + }; + setReviews(next); + localStorage.setItem(REVIEW_STORAGE_KEY, JSON.stringify(next)); + }; + + const copyReviewLink = async () => { + await navigator.clipboard?.writeText?.(window.location.href); + }; + + const animations = previewRef.current?.preview?.animationGroups ?? []; + const selectedAnimation = animations.find((group) => group.name === animationName); + const animationFrom = Number(selectedAnimation?.from ?? 0); + const animationTo = Math.max(animationFrom, Number(selectedAnimation?.to ?? animationFrom)); + const selectedIndex = filteredInventory.findIndex((entry) => entry.model === selectedModel); + + return ( +
+ + + + +
+
+ +
+ {selectedEntry?.model?.toUpperCase()} + + {selectedVariant?.raceName ?? 'Unknown race'} · {Math.max(0, selectedIndex) + 1} / {filteredInventory.length} + +
+ +
+
+ {VIEW_OPTIONS.map((option) => ( + + ))} +
+
+ + {runtimeError || modelError || (runtimeReady ? modelStage : runtimeStage)} +
+
+ +