From c6df91c0b462bb20d948788453526f89aae28ce2 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Wed, 24 Jun 2026 17:44:19 +0200 Subject: [PATCH 01/35] Fix validation worker loading issue --- packages/ui/spa/ValSyncEngine.ts | 12 ++- packages/ui/spa/components/ValProvider.tsx | 15 ++-- .../spa/validation/ValidationWorkerClient.ts | 67 ++++++++-------- .../spa/validation/createValidationWorker.ts | 14 +++- .../ui/spa/validation/validateModule.test.ts | 79 +++++++++++++++++++ packages/ui/spa/validation/validateModule.ts | 40 ++++++++++ .../ui/spa/validation/validation.worker.ts | 29 ++----- 7 files changed, 189 insertions(+), 67 deletions(-) create mode 100644 packages/ui/spa/validation/validateModule.test.ts create mode 100644 packages/ui/spa/validation/validateModule.ts diff --git a/packages/ui/spa/ValSyncEngine.ts b/packages/ui/spa/ValSyncEngine.ts index f2d6d9229..d4923ed13 100644 --- a/packages/ui/spa/ValSyncEngine.ts +++ b/packages/ui/spa/ValSyncEngine.ts @@ -33,7 +33,10 @@ import { import { canMerge } from "./utils/mergePatches"; import { PatchSets, SerializedPatchSet } from "./utils/PatchSets"; import { ReifiedRender } from "@valbuild/core"; -import { ValidationWorkerClient } from "./validation/ValidationWorkerClient"; +import { + ValidationWorkerClient, + ValidationWorkerFactory, +} from "./validation/ValidationWorkerClient"; import { partitionValidationErrors } from "./validation/partitionValidationErrors"; /** @@ -238,6 +241,12 @@ export class ValSyncEngine { private readonly overlayEmitter: | typeof defaultOverlayEmitter | undefined = undefined, + // Injected by the composition root (ValProvider). Kept out of this file so + // the worker's import.meta reference never reaches the Jest-compiled core. + // When undefined (tests / SSR / stories) validation runs on the main thread. + private readonly createValidationWorker: + | ValidationWorkerFactory + | undefined = undefined, ) { this.initializedAt = null; this.forceSyncAllModules = true; @@ -1436,6 +1445,7 @@ export class ValSyncEngine { (moduleFilePath, errors) => { this.applyValidationResult(moduleFilePath, errors); }, + this.createValidationWorker, ); } return this.validationWorker; diff --git a/packages/ui/spa/components/ValProvider.tsx b/packages/ui/spa/components/ValProvider.tsx index ab052ce5b..4f67ac1c2 100644 --- a/packages/ui/spa/components/ValProvider.tsx +++ b/packages/ui/spa/components/ValProvider.tsx @@ -34,6 +34,7 @@ import { isJsonArray } from "../utils/isJsonArray"; import { AuthenticationState, useStatus } from "../hooks/useStatus"; import { findRequiredRemoteFiles } from "../utils/findRequiredRemoteFiles"; import { defaultOverlayEmitter, ValSyncEngine } from "../ValSyncEngine"; +import { createValidationWorker } from "../validation/createValidationWorker"; import { SerializedPatchSet } from "../utils/PatchSets"; import { z } from "zod"; import { @@ -314,11 +315,15 @@ export function ValProvider({ const syncEngine = useMemo( () => - new ValSyncEngine(client, (moduleFilePath, newSource) => { - if (dispatchValEvents) { - defaultOverlayEmitter(moduleFilePath, newSource); - } - }), + new ValSyncEngine( + client, + (moduleFilePath, newSource) => { + if (dispatchValEvents) { + defaultOverlayEmitter(moduleFilePath, newSource); + } + }, + createValidationWorker, + ), // TODO: add client to dependency array NOTE: we need to make sure syncing works if when syncEngine is instantiated anew [dispatchValEvents], ); diff --git a/packages/ui/spa/validation/ValidationWorkerClient.ts b/packages/ui/spa/validation/ValidationWorkerClient.ts index 129eb5906..b110910ec 100644 --- a/packages/ui/spa/validation/ValidationWorkerClient.ts +++ b/packages/ui/spa/validation/ValidationWorkerClient.ts @@ -1,16 +1,14 @@ import { - deserializeSchema, ModuleFilePath, - SelectorSource, SerializedSchema, Source, - SourcePath, ValidationErrors, } from "@valbuild/core"; import type { ValidationWorkerRequest, ValidationWorkerResponse, } from "./worker-types"; +import { SchemaValidator } from "./validateModule"; const supportsWorker = typeof window !== "undefined" && typeof Worker !== "undefined"; @@ -20,40 +18,40 @@ export type ValidationResultCallback = ( errors: ValidationErrors, ) => void; +// The factory is injected by the composition root (ValProvider) so the +// `new URL(..., import.meta.url)` worker reference — and thus import.meta — stays +// out of this file and ValSyncEngine, which are compiled by Jest. See +// createValidationWorker.ts. +export type ValidationWorkerFactory = () => Worker; + export class ValidationWorkerClient { private worker: Worker | null = null; - // Worker setup is async (dynamic import) so we can keep the import.meta.url - // reference out of files parsed by Jest. Requests posted before the worker - // resolves are buffered here. - private pending: ValidationWorkerRequest[] | null = supportsWorker - ? [] - : null; + // Requests posted before the worker is installed are buffered here. Null when + // no worker will be installed (no factory, or unsupported env) — validation + // then runs on the main thread via the fallback validator. + private pending: ValidationWorkerRequest[] | null = null; private requestIdCounter = 0; private disposed = false; private latestRequestId = new Map(); - // Fallback cache used when the worker is unavailable (jsdom / SSR). - private fallbackSchemaCache = new Map< - ModuleFilePath, - { - schemaSha: string; - schema: ReturnType; - } - >(); + // Fallback validator used when the worker is unavailable (jsdom / SSR / node). + private fallbackValidator = new SchemaValidator(); - constructor(private onResult: ValidationResultCallback) { - if (supportsWorker) { - void this.setupWorker(); + constructor( + private onResult: ValidationResultCallback, + private createWorker?: ValidationWorkerFactory, + ) { + if (supportsWorker && this.createWorker) { + this.pending = []; + this.setupWorker(this.createWorker); } } - private async setupWorker(): Promise { + private setupWorker(createWorker: ValidationWorkerFactory): void { try { - const { createValidationWorker } = - await import("./createValidationWorker"); - const worker = createValidationWorker(); - // dispose() may have been called while the dynamic import was in flight — - // don't install a worker the client no longer owns (it would leak the - // thread and keep handlers alive). + const worker = createWorker(); + // dispose() may have been called before setup completed — don't install a + // worker the client no longer owns (it would leak the thread and keep + // handlers alive). if (this.disposed) { worker.terminate(); return; @@ -130,14 +128,11 @@ export class ValidationWorkerClient { } // Main-thread fallback (tests, SSR, or worker construction failed). try { - let cached = this.fallbackSchemaCache.get(moduleFilePath); - if (!cached || cached.schemaSha !== schemaSha) { - cached = { schemaSha, schema: deserializeSchema(serializedSchema) }; - this.fallbackSchemaCache.set(moduleFilePath, cached); - } - const errors = cached.schema["executeValidate"]( - moduleFilePath as string as SourcePath, - source as SelectorSource, + const errors = this.fallbackValidator.validate( + moduleFilePath, + source, + serializedSchema, + schemaSha, ); this.onResult(moduleFilePath, errors); } catch (error) { @@ -158,6 +153,6 @@ export class ValidationWorkerClient { } this.pending = null; this.latestRequestId.clear(); - this.fallbackSchemaCache.clear(); + this.fallbackValidator = new SchemaValidator(); } } diff --git a/packages/ui/spa/validation/createValidationWorker.ts b/packages/ui/spa/validation/createValidationWorker.ts index c491636ef..a5bed576c 100644 --- a/packages/ui/spa/validation/createValidationWorker.ts +++ b/packages/ui/spa/validation/createValidationWorker.ts @@ -1,5 +1,13 @@ -export function createValidationWorker(): Worker { - return new Worker(new URL("./validation.worker.ts", import.meta.url), { +import type { ValidationWorkerFactory } from "./ValidationWorkerClient"; + +// Constructed the same way as the search and patchsets workers: Vite rewrites +// new URL(..., import.meta.url) to an absolute, base-prefixed asset URL, so the +// worker resolves regardless of where the SPA entry is served. +// +// This is the only file that references import.meta. It is imported solely by +// the app composition root (ValProvider) and passed into ValSyncEngine, keeping +// import.meta out of the test-compiled core (ValSyncEngine / ValidationWorkerClient). +export const createValidationWorker: ValidationWorkerFactory = () => + new Worker(new URL("./validation.worker.ts", import.meta.url), { type: "module", }); -} diff --git a/packages/ui/spa/validation/validateModule.test.ts b/packages/ui/spa/validation/validateModule.test.ts new file mode 100644 index 000000000..744edb2f1 --- /dev/null +++ b/packages/ui/spa/validation/validateModule.test.ts @@ -0,0 +1,79 @@ +import { initVal, Internal, ModuleFilePath } from "@valbuild/core"; +import { SchemaValidator } from "./validateModule"; + +const { s, c } = initVal(); + +function serialize(module: ReturnType) { + const path = Internal.getValPath(module) as unknown as ModuleFilePath; + const schema = Internal.getSchema(module)!; + const serializedSchema = schema["executeSerialize"](); + const source = Internal.getSource(module); + return { path, serializedSchema, source }; +} + +describe("SchemaValidator", () => { + test("returns no errors for a valid source", () => { + const validator = new SchemaValidator(); + const { path, serializedSchema, source } = serialize( + c.define("/test.val.ts", s.string().minLength(2), "valid"), + ); + const errors = validator.validate(path, source, serializedSchema, "sha1"); + expect(errors).toBe(false); + }); + + test("returns validation errors for an invalid source", () => { + const validator = new SchemaValidator(); + const { path, serializedSchema, source } = serialize( + c.define("/test.val.ts", s.string().minLength(2), "a"), + ); + const errors = validator.validate(path, source, serializedSchema, "sha1"); + expect(errors).not.toBe(false); + expect( + Object.keys(errors as Record).length, + ).toBeGreaterThan(0); + }); + + test("reuses the cached schema while the sha is unchanged", () => { + const validator = new SchemaValidator(); + // Lenient schema: "valid" (5 chars) passes minLength(2). + const lenient = serialize( + c.define("/test.val.ts", s.string().minLength(2), "valid"), + ); + // Strict schema for the same path/source: "valid" fails minLength(10). + const strict = serialize( + c.define("/test.val.ts", s.string().minLength(10), "valid"), + ); + + // First call deserializes and caches the lenient schema under "sha1". + expect( + validator.validate( + lenient.path, + lenient.source, + lenient.serializedSchema, + "sha1", + ), + ).toBe(false); + + // Same sha → cached lenient schema is reused even though a different + // serialized schema is passed, so the source is still considered valid. + expect( + validator.validate( + lenient.path, + lenient.source, + strict.serializedSchema, + "sha1", + ), + ).toBe(false); + + // New sha → schema is re-derived from the strict definition and the source + // now fails validation. + expect( + validator.validate( + lenient.path, + lenient.source, + strict.serializedSchema, + "sha2", + ), + ).not.toBe(false); + }); +}); diff --git a/packages/ui/spa/validation/validateModule.ts b/packages/ui/spa/validation/validateModule.ts new file mode 100644 index 000000000..66ee5bfd3 --- /dev/null +++ b/packages/ui/spa/validation/validateModule.ts @@ -0,0 +1,40 @@ +import { + deserializeSchema, + ModuleFilePath, + Schema, + SelectorSource, + SerializedSchema, + Source, + SourcePath, + ValidationErrors, +} from "@valbuild/core"; + +/** + * Framework-free validation logic shared by the validation Web Worker and the + * main-thread fallback in ValidationWorkerClient. Keeps a per-instance cache of + * deserialized schemas keyed by module path, re-deriving only when the schema + * sha changes. + */ +export class SchemaValidator { + private cache = new Map< + ModuleFilePath, + { schemaSha: string; schema: Schema } + >(); + + validate( + moduleFilePath: ModuleFilePath, + source: Source, + serializedSchema: SerializedSchema, + schemaSha: string, + ): ValidationErrors { + let cached = this.cache.get(moduleFilePath); + if (!cached || cached.schemaSha !== schemaSha) { + cached = { schemaSha, schema: deserializeSchema(serializedSchema) }; + this.cache.set(moduleFilePath, cached); + } + return cached.schema["executeValidate"]( + moduleFilePath as string as SourcePath, + source as SelectorSource, + ); + } +} diff --git a/packages/ui/spa/validation/validation.worker.ts b/packages/ui/spa/validation/validation.worker.ts index 13949c859..11fab956f 100644 --- a/packages/ui/spa/validation/validation.worker.ts +++ b/packages/ui/spa/validation/validation.worker.ts @@ -1,39 +1,24 @@ /// -import { - deserializeSchema, - ModuleFilePath, - Schema, - SelectorSource, - SourcePath, -} from "@valbuild/core"; import type { ValidationWorkerRequest, ValidationWorkerResponse, } from "./worker-types"; +import { SchemaValidator } from "./validateModule"; const ctx: DedicatedWorkerGlobalScope = self as unknown as DedicatedWorkerGlobalScope; -const schemaCache = new Map< - ModuleFilePath, - { schemaSha: string; schema: Schema } ->(); +const validator = new SchemaValidator(); ctx.onmessage = (event: MessageEvent) => { const request = event.data; if (request.type !== "validate") return; try { - let cached = schemaCache.get(request.moduleFilePath); - if (!cached || cached.schemaSha !== request.schemaSha) { - cached = { - schemaSha: request.schemaSha, - schema: deserializeSchema(request.serializedSchema), - }; - schemaCache.set(request.moduleFilePath, cached); - } - const errors = cached.schema["executeValidate"]( - request.moduleFilePath as string as SourcePath, - request.source as SelectorSource, + const errors = validator.validate( + request.moduleFilePath, + request.source, + request.serializedSchema, + request.schemaSha, ); const response: ValidationWorkerResponse = { type: "result", From 88dd4ded97eb2cf6ba051ed4599653ca29f92401 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Wed, 24 Jun 2026 17:48:45 +0200 Subject: [PATCH 02/35] Changeset --- .changeset/witty-mangos-fix.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/witty-mangos-fix.md diff --git a/.changeset/witty-mangos-fix.md b/.changeset/witty-mangos-fix.md new file mode 100644 index 000000000..a2f8dce8f --- /dev/null +++ b/.changeset/witty-mangos-fix.md @@ -0,0 +1,5 @@ +--- +"@valbuild/ui": patch +--- + +Fix client-side validation worker failing to load ("Unexpected token '<'"). The worker was reached via a dynamic import, which Vite emitted as a code-split chunk resolved relative to the entry's served URL (`/api/val/static/{version}/app`) rather than its real `/assets/` location, so the request returned the SPA HTML shell instead of JavaScript. The worker is now created with `new URL("./validation.worker.ts", import.meta.url)` like the search and patchsets workers (an absolute asset URL), and the worker factory is injected from the composition root so it loads reliably regardless of where the SPA entry is served. From 33f775cb4c944e0f722bdbf07a533c77e53f37b8 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Wed, 24 Jun 2026 20:30:49 +0200 Subject: [PATCH 03/35] Revert the maxLength in the example causing validation issues --- examples/next/app/blogs/[blog]/page.val.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/next/app/blogs/[blog]/page.val.ts b/examples/next/app/blogs/[blog]/page.val.ts index f9544a619..a833f4d7e 100644 --- a/examples/next/app/blogs/[blog]/page.val.ts +++ b/examples/next/app/blogs/[blog]/page.val.ts @@ -3,7 +3,7 @@ import authorsVal from "../../../content/authors.val"; import { linkSchema } from "../../../components/link.val"; const blogSchema = s.object({ - title: s.string().maxLength(3), + title: s.string(), content: s.richtext({ inline: { a: s.route(), @@ -228,7 +228,7 @@ export default c.define( }, }, "/blogs/blog-15": { - title: "Blog 1", + title: "Blog dette er jo sinnsyjt! men hvorfor har vi to feil?", content: [ { tag: "p", From 0fcfecf07444d009f2f24ffdba69efb00eebf149 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Wed, 24 Jun 2026 20:32:59 +0200 Subject: [PATCH 04/35] Fix issue where keyOf and router stopped publishing from working --- .changeset/witty-mangos-fix.md | 5 +++++ packages/ui/spa/ValSyncEngine.ts | 22 ++++++++-------------- 2 files changed, 13 insertions(+), 14 deletions(-) create mode 100644 .changeset/witty-mangos-fix.md diff --git a/.changeset/witty-mangos-fix.md b/.changeset/witty-mangos-fix.md new file mode 100644 index 000000000..756930d30 --- /dev/null +++ b/.changeset/witty-mangos-fix.md @@ -0,0 +1,5 @@ +--- +"@valbuild/ui": patch +--- + +Fix publish being blocked by keyOf/route validation fixes that are resolved at read time. The manual and fs-mode auto-publish gates now consult the resolved/partitioned validation errors (the same set the UI surfaces) instead of the raw worker output, which always contains `keyof:check-keys`/`router:check-route` placeholders. diff --git a/packages/ui/spa/ValSyncEngine.ts b/packages/ui/spa/ValSyncEngine.ts index f2d6d9229..f421d350b 100644 --- a/packages/ui/spa/ValSyncEngine.ts +++ b/packages/ui/spa/ValSyncEngine.ts @@ -2989,17 +2989,10 @@ export class ValSyncEngine { this.globalServerSidePatchIds && this.globalServerSidePatchIds.length > 0 ) { - let hasValidationError = false; - for (const sourcePathS in this.errors.validationErrors || {}) { - const sourcePath = sourcePathS as SourcePath; - if ( - this.errors?.validationErrors?.[sourcePath] && - this.errors?.validationErrors?.[sourcePath]!.length > 0 - ) { - hasValidationError = true; - break; - } - } + const surfacedValidationErrors = this.getAllValidationErrorsSnapshot(); + const hasValidationError = Object.values( + surfacedValidationErrors || {}, + ).some((errors) => errors && errors.length > 0); if (!hasValidationError) { await this.publish( this.globalServerSidePatchIds.concat( @@ -3012,7 +3005,7 @@ export class ValSyncEngine { } else { console.debug( "Skip auto-publish since there's validation errors", - this.errors.validationErrors, + surfacedValidationErrors, ); } } @@ -3055,14 +3048,15 @@ export class ValSyncEngine { this.publishDisabled = true; this.invalidatePublishDisabled(); + const surfacedValidationErrors = this.getAllValidationErrorsSnapshot(); const hasValidationError = - Object.values(this.errors.validationErrors || {}).flatMap( + Object.values(surfacedValidationErrors || {}).flatMap( (errors) => errors || [], ).length > 0; if (hasValidationError) { console.debug( "Skipping publish since there's validation errors", - this.errors.validationErrors, + surfacedValidationErrors, ); this.addGlobalTransientError( "Could not publish changes, since there are validation errors", From 08209c62514b230652fdbeeb77e4daa6fbccf371 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 18:33:35 +0000 Subject: [PATCH 05/35] Version Packages --- .changeset/witty-mangos-fix.md | 5 ----- packages/cli/package.json | 2 +- packages/next/package.json | 2 +- packages/react/package.json | 2 +- packages/server/package.json | 2 +- packages/ui/package.json | 2 +- 6 files changed, 5 insertions(+), 10 deletions(-) delete mode 100644 .changeset/witty-mangos-fix.md diff --git a/.changeset/witty-mangos-fix.md b/.changeset/witty-mangos-fix.md deleted file mode 100644 index 756930d30..000000000 --- a/.changeset/witty-mangos-fix.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@valbuild/ui": patch ---- - -Fix publish being blocked by keyOf/route validation fixes that are resolved at read time. The manual and fs-mode auto-publish gates now consult the resolved/partitioned validation errors (the same set the UI surfaces) instead of the raw worker output, which always contains `keyof:check-keys`/`router:check-route` placeholders. diff --git a/packages/cli/package.json b/packages/cli/package.json index adea840d4..aa8cccbd8 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@valbuild/cli", "private": false, - "version": "0.97.1", + "version": "0.97.2", "description": "Val CLI tools", "repository": { "type": "git", diff --git a/packages/next/package.json b/packages/next/package.json index 300a91fb5..93b039214 100644 --- a/packages/next/package.json +++ b/packages/next/package.json @@ -12,7 +12,7 @@ "next", "react" ], - "version": "0.97.1", + "version": "0.97.2", "scripts": { "typecheck": "tsc --noEmit", "test": "jest" diff --git a/packages/react/package.json b/packages/react/package.json index f2acaaafa..8aedfed24 100644 --- a/packages/react/package.json +++ b/packages/react/package.json @@ -1,6 +1,6 @@ { "name": "@valbuild/react", - "version": "0.97.1", + "version": "0.97.2", "private": false, "description": "Val - React internal helpers", "repository": { diff --git a/packages/server/package.json b/packages/server/package.json index 81f5e29d2..9448fea6a 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -16,7 +16,7 @@ "./package.json": "./package.json" }, "types": "dist/valbuild-server.cjs.d.ts", - "version": "0.97.1", + "version": "0.97.2", "scripts": { "typecheck": "tsc --noEmit", "test": "jest", diff --git a/packages/ui/package.json b/packages/ui/package.json index a355590f7..04617b560 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -1,6 +1,6 @@ { "name": "@valbuild/ui", - "version": "0.97.1", + "version": "0.97.2", "sideEffects": false, "repository": { "type": "git", From be33f33fd28546b560fef4188fef84fab6069e43 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Wed, 24 Jun 2026 21:59:48 +0200 Subject: [PATCH 06/35] Use type-only import for ValidationWorkerFactory in ValSyncEngine Address Copilot review on #446: mark ValidationWorkerFactory as a type import so it is erased at emit time. Co-Authored-By: Claude Opus 4.8 --- packages/ui/spa/ValSyncEngine.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ui/spa/ValSyncEngine.ts b/packages/ui/spa/ValSyncEngine.ts index d4923ed13..9de9b3df7 100644 --- a/packages/ui/spa/ValSyncEngine.ts +++ b/packages/ui/spa/ValSyncEngine.ts @@ -35,7 +35,7 @@ import { PatchSets, SerializedPatchSet } from "./utils/PatchSets"; import { ReifiedRender } from "@valbuild/core"; import { ValidationWorkerClient, - ValidationWorkerFactory, + type ValidationWorkerFactory, } from "./validation/ValidationWorkerClient"; import { partitionValidationErrors } from "./validation/partitionValidationErrors"; From 539ae72c02b2676943d6a7f70594ddb84385ebac Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Wed, 24 Jun 2026 21:59:48 +0200 Subject: [PATCH 07/35] Use type-only import from @valbuild/core in ValidationWorkerClient Address Copilot review on #446: the @valbuild/core imports are all types, so import them with `import type` to avoid emitting a runtime dependency. Co-Authored-By: Claude Opus 4.8 --- packages/ui/spa/validation/ValidationWorkerClient.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/ui/spa/validation/ValidationWorkerClient.ts b/packages/ui/spa/validation/ValidationWorkerClient.ts index b110910ec..9c78dbfc5 100644 --- a/packages/ui/spa/validation/ValidationWorkerClient.ts +++ b/packages/ui/spa/validation/ValidationWorkerClient.ts @@ -1,4 +1,4 @@ -import { +import type { ModuleFilePath, SerializedSchema, Source, From cb8360e8e1f08d80c7c1365c1dc0d01b1c4ced65 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Wed, 24 Jun 2026 21:59:49 +0200 Subject: [PATCH 08/35] Split runtime and type-only imports in validateModule Address Copilot review on #446: only deserializeSchema is needed at runtime; import the rest with `import type`. Co-Authored-By: Claude Opus 4.8 --- packages/ui/spa/validation/validateModule.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/ui/spa/validation/validateModule.ts b/packages/ui/spa/validation/validateModule.ts index 66ee5bfd3..70e894de6 100644 --- a/packages/ui/spa/validation/validateModule.ts +++ b/packages/ui/spa/validation/validateModule.ts @@ -1,5 +1,5 @@ -import { - deserializeSchema, +import { deserializeSchema } from "@valbuild/core"; +import type { ModuleFilePath, Schema, SelectorSource, From 0f635b11c074a0878e93383157ec3dad351a6ac6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Wed, 24 Jun 2026 20:28:10 +0000 Subject: [PATCH 09/35] Version Packages --- .changeset/witty-mangos-fix.md | 5 ----- packages/cli/package.json | 2 +- packages/next/package.json | 2 +- packages/react/package.json | 2 +- packages/server/package.json | 2 +- packages/ui/package.json | 2 +- 6 files changed, 5 insertions(+), 10 deletions(-) delete mode 100644 .changeset/witty-mangos-fix.md diff --git a/.changeset/witty-mangos-fix.md b/.changeset/witty-mangos-fix.md deleted file mode 100644 index a2f8dce8f..000000000 --- a/.changeset/witty-mangos-fix.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"@valbuild/ui": patch ---- - -Fix client-side validation worker failing to load ("Unexpected token '<'"). The worker was reached via a dynamic import, which Vite emitted as a code-split chunk resolved relative to the entry's served URL (`/api/val/static/{version}/app`) rather than its real `/assets/` location, so the request returned the SPA HTML shell instead of JavaScript. The worker is now created with `new URL("./validation.worker.ts", import.meta.url)` like the search and patchsets workers (an absolute asset URL), and the worker factory is injected from the composition root so it loads reliably regardless of where the SPA entry is served. diff --git a/packages/cli/package.json b/packages/cli/package.json index aa8cccbd8..7f393bff4 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@valbuild/cli", "private": false, - "version": "0.97.2", + "version": "0.97.3", "description": "Val CLI tools", "repository": { "type": "git", diff --git a/packages/next/package.json b/packages/next/package.json index 93b039214..2b302f97c 100644 --- a/packages/next/package.json +++ b/packages/next/package.json @@ -12,7 +12,7 @@ "next", "react" ], - "version": "0.97.2", + "version": "0.97.3", "scripts": { "typecheck": "tsc --noEmit", "test": "jest" diff --git a/packages/react/package.json b/packages/react/package.json index 8aedfed24..082d242d3 100644 --- a/packages/react/package.json +++ b/packages/react/package.json @@ -1,6 +1,6 @@ { "name": "@valbuild/react", - "version": "0.97.2", + "version": "0.97.3", "private": false, "description": "Val - React internal helpers", "repository": { diff --git a/packages/server/package.json b/packages/server/package.json index 9448fea6a..ccea11792 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -16,7 +16,7 @@ "./package.json": "./package.json" }, "types": "dist/valbuild-server.cjs.d.ts", - "version": "0.97.2", + "version": "0.97.3", "scripts": { "typecheck": "tsc --noEmit", "test": "jest", diff --git a/packages/ui/package.json b/packages/ui/package.json index 04617b560..838da425b 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -1,6 +1,6 @@ { "name": "@valbuild/ui", - "version": "0.97.2", + "version": "0.97.3", "sideEffects": false, "repository": { "type": "git", From 7947ac640a222fd05e644236702dd1a9a6ec32bf Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Sat, 27 Jun 2026 22:51:20 +0200 Subject: [PATCH 10/35] Initialize support for jsonValues --- docs/plans/jsonValues.md | 150 +++++++++++++++++ packages/core/src/index.ts | 4 + packages/core/src/initVal.ts | 3 + packages/core/src/module.ts | 18 +++ packages/core/src/schema/deserialize.ts | 1 + packages/core/src/schema/jsonValues.test.ts | 153 ++++++++++++++++++ packages/core/src/schema/record.ts | 94 ++++++++++- packages/core/src/selector/index.ts | 34 ++-- packages/core/src/source/index.ts | 2 + packages/core/src/source/json.ts | 105 ++++++++++++ .../src/loadValModules.jsonValues.test.ts | 38 +++++ packages/server/src/loadValModules.ts | 22 ++- .../server/src/patch/ts/jsonReference.test.ts | 32 ++++ packages/server/src/patch/ts/ops.ts | 35 ++++ .../test/jsonValues-fixture/blogs.val.ts | 12 ++ .../page/blogs/test.val.json | 3 + .../test/jsonValues-fixture/tsconfig.json | 12 ++ .../test/jsonValues-fixture/val.config.ts | 4 + .../test/jsonValues-fixture/val.modules.ts | 4 + 19 files changed, 709 insertions(+), 17 deletions(-) create mode 100644 docs/plans/jsonValues.md create mode 100644 packages/core/src/schema/jsonValues.test.ts create mode 100644 packages/core/src/source/json.ts create mode 100644 packages/server/src/loadValModules.jsonValues.test.ts create mode 100644 packages/server/src/patch/ts/jsonReference.test.ts create mode 100644 packages/server/test/jsonValues-fixture/blogs.val.ts create mode 100644 packages/server/test/jsonValues-fixture/page/blogs/test.val.json create mode 100644 packages/server/test/jsonValues-fixture/tsconfig.json create mode 100644 packages/server/test/jsonValues-fixture/val.config.ts create mode 100644 packages/server/test/jsonValues-fixture/val.modules.ts diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md new file mode 100644 index 000000000..c6b2de238 --- /dev/null +++ b/docs/plans/jsonValues.md @@ -0,0 +1,150 @@ +# Implementation tracker: `.jsonValues()` — lazily-loaded JSON entries + +> Living implementation plan for `s.record(...).jsonValues()` / `s.router(...).jsonValues()`. +> The **design rationale + locked decisions** live in the approved design doc +> (`~/.claude/plans/i-want-a-new-humming-zebra.md`). This file tracks **what's done / +> what's next** so we can resume across sessions. Keep the "Current state" block at the +> top up to date after every work chunk. + +--- + +## Current state / resume here + +- **Phase**: 1 ✅ complete. Phase 2 in progress — loader ✅, `createValJsonReference` ✅; + **next: the ValOps commit/serialize flow (task below)**. +- **Next step (the deep part)**: wire the json commit flow in `ValOps.ts`/`ValOpsFS`/`ValOpsHttp`: + (a) shallow-serialize json records to `{ key: { _type:"json", _sha } }` markers for `/sources/~` + (the runtime thunk must be dropped — `getJsonImport` reads it at runtime only); (b) on commit, + write/replace/delete `*.val.json` files + emit/update the `c.json(...)` thunk in the `.val.ts` + using `createValJsonReference` (already in `patch/ts/ops.ts`) via `insertAt`/`removeAt`/ + `replaceNodeValue`; (c) per-entry content load (invoke `getJsonImport(marker)?.()` or read the + file by AST-derived path) + sha-keyed incremental validation calling + `RecordSchema.validateJsonEntryContent`. Then the single-entry fetch endpoint in `ValServer.ts`. +- **Last verified green**: whole-monorepo `pnpm run -r typecheck` (only pre-existing unrelated + `packages/cli` chokidar failure); `pnpm test packages/core` (454) + the 2 new server json suites. +- **Key API note**: `JsonSource` is a phantom-typed pure-JSON marker; the lazy thunk is + runtime-only — read it with `Internal.getJsonImport(source)`, never `source._import` in typed code. + +--- + +## Goal (one paragraph) + +Let `s.record(...)` and `s.router(...)` (NOT `s.images()` / `s.files()` galleries) opt into +`.jsonValues()`, so each entry's value lives in its own `*.val.json` file referenced by a lazy +thunk `c.json(() => import("./x.val.json"), "")`. Keeps `.val.ts` tiny at 10K+ +entries; runtime/Studio/validation work one entry at a time; zero overhead when Val is disabled. + +## Locked decisions (do not relitigate) + +1. `fetchVal`/`useVal` stay eager; new `fetchValKey`/`useValKey` + `fetchValRoute`/`useValRoute` + load a single entry. +2. Hybrid authoring: Val generates/maintains json files + thunks; hand-edits re-validated. +3. The sha = content hash → validation-cache key (auto-generated). +4. Type precision: keep object/array structure, widen only what JSON can't carry + (literals → base, drop `RawString`/brand, widen `_type` literals). Runtime validation enforces + strictness. Val object-unions are always discriminated, so distribute+recurse suffices. +5. i18n deferred; design json format locale-agnostic. +6. All-or-nothing: every entry of a `.jsonValues()` record is a `c.json` thunk (no mixing). + +## Key runtime shapes + +`JsonSource` is a phantom-typed **pure-JSON marker** so `Source` stays JSON-serializable: + +```ts +// JsonSource TYPE (what flows through Source/SelectorSource): +{ _type: "json", _sha: string, patch_id?: string } & PhantomType + +// RUNTIME value produced by c.json(thunk, sha) — also carries the thunk, which is +// NOT in the type; read it via Internal.getJsonImport(source): +{ _type: "json", _import: () => Promise<{ default: T }>, _sha: string } + +// Over the wire (/sources/~): the marker only (thunk dropped): +{ _type: "json", _sha: string, patch_id?: string } +``` + +--- + +## Phase 1 — Core (`packages/core`) ✅ + +- [x] `source/json.ts`: `JsonSource` (default `T = unknown` so the unions accept any + content), `_type:"json"` const (`JSON_VAL_EXTENSION_TAG`), `json()` ctor, `isJson()`, + `JsonOf` transform (distribute + recurse + widen leaves). +- [x] `source/index.ts`: `JsonSource` added to `Source` union. +- [x] `selector/index.ts`: `JsonSource` in `SelectorSource`; mapped in `Selector` → + `GenericSelector`. +- [x] `initVal.ts`: `json` added to `c` + `ContentConstructor`. +- [x] `schema/record.ts`: `.jsonValues()` modifier + `isJsonValues` flag; `Src` widened to + `JsonValuesRecordSrc`; serializes `jsonValues`; throws on media galleries; defers value + validation (record-level only asserts `isJson` marker); `validateJsonEntryContent()` helper. +- [x] `schema/record.ts` (`SerializedRecordSchema`) + `schema/deserialize.ts`: carry `jsonValues`. +- [x] `module.ts` `resolvePath` (both variants): descend a json entry → schema becomes `item`; + throws/returns clear error when traversing deeper into an unloaded marker. +- [x] `index.ts`: export `JsonSource`/`JsonOf` types + `Internal.isJson`. +- [x] Tests: `schema/jsonValues.test.ts` — `JsonOf` compile-time, `c.json` unit, serialize/ + deserialize round-trip, router compose, gallery rejection, deferred validation, authoring + surface (`c.define` + `c.json`). +- [x] **Verified**: `pnpm test packages/core` (454 tests) + core typecheck green. + +## Phase 2 — Server (`packages/server`) + +- [x] `loadValModules.ts`: `loadModule` parses `.json` (mirrors Node `require` json); `.json` added + to `RESOLVE_EXTENSIONS`; dynamic `import()` transpiles to a lazy `require` via `customRequire` + so thunks stay lazy. Fixture: `test/jsonValues-fixture/` + `loadValModules.jsonValues.test.ts` + (verifies marker shape, laziness, and thunk-loads-json). ✅ +- [x] `patch/ts/ops.ts`: `createValJsonReference(importPath, sha)` — emits + `c.json(() => import("..."), "sha")` (factory-built; uses `createIdentifier("import")` to print + a dynamic import without casting the ImportKeyword token). Tested in `jsonReference.test.ts`. ✅ +- [ ] `patch/ts/ops.ts` (remaining): wire add/replace/remove of json entries through + `insertAt`/`removeAt`/`replaceNodeValue` (+ write/replace-sha/delete `*.val.json`) — done with + the ValOps commit flow below. +- [ ] `ValOps.ts` / `ValOpsFS.ts` / `ValOpsHttp.ts`: shallow source serialization for json records + (`{ key: { _type:"json", _sha } }`); per-entry content load (invoke thunk / read by + AST-derived path); sha-keyed incremental validation w/ cache; commit writes `*.val.json` + + updates `.val.ts` shas/thunks; recompute SHAs. +- [ ] `ValServer.ts`: endpoint to fetch one entry's content (draft-aware via `patch_id`); + `/sources/~` returns shallow markers for json records. +- [ ] **Verify**: `pnpm test packages/server/...` green (ops add/replace/remove, loader fixture, + incremental validation). + +## Phase 3 — UI (`packages/ui/spa`) + +- [ ] `ValSyncEngine.ts`: model json records as `{ key → { sha, patch_id? } }`; lazy fetch + cache + one entry on open; per-entry patches; sha-aware invalidation. +- [ ] `components/ValFieldProvider.tsx`: reuse `useShallowSourceAtPath` for the key list; add a + hook to lazily fetch one entry's content (`useJsonEntrySource` / extend `useSourceAtPath`). +- [ ] `components/fields/RecordFields.tsx` + router nav: render keys without loading content; load + on open; build per-entry add/replace/remove patches. +- [ ] **Verify**: manual Studio pass (list shows keys w/o loading; open fetches one json; + edit→commit writes file + updates sha; add/remove a route inserts/removes thunk + file). + +## Phase 4 — Runtime APIs (`packages/next`, `packages/react`) + +- [ ] `rsc/initValRsc.ts`: `fetchValKey` / `fetchValRoute` (route matching reuses `ValRouter`). +- [ ] `client/initValClient.ts`: `useValKey` / `useValRoute`. +- [ ] Apply existing stega/transform per resolved entry. +- [ ] **Verify**: `fetchVal` still returns all; `fetchValRoute` imports only the requested entry. + +## Phase 5 — Example + CI gate + +- [ ] Add a `.jsonValues()` router to `examples/next` with a few `*.val.json` entries. +- [ ] `cd examples/next && pnpm run build` green; confirm single-entry import in output. +- [ ] Full CI: `pnpm run lint`, `pnpm -w run format`, `pnpm run -r typecheck`, `pnpm test`, + `pnpm run build`, `cd examples/next && pnpm run build`. + +--- + +## Open questions / watch-list + +- `JsonOf` correctness vs `resolveJsonModule` inference (esp. images inside json: `_type` + widens to `string`, so json must NOT keep the literal `"file"` brand — `JsonOf` widens it). +- `vm` loader dynamic `import()` + `.json` resolution. +- resolvePath/selector must never throw on a not-yet-loaded entry. +- Canonical content hashing so hybrid hand-edits and Val-writes agree on the sha. + +## Changelog + +- **Session 1**: Phase 1 (core) complete + tested; Phase 2 loader done + tested; + `createValJsonReference` primitive done + tested. Whole monorepo typechecks (except pre-existing + unrelated `packages/cli` chokidar error). `JsonSource` redesigned to a phantom-typed pure-JSON + marker (`_import` is runtime-only via `getJsonImport`) so `Source` stays JSON-serializable. +- _(baseline)_ tracker created; design approved. diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 1ac1a663f..0f9f601e2 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -17,6 +17,7 @@ export type { FileMetadata } from "./schema/file"; export type { ValModule, SerializedModule, InferValModuleType } from "./module"; export type { SourceObject, SourcePrimitive, Source } from "./source"; export type { FileSource } from "./source/file"; +export type { JsonSource, JsonOf, JsonImportThunk } from "./source/json"; export type { RemoteSource, RemoteRef } from "./source/remote"; export { DEFAULT_VAL_REMOTE_HOST } from "./schema/remote"; export type { RawString } from "./schema/string"; @@ -98,6 +99,7 @@ import { import { type ImageMetadata } from "./schema/image"; import { type FileMetadata } from "./schema/file"; import { isFile } from "./source/file"; +import { isJson, getJsonImport } from "./source/json"; import { createRemoteRef } from "./source/remote"; import { getValidationBasis, @@ -197,6 +199,8 @@ const Internal = { }, isVal, isFile, + isJson, + getJsonImport, createValPathOfItem, getSHA256Hash, initSchema, diff --git a/packages/core/src/initVal.ts b/packages/core/src/initVal.ts index ff738279b..307ebe322 100644 --- a/packages/core/src/initVal.ts +++ b/packages/core/src/initVal.ts @@ -3,6 +3,7 @@ import { InitSchema, initSchema } from "./initSchema"; import { getValPath as getPath } from "./val"; import { initFile } from "./source/file"; import { initImage } from "./source/image"; +import { json } from "./source/json"; import { initRemote } from "./source/remote"; // import { i18n, I18n } from "./source/future/i18n"; // import { remote } from "./source/future/remote"; @@ -13,6 +14,7 @@ export type ContentConstructor = { file: ReturnType; image: ReturnType; remote: ReturnType; + json: typeof json; }; export type ValConstructor = { unstable_getPath: typeof getPath; @@ -96,6 +98,7 @@ InitVal => { remote: initRemote(config), file: initFile(config), image: initImage(config), + json, }, s, config, diff --git a/packages/core/src/module.ts b/packages/core/src/module.ts index 9cae09e7a..9806a0b2f 100644 --- a/packages/core/src/module.ts +++ b/packages/core/src/module.ts @@ -20,6 +20,7 @@ import { SerializedImageSchema, } from "./schema/image"; import { FILE_REF_PROP, FileSource } from "./source/file"; +import { isJson } from "./source/json"; import { AllRichTextOptions, RichTextSource } from "./source/richtext"; import { RecordSchema, SerializedRecordSchema } from "./schema/record"; import { RawString } from "./schema/string"; @@ -342,6 +343,15 @@ export function resolvePath< resolvedSchema instanceof RecordSchema ? resolvedSchema?.["item"] : resolvedSchema.item; + // A `.jsonValues()` entry value is a lazily-loaded JsonSource marker. We + // can resolve the entry itself (schema is now `item`), but we cannot + // descend deeper until the backing `*.val.json` content has been loaded + // and substituted into the source tree (done by the UI/server layer). + if (isJson(resolvedSource) && parts.length > 0) { + throw Error( + `Cannot resolve path into a jsonValues entry until its content is loaded. Path: ${path}`, + ); + } } else if (isObjectSchema(resolvedSchema)) { if (typeof resolvedSource !== "object") { throw Error( @@ -593,6 +603,14 @@ export function safeResolvePath< resolvedSchema instanceof RecordSchema ? resolvedSchema?.["item"] : resolvedSchema.item; + // jsonValues entry: cannot descend deeper until the backing `*.val.json` + // content is loaded into the source tree. + if (isJson(resolvedSource) && parts.length > 0) { + return { + status: "error", + message: `Cannot resolve path into a jsonValues entry until its content is loaded. Path: ${path}`, + }; + } } else if (isObjectSchema(resolvedSchema)) { if (resolvedSource === undefined) { return { diff --git a/packages/core/src/schema/deserialize.ts b/packages/core/src/schema/deserialize.ts index 8da5e0a92..98d593ab5 100644 --- a/packages/core/src/schema/deserialize.ts +++ b/packages/core/src/schema/deserialize.ts @@ -179,6 +179,7 @@ function deserializeSchemaImpl( false, false, serialized.description, + serialized.jsonValues ?? false, ); case "keyOf": return new KeyOfSchema( diff --git a/packages/core/src/schema/jsonValues.test.ts b/packages/core/src/schema/jsonValues.test.ts new file mode 100644 index 000000000..6cd05bcf0 --- /dev/null +++ b/packages/core/src/schema/jsonValues.test.ts @@ -0,0 +1,153 @@ +import { SourcePath } from "../val"; +import { json, isJson, getJsonImport, JsonOf } from "../source/json"; +import { VAL_EXTENSION } from "../source"; +import { deserializeSchema } from "./deserialize"; +import { Schema } from "."; +import { SelectorSource } from "../selector"; +import { object } from "./object"; +import { record } from "./record"; +import { router } from "./router"; +import { string } from "./string"; +import { images } from "./images"; +import { nextAppRouter } from "../router"; +import { initVal } from "../initVal"; + +describe("c.json + .jsonValues()", () => { + test("json() returns a JsonSource marker with thunk + sha", () => { + const thunk = () => Promise.resolve({ default: { title: "hi" } }); + const src = json(thunk, "abc123"); + expect(src[VAL_EXTENSION]).toBe("json"); + expect(src._sha).toBe("abc123"); + expect(getJsonImport(src)).toBe(thunk); + expect(isJson(src)).toBe(true); + expect(isJson({ title: "hi" })).toBe(false); + expect(isJson(null)).toBe(false); + }); + + test(".jsonValues() serializes with jsonValues: true", () => { + const schema = record(object({ title: string() })).jsonValues(); + const serialized = schema["executeSerialize"](); + expect(serialized.type).toBe("record"); + expect(serialized.jsonValues).toBe(true); + // item schema is still serialized so the UI/validation can use it + expect(serialized.item.type).toBe("object"); + }); + + test("non-jsonValues record does not set jsonValues", () => { + const schema = record(object({ title: string() })); + const serialized = schema["executeSerialize"](); + expect(serialized.jsonValues).toBeUndefined(); + }); + + test("deserialize round-trips jsonValues flag", () => { + const schema = record(object({ title: string() })).jsonValues(); + const serialized = schema["executeSerialize"](); + const deserialized = deserializeSchema(serialized); + const reserialized = deserialized["executeSerialize"](); + expect(reserialized.type).toBe("record"); + if (reserialized.type === "record") { + expect(reserialized.jsonValues).toBe(true); + } + }); + + test(".jsonValues() composes with s.router()", () => { + const schema = router( + nextAppRouter, + object({ title: string() }), + ).jsonValues(); + const serialized = schema["executeSerialize"](); + expect(serialized.jsonValues).toBe(true); + expect(serialized.router).toBe("next-app-router"); + }); + + test(".jsonValues() throws on image galleries", () => { + expect(() => images().jsonValues()).toThrow(/jsonValues/); + }); + + describe("validation", () => { + const schema = record(object({ title: string() })).jsonValues(); + // A loosely-typed reference so we can feed untyped values (as the server + // does when validating sources loaded over the wire) without casts. + const looseSchema: Schema = schema; + + test("accepts json markers and never loads content during validation", () => { + let loaded = false; + const src = { + a: json(() => { + // deep content is never inspected at the record level — deferred to + // validateJsonEntryContent once the file is loaded. + loaded = true; + return Promise.resolve({ default: { title: "ok" } }); + }, "s1"), + }; + const errors = schema["executeValidate"]( + "/test.val.ts" as SourcePath, + src, + ); + expect(errors).toBe(false); + expect(loaded).toBe(false); + }); + + test("rejects a non-json entry value", () => { + const errors = looseSchema["executeValidate"]( + "/test.val.ts" as SourcePath, + // deliberately an inline value, which is invalid for a jsonValues record + { a: { title: "inline-not-allowed" } }, + ); + expect(errors).not.toBe(false); + }); + + test("validateJsonEntryContent validates loaded content against item", () => { + const okErrors = schema.validateJsonEntryContent( + '/test.val.ts?p="a"' as SourcePath, + { title: "ok" }, + ); + expect(okErrors).toBe(false); + + const badErrors = schema.validateJsonEntryContent( + '/test.val.ts?p="a"' as SourcePath, + // wrong leaf type + { title: 123 }, + ); + expect(badErrors).not.toBe(false); + }); + }); +}); + +describe("c.define authoring surface (compile-time)", () => { + test("define accepts c.json entries for a .jsonValues() router", () => { + const { s, c } = initVal(); + // Simulates `import("./blogs/test.val.json")` whose JSON-inferred type is + // the widened content shape. + const mod = c.define( + "/blogs/[slug]/page.val.ts", + s.router(nextAppRouter, s.object({ title: s.string() })).jsonValues(), + { + "/blogs/test": c.json( + () => Promise.resolve({ default: { title: "Hello" } }), + "1232132", + ), + }, + ); + expect(mod).toBeDefined(); + }); +}); + +describe("JsonOf type transform", () => { + test("widens literals, keeps structure (compile-time)", () => { + // string-literal union -> string + const a: JsonOf<"x" | "y"> = "anything"; + // number literal -> number + const b: JsonOf<1 | 2> = 5; + // object structure preserved, leaves widened + const c: JsonOf<{ kind: "a"; n: 1 }> = { kind: "whatever", n: 42 }; + // array of widened + const d: JsonOf = ["x", "y"]; + // discriminated union (distributes + recurses) + const e: JsonOf<{ k: "a"; x: number } | { k: "b"; y: string }> = { + k: "a", + x: 1, + }; + expect([a, b, c, d, e]).toBeDefined(); + }); +}); diff --git a/packages/core/src/schema/record.ts b/packages/core/src/schema/record.ts index 7b1aa3794..63170021d 100644 --- a/packages/core/src/schema/record.ts +++ b/packages/core/src/schema/record.ts @@ -14,6 +14,7 @@ import { unsafeCreateSourcePath, } from "../selector/SelectorProxy"; import { ImageSource } from "../source/image"; +import { JsonOf, JsonSource, isJson } from "../source/json"; import { RemoteSource } from "../source/remote"; import { ModuleFilePath, SourcePath } from "../val"; import { ImageMetadata } from "./image"; @@ -44,15 +45,31 @@ export type SerializedRecordSchema = { directory?: string; remote?: boolean; alt?: SerializedSchema; + // When true, entry values are stored in separate lazily-loaded `*.val.json` + // files (see `.jsonValues()`). + jsonValues?: boolean; readonly?: boolean; hidden?: boolean; description?: string; }; +/** + * The source type of a `.jsonValues()` record: every entry value is a lazily + * loaded {@link JsonSource} whose resolved content is the (loosened, see + * {@link JsonOf}) item type. + */ +export type JsonValuesRecordSrc< + T extends Schema, + K extends Schema, +> = Record, JsonSource>>>; + export class RecordSchema< T extends Schema, K extends Schema, - Src extends Record, SelectorOfSchema> | null, + Src extends + | Record, SelectorOfSchema> + | JsonValuesRecordSrc + | null, > extends Schema { constructor( private readonly item: T, @@ -64,6 +81,8 @@ export class RecordSchema< private readonly isReadonly: boolean = false, private readonly isHidden: boolean = false, private readonly description?: string, + /** When true, entry values are lazily loaded {@link JsonSource} thunks. */ + private readonly isJsonValues: boolean = false, ) { super(); } @@ -79,6 +98,7 @@ export class RecordSchema< this.isReadonly, this.isHidden, description ?? undefined, + this.isJsonValues, ); } @@ -95,6 +115,7 @@ export class RecordSchema< this.isReadonly, this.isHidden, this.description, + this.isJsonValues, ); } @@ -238,6 +259,22 @@ export class RecordSchema< this.markKeyErrorsAtPath(entryErr, subPath); error = error ? { ...error, ...entryErr } : entryErr; } + } else if (this.isJsonValues) { + // jsonValues record: the value is a lazily-loaded JsonSource marker. + // Only assert the marker shape here; the deep content validation is + // deferred and run per-entry by the server (validateJsonEntryContent) + // once the backing `*.val.json` is loaded. + if (!isJson(elem)) { + error = this.appendValidationError( + error, + subPath, + `Expected a c.json(...) entry, got '${ + elem === null ? "null" : typeof elem + }'`, + elem, + true, + ); + } } else { const subError = this.item["executeValidate"]( subPath, @@ -532,6 +569,7 @@ export class RecordSchema< this.isReadonly, this.isHidden, this.description, + this.isJsonValues, ) as RecordSchema; } @@ -546,6 +584,7 @@ export class RecordSchema< true, this.isHidden, this.description, + this.isJsonValues, ); } @@ -560,6 +599,7 @@ export class RecordSchema< this.isReadonly, true, this.description, + this.isJsonValues, ); } @@ -574,6 +614,7 @@ export class RecordSchema< this.isReadonly, this.isHidden, this.description, + this.isJsonValues, ); } @@ -588,9 +629,41 @@ export class RecordSchema< this.isReadonly, this.isHidden, this.description, + this.isJsonValues, ); } + /** + * Store each entry's value in its own lazily-loaded `*.val.json` file instead + * of inlining it in the `.val.ts` module. Entry values become + * {@link JsonSource} thunks (`c.json(() => import("./entry.val.json"), sha)`), + * which lets the runtime, the Studio and validation work one entry at a time + * so a record/router can scale to many thousands of entries. + * + * Not supported on image/file galleries (`s.images()` / `s.files()`). + */ + jsonValues(): RecordSchema> { + if (this.mediaOptions) { + throw new Error( + ".jsonValues() cannot be used with image/file galleries (s.images()/s.files())", + ); + } + return new RecordSchema( + this.item, + this.opt, + // custom validate functions are typed against the previous Src; drop them + // since the source shape changes to JsonSource entries. + [], + this.currentRouter, + this.keySchema, + this.mediaOptions, + this.isReadonly, + this.isHidden, + this.description, + true, + ) as RecordSchema>; + } + private getRouterValidations(path: SourcePath, src: Src): ValidationErrors { if (!this.currentRouter) { return false; @@ -663,6 +736,7 @@ export class RecordSchema< customValidate: this.customValidateFunctions && this.customValidateFunctions?.length > 0, + jsonValues: this.isJsonValues ? true : undefined, readonly: this.isReadonly, hidden: this.isHidden, description: this.description, @@ -688,6 +762,19 @@ export class RecordSchema< }; } | null = null; + /** + * Validate the loaded content of a single `.jsonValues()` entry against the + * item schema. The server calls this once it has loaded the backing + * `*.val.json` for an entry (the deep validation that `executeValidate` + * defers). + */ + validateJsonEntryContent( + path: SourcePath, + content: SelectorSource, + ): ValidationErrors { + return this.item["executeValidate"](path, content); + } + protected override executeRender( sourcePath: SourcePath | ModuleFilePath, src: Src, @@ -696,6 +783,11 @@ export class RecordSchema< if (src === null) { return res; } + if (this.isJsonValues) { + // jsonValues entries are not inlined, so there is no per-entry source to + // render at the record level. Rendering happens per entry once loaded. + return res; + } for (const key in src) { const itemSrc = src[key as unknown as SelectorOfSchema]; if (itemSrc === null || itemSrc === undefined) { diff --git a/packages/core/src/selector/index.ts b/packages/core/src/selector/index.ts index aea8ad54d..8519603ac 100644 --- a/packages/core/src/selector/index.ts +++ b/packages/core/src/selector/index.ts @@ -15,6 +15,7 @@ import { ImageSelector } from "./image"; import { RichTextSelector } from "./richtext"; import { ImageSource } from "../source/image"; import { RemoteSource } from "../source/remote"; +import { JsonSource } from "../source/json"; export type Selector = Source extends T ? GenericSelector @@ -22,21 +23,23 @@ export type Selector = Source extends T ? ImageSelector : T extends FileSource ? FileSelector - : T extends RichTextSource - ? RichTextSelector - : T extends SourceObject - ? ObjectSelector - : T extends SourceArray - ? ArraySelector - : T extends string - ? StringSelector - : T extends number - ? NumberSelector - : T extends boolean - ? BooleanSelector - : T extends null - ? PrimitiveSelector - : never; + : T extends JsonSource + ? GenericSelector + : T extends RichTextSource + ? RichTextSelector + : T extends SourceObject + ? ObjectSelector + : T extends SourceArray + ? ArraySelector + : T extends string + ? StringSelector + : T extends number + ? NumberSelector + : T extends boolean + ? BooleanSelector + : T extends null + ? PrimitiveSelector + : never; export type SelectorSource = | SourcePrimitive @@ -48,6 +51,7 @@ export type SelectorSource = | ImageSource | FileSource | RemoteSource + | JsonSource | RichTextSource | GenericSelector; diff --git a/packages/core/src/source/index.ts b/packages/core/src/source/index.ts index 47de4448a..f8a4d0eb8 100644 --- a/packages/core/src/source/index.ts +++ b/packages/core/src/source/index.ts @@ -1,5 +1,6 @@ import { FileSource } from "./file"; import { ImageSource } from "./image"; +import { JsonSource } from "./json"; import { RemoteSource } from "./remote"; import { RichTextOptions, RichTextSource } from "./richtext"; @@ -10,6 +11,7 @@ export type Source = | RemoteSource | FileSource | ImageSource + | JsonSource | RichTextSource; export type SourceObject = { [key in string]: Source } & { diff --git a/packages/core/src/source/json.ts b/packages/core/src/source/json.ts new file mode 100644 index 000000000..82422523d --- /dev/null +++ b/packages/core/src/source/json.ts @@ -0,0 +1,105 @@ +import { PhantomType, VAL_EXTENSION } from "."; +import { Json } from "../Json"; + +/** + * The string used as the `_type` discriminator of a {@link JsonSource}. + */ +export const JSON_VAL_EXTENSION_TAG = "json" as const; + +/** + * The lazy import thunk a {@link JsonSource} carries at runtime. It is NOT part + * of the {@link JsonSource} type because `Source` must stay JSON-serializable + * (it is sent over the wire as `Json`); the thunk is a runtime-only detail, + * read via {@link getJsonImport}. + */ +export type JsonImportThunk = () => Promise<{ default: T }>; + +/** + * A JSON source represents a record/router entry whose value is stored in a + * separate `*.val.json` file and loaded lazily via a dynamic `import()` thunk. + * + * This is what `c.json(() => import("./entry.val.json"), "")` returns. + * + * The *type* is a pure-JSON marker (`{ _type: "json", _sha, patch_id? }`) so that + * `Source` stays JSON-serializable. At runtime the value additionally carries a + * lazy import thunk (read via {@link getJsonImport}); over the wire only the + * marker is sent. + * + * The phantom type parameter `T` is the (loosened, see {@link JsonOf}) type of + * the value the JSON file resolves to. It is covariant: a `JsonSource` of a + * narrower value type is assignable to a `JsonSource` of a wider one. + */ +export type JsonSource = { + readonly [VAL_EXTENSION]: typeof JSON_VAL_EXTENSION_TAG; + /** Content hash of the backing JSON, used as a validation-cache key. */ + readonly _sha: string; + /** Set on uncommitted/draft entries (mirrors FileSource/RemoteSource). */ + readonly patch_id?: string; +} & PhantomType; + +export function json( + importThunk: JsonImportThunk, + sha: string, +): JsonSource { + return { + [VAL_EXTENSION]: JSON_VAL_EXTENSION_TAG, + // runtime-only: not described by JsonSource so Source stays JSON-serializable + _import: importThunk, + _sha: sha, + } as unknown as JsonSource; +} + +export function isJson(obj: unknown): obj is JsonSource { + return ( + typeof obj === "object" && + obj !== null && + VAL_EXTENSION in obj && + (obj as { [VAL_EXTENSION]?: unknown })[VAL_EXTENSION] === + JSON_VAL_EXTENSION_TAG + ); +} + +/** + * Reads the runtime-only lazy import thunk from a {@link JsonSource}. Returns + * `undefined` for a transport marker (sent over the wire without the thunk). + */ +export function getJsonImport(source: JsonSource): JsonImportThunk | undefined { + const candidate = (source as { _import?: unknown })._import; + return typeof candidate === "function" + ? (candidate as JsonImportThunk) + : undefined; +} + +/** + * Loosens a (strict) schema source type `T` into the type that a + * `resolveJsonModule` import of the backing `*.val.json` can actually satisfy. + * + * Why this is needed: TypeScript widens literals when it infers the type of a + * JSON module (e.g. `"a"` becomes `string`, `1` becomes `number`, and a branded + * `_type: "file"` becomes `_type: string`). A strict schema type (with literal + * unions, `RawString` brands, etc.) would therefore reject a perfectly valid + * JSON file. `JsonOf` performs the same widening at the type level so the + * JSON content typechecks against the schema as strictly as JSON allows. Runtime + * validation enforces the rest. + * + * The transform distributes over unions and recurses through objects/arrays, + * preserving structure while widening every leaf literal. Val object-unions are + * always discriminated, so distribute-and-recurse is sufficient (the discriminant + * literal widens to its base type, but the per-variant shape stays distinct and a + * concrete JSON value still matches exactly one variant). + */ +export type JsonOf = T extends string + ? string + : T extends number + ? number + : T extends boolean + ? boolean + : T extends null + ? null + : T extends undefined + ? undefined + : T extends readonly (infer E)[] + ? JsonOf[] + : T extends object + ? { [K in keyof T]: JsonOf } + : T; diff --git a/packages/server/src/loadValModules.jsonValues.test.ts b/packages/server/src/loadValModules.jsonValues.test.ts new file mode 100644 index 000000000..b36235001 --- /dev/null +++ b/packages/server/src/loadValModules.jsonValues.test.ts @@ -0,0 +1,38 @@ +import path from "path"; +import { Internal } from "@valbuild/core"; +import { loadValModules } from "./loadValModules"; + +const fixtureRoot = path.join(__dirname, "..", "test", "jsonValues-fixture"); + +describe("loadValModules with .jsonValues() + c.json", () => { + test("loads the module without invoking entry thunks (stays lazy)", async () => { + const valModules = loadValModules(fixtureRoot); + expect(valModules.modules).toHaveLength(1); + + const mod = (await valModules.modules[0].def()).default; + const source = Internal.getSource(mod) as Record; + const entry = source["/blogs/test"] as { + _type: string; + _sha: string; + _import: () => Promise<{ default: unknown }>; + }; + + // The entry is a json marker, not the loaded content. + expect(Internal.isJson(entry)).toBe(true); + expect(entry._type).toBe("json"); + expect(entry._sha).toBe("testsha123"); + expect(typeof entry._import).toBe("function"); + }); + + test("invoking the entry thunk loads the backing *.val.json", async () => { + const valModules = loadValModules(fixtureRoot); + const mod = (await valModules.modules[0].def()).default; + const source = Internal.getSource(mod) as Record; + const entry = source["/blogs/test"] as { + _import: () => Promise<{ default: unknown }>; + }; + + const loaded = await entry._import(); + expect(loaded.default).toEqual({ title: "Hello from JSON" }); + }); +}); diff --git a/packages/server/src/loadValModules.ts b/packages/server/src/loadValModules.ts index 51463a1b7..3dcc3bfe2 100644 --- a/packages/server/src/loadValModules.ts +++ b/packages/server/src/loadValModules.ts @@ -56,7 +56,15 @@ function findValModulesPath(projectRoot: string): string | null { return null; } -const RESOLVE_EXTENSIONS = [".ts", ".tsx", ".js", ".jsx", ".cjs", ".mjs"]; +const RESOLVE_EXTENSIONS = [ + ".ts", + ".tsx", + ".js", + ".jsx", + ".cjs", + ".mjs", + ".json", +]; // Specifiers that user val files must not actually use. We stub them so that // importing is fine, but using a value throws a clear error. Real @valbuild @@ -110,6 +118,18 @@ function loadModule( if (cached) { return cached; } + // JSON modules (e.g. the `*.val.json` files backing `.jsonValues()` entries) + // are loaded by parsing, mirroring Node's `require("./x.json")` which returns + // the parsed object as `module.exports`. The importing `.val.ts` wraps this + // with `__importStar` so `import("./x.val.json")` yields `{ default, ... }`. + // These are only loaded when an entry thunk is invoked, never during + // `extractValModules`, so this stays lazy. + if (absPath.endsWith(".json")) { + const parsed = JSON.parse(fs.readFileSync(absPath, "utf-8")); + const jsonModule = { exports: parsed as Record }; + cache[absPath] = jsonModule; + return jsonModule; + } const code = fs.readFileSync(absPath, "utf-8"); const transpiled = ts.transpileModule(code, { compilerOptions: { diff --git a/packages/server/src/patch/ts/jsonReference.test.ts b/packages/server/src/patch/ts/jsonReference.test.ts new file mode 100644 index 000000000..46415e424 --- /dev/null +++ b/packages/server/src/patch/ts/jsonReference.test.ts @@ -0,0 +1,32 @@ +import ts from "typescript"; +import { createValJsonReference } from "./ops"; + +const printer = ts.createPrinter({ newLine: ts.NewLineKind.LineFeed }); + +function print(node: ts.Node): string { + const doc = ts.createSourceFile("out.ts", "", ts.ScriptTarget.ES2020, true); + return printer.printNode(ts.EmitHint.Unspecified, node, doc); +} + +describe("createValJsonReference", () => { + test("prints c.json(() => import(...), sha)", () => { + const node = createValJsonReference("./page/blogs/test.val.json", "abc123"); + const out = print(node); + expect(out).toContain("c.json("); + expect(out).toContain('import("./page/blogs/test.val.json")'); + expect(out).toContain('"abc123"'); + expect(out).toContain("() =>"); + }); + + test("produces valid, re-parseable output", () => { + const node = createValJsonReference("./a.val.json", "sha"); + const out = print(node); + const reparsed = ts.createSourceFile( + "reparse.ts", + `const x = ${out};`, + ts.ScriptTarget.ES2020, + true, + ); + expect(reparsed.statements).toHaveLength(1); + }); +}); diff --git a/packages/server/src/patch/ts/ops.ts b/packages/server/src/patch/ts/ops.ts index 51380f5fe..064cfa4e1 100644 --- a/packages/server/src/patch/ts/ops.ts +++ b/packages/server/src/patch/ts/ops.ts @@ -126,6 +126,41 @@ function createValRemoteReference(value: RemoteSource) { ); } +/** + * Builds the expression `c.json(() => import(""), "")` used to + * reference a lazily-loaded `*.val.json` entry of a `.jsonValues()` record. + */ +export function createValJsonReference( + importPath: string, + sha: string, +): ts.Expression { + // () => import("") + // NOTE: an `import` identifier prints as the dynamic-import keyword call, + // which avoids casting the ImportKeyword token (not typed as an Expression). + const importCall = ts.factory.createCallExpression( + ts.factory.createIdentifier("import"), + undefined, + [ts.factory.createStringLiteral(importPath)], + ); + const thunk = ts.factory.createArrowFunction( + undefined, + undefined, + [], + undefined, + ts.factory.createToken(ts.SyntaxKind.EqualsGreaterThanToken), + importCall, + ); + // c.json(, "") + return ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier("c"), + ts.factory.createIdentifier("json"), + ), + undefined, + [thunk, ts.factory.createStringLiteral(sha)], + ); +} + function toExpression(value: JSONValue): ts.Expression { if (typeof value === "string") { // TODO: Use configuration/heuristics to determine use of single quote or double quote diff --git a/packages/server/test/jsonValues-fixture/blogs.val.ts b/packages/server/test/jsonValues-fixture/blogs.val.ts new file mode 100644 index 000000000..392e8fae2 --- /dev/null +++ b/packages/server/test/jsonValues-fixture/blogs.val.ts @@ -0,0 +1,12 @@ +import { s, c } from "./val.config"; + +export default c.define( + "/blogs.val.ts", + s.record(s.object({ title: s.string() })).jsonValues(), + { + "/blogs/test": c.json( + () => import("./page/blogs/test.val.json"), + "testsha123", + ), + }, +); diff --git a/packages/server/test/jsonValues-fixture/page/blogs/test.val.json b/packages/server/test/jsonValues-fixture/page/blogs/test.val.json new file mode 100644 index 000000000..ff16dfa0f --- /dev/null +++ b/packages/server/test/jsonValues-fixture/page/blogs/test.val.json @@ -0,0 +1,3 @@ +{ + "title": "Hello from JSON" +} diff --git a/packages/server/test/jsonValues-fixture/tsconfig.json b/packages/server/test/jsonValues-fixture/tsconfig.json new file mode 100644 index 000000000..f08c25e6d --- /dev/null +++ b/packages/server/test/jsonValues-fixture/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "CommonJS", + "moduleResolution": "Node", + "esModuleInterop": true, + "resolveJsonModule": true, + "strict": true, + "skipLibCheck": true + }, + "include": ["**/*.ts", "**/*.json"] +} diff --git a/packages/server/test/jsonValues-fixture/val.config.ts b/packages/server/test/jsonValues-fixture/val.config.ts new file mode 100644 index 000000000..bb301c8ba --- /dev/null +++ b/packages/server/test/jsonValues-fixture/val.config.ts @@ -0,0 +1,4 @@ +import { initVal } from "@valbuild/core"; + +const { s, c, config } = initVal(); +export { s, c, config }; diff --git a/packages/server/test/jsonValues-fixture/val.modules.ts b/packages/server/test/jsonValues-fixture/val.modules.ts new file mode 100644 index 000000000..ffb0c1b24 --- /dev/null +++ b/packages/server/test/jsonValues-fixture/val.modules.ts @@ -0,0 +1,4 @@ +import { modules } from "@valbuild/core"; +import { config } from "./val.config"; + +export default modules(config, [{ def: () => import("./blogs.val.ts") }]); From 6cd29ed3eccf87bc372feb4bf438da9766ccaa6f Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Sun, 28 Jun 2026 19:28:20 +0200 Subject: [PATCH 11/35] Add server-side jsonValues content validation + eager resolver Validate the content of every .jsonValues() entry server-side by loading each backing *.val.json via its lazy import thunk and checking it against the record's item schema (validateJsonValuesEntries), wired into ValOps.validateSources. The record-level executeValidate only asserts the marker shape, so this performs the deferred deep validation. Add Internal.resolveJsonValues(source): recursively resolves all JsonSource markers by invoking their thunks, returning a fully-inlined source. This backs the eager fetchVal/useVal path (a jsonValues module's local source is markers, not content, so it must be resolved before stega-encoding). Co-Authored-By: Claude Opus 4.8 --- docs/plans/jsonValues.md | 29 +++++-- packages/core/src/index.ts | 3 +- packages/core/src/schema/jsonValues.test.ts | 45 ++++++++++- packages/core/src/source/json.ts | 39 +++++++++- packages/server/src/ValOps.ts | 20 +++++ .../server/src/validateJsonValues.test.ts | 72 +++++++++++++++++ packages/server/src/validateJsonValues.ts | 78 +++++++++++++++++++ 7 files changed, 277 insertions(+), 9 deletions(-) create mode 100644 packages/server/src/validateJsonValues.test.ts create mode 100644 packages/server/src/validateJsonValues.ts diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md index c6b2de238..a6be4d02f 100644 --- a/docs/plans/jsonValues.md +++ b/docs/plans/jsonValues.md @@ -20,10 +20,21 @@ `replaceNodeValue`; (c) per-entry content load (invoke `getJsonImport(marker)?.()` or read the file by AST-derived path) + sha-keyed incremental validation calling `RecordSchema.validateJsonEntryContent`. Then the single-entry fetch endpoint in `ValServer.ts`. -- **Last verified green**: whole-monorepo `pnpm run -r typecheck` (only pre-existing unrelated - `packages/cli` chokidar failure); `pnpm test packages/core` (454) + the 2 new server json suites. +- **Last verified green**: core json suite (14 tests) + server `validateJsonValues`/loader/ + `jsonReference` suites; core + server typecheck. (Earlier: whole-monorepo `-r typecheck` clean + except the pre-existing unrelated `packages/cli` chokidar failure.) - **Key API note**: `JsonSource` is a phantom-typed pure-JSON marker; the lazy thunk is runtime-only — read it with `Internal.getJsonImport(source)`, never `source._import` in typed code. +- **Done since**: server-side per-entry json validation (`validateJsonValues.ts`, wired into + `ValOps.validateSources`); core `Internal.resolveJsonValues(source)` (eager resolver for the + `fetchVal`/`useVal` path). **Discovered gap**: even eager `fetchVal` must resolve markers before + stega-encoding (a jsonValues module's local source is markers, not content) — use + `resolveJsonValues` there. +- **Runtime integration notes (for fetchValKey/fetchVal in next/react)**: disabled/production path + reads the local module (`Internal.getSource`) whose markers still carry thunks → resolve locally + (`getJsonImport` for one key, `resolveJsonValues` for all). Enabled/Studio path gets shallow + markers from `/sources/~` WITHOUT thunks → needs the single-entry fetch endpoint (Phase 2) to load + draft content; until then it can fall back to the local thunk (committed content only). --- @@ -97,10 +108,16 @@ entries; runtime/Studio/validation work one entry at a time; zero overhead when - [ ] `patch/ts/ops.ts` (remaining): wire add/replace/remove of json entries through `insertAt`/`removeAt`/`replaceNodeValue` (+ write/replace-sha/delete `*.val.json`) — done with the ValOps commit flow below. -- [ ] `ValOps.ts` / `ValOpsFS.ts` / `ValOpsHttp.ts`: shallow source serialization for json records - (`{ key: { _type:"json", _sha } }`); per-entry content load (invoke thunk / read by - AST-derived path); sha-keyed incremental validation w/ cache; commit writes `*.val.json` + - updates `.val.ts` shas/thunks; recompute SHAs. +- [x] **Per-entry validation**: `validateJsonValues.ts` (`validateJsonValuesEntries`) loads each + entry's content via `getJsonImport` and validates against the item schema; wired into + `ValOps.validateSources` (runs before the `res === false` early-continue). Tested in + `validateJsonValues.test.ts` (valid/invalid/load-error/non-jsonValues-skip). ✅ +- [ ] `ValOps.ts` / `ValOpsFS.ts` / `ValOpsHttp.ts` (remaining): confirm shallow source + serialization on `/sources/~` (JSON.stringify already drops the thunk → `{_type,_sha}`); + commit writes `*.val.json` + updates `.val.ts` shas/thunks (use `createValJsonReference` + + `insertAt`/`removeAt`/`replaceNodeValue`); sha-keyed incremental validation (optimization); + recompute SHAs. +- [x] Core eager resolver `Internal.resolveJsonValues(source)` (for `fetchVal`/`useVal`). ✅ - [ ] `ValServer.ts`: endpoint to fetch one entry's content (draft-aware via `patch_id`); `/sources/~` returns shallow markers for json records. - [ ] **Verify**: `pnpm test packages/server/...` green (ops add/replace/remove, loader fixture, diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 0f9f601e2..2f38a0782 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -99,7 +99,7 @@ import { import { type ImageMetadata } from "./schema/image"; import { type FileMetadata } from "./schema/file"; import { isFile } from "./source/file"; -import { isJson, getJsonImport } from "./source/json"; +import { isJson, getJsonImport, resolveJsonValues } from "./source/json"; import { createRemoteRef } from "./source/remote"; import { getValidationBasis, @@ -201,6 +201,7 @@ const Internal = { isFile, isJson, getJsonImport, + resolveJsonValues, createValPathOfItem, getSHA256Hash, initSchema, diff --git a/packages/core/src/schema/jsonValues.test.ts b/packages/core/src/schema/jsonValues.test.ts index 6cd05bcf0..fd5287d5c 100644 --- a/packages/core/src/schema/jsonValues.test.ts +++ b/packages/core/src/schema/jsonValues.test.ts @@ -1,5 +1,12 @@ import { SourcePath } from "../val"; -import { json, isJson, getJsonImport, JsonOf } from "../source/json"; +import { + json, + isJson, + getJsonImport, + resolveJsonValues, + JsonOf, +} from "../source/json"; +import { Source } from "../source"; import { VAL_EXTENSION } from "../source"; import { deserializeSchema } from "./deserialize"; import { Schema } from "."; @@ -114,6 +121,42 @@ describe("c.json + .jsonValues()", () => { }); }); +describe("resolveJsonValues", () => { + test("resolves a record of markers into inlined content", async () => { + const source: Source = { + "/a": json(() => Promise.resolve({ default: { title: "A" } }), "sa"), + "/b": json(() => Promise.resolve({ default: { title: "B" } }), "sb"), + }; + const resolved = await resolveJsonValues(source); + expect(resolved).toEqual({ "/a": { title: "A" }, "/b": { title: "B" } }); + }); + + test("resolves nested markers recursively", async () => { + const source: Source = { + "/outer": json( + () => + Promise.resolve({ + default: { + inner: json( + () => Promise.resolve({ default: { deep: "value" } }), + "si", + ), + }, + }), + "so", + ), + }; + const resolved = await resolveJsonValues(source); + expect(resolved).toEqual({ "/outer": { inner: { deep: "value" } } }); + }); + + test("leaves transport markers (no thunk) as-is", async () => { + const marker = { _type: "json", _sha: "x" } as unknown as Source; + const resolved = await resolveJsonValues({ "/a": marker }); + expect(resolved).toEqual({ "/a": { _type: "json", _sha: "x" } }); + }); +}); + describe("c.define authoring surface (compile-time)", () => { test("define accepts c.json entries for a .jsonValues() router", () => { const { s, c } = initVal(); diff --git a/packages/core/src/source/json.ts b/packages/core/src/source/json.ts index 82422523d..8c10ca1ce 100644 --- a/packages/core/src/source/json.ts +++ b/packages/core/src/source/json.ts @@ -1,4 +1,4 @@ -import { PhantomType, VAL_EXTENSION } from "."; +import { PhantomType, Source, VAL_EXTENSION } from "."; import { Json } from "../Json"; /** @@ -70,6 +70,43 @@ export function getJsonImport(source: JsonSource): JsonImportThunk | undefined { : undefined; } +/** + * Recursively resolves every {@link JsonSource} marker in a source tree by + * invoking its lazy import thunk, returning a fully-inlined source. This backs + * the eager `fetchVal`/`useVal` path (load everything). + * + * Markers without a runtime thunk (transport markers received over the wire) + * are left as-is — there is nothing to load from them locally. Other branded + * leaf sources (file/image/remote/richtext) are returned untouched. + */ +export async function resolveJsonValues(source: Source): Promise { + if (isJson(source)) { + const thunk = getJsonImport(source); + if (!thunk) { + return source; + } + const loaded = (await thunk()).default; + return resolveJsonValues(loaded as Source); + } + if (Array.isArray(source)) { + return Promise.all(source.map((item) => resolveJsonValues(item))); + } + if (source !== null && typeof source === "object") { + if (VAL_EXTENSION in source) { + // a branded leaf source (file/image/remote) — no nested json markers + return source; + } + const entries = await Promise.all( + Object.entries(source).map( + async ([key, value]) => + [key, await resolveJsonValues(value as Source)] as const, + ), + ); + return Object.fromEntries(entries) as Source; + } + return source; +} + /** * Loosens a (strict) schema source type `T` into the type that a * `resolveJsonModule` import of the backing `*.val.json` can actually satisfy. diff --git a/packages/server/src/ValOps.ts b/packages/server/src/ValOps.ts index 4911d15b0..9c9f0b947 100644 --- a/packages/server/src/ValOps.ts +++ b/packages/server/src/ValOps.ts @@ -34,6 +34,7 @@ import { } from "@valbuild/core/patch"; import { TSOps } from "./patch/ts/ops"; import { analyzeValModule } from "./patch/ts/valModule"; +import { validateJsonValuesEntries } from "./validateJsonValues"; import ts from "typescript"; import { ValSyntaxError, ValSyntaxErrorTree } from "./patch/ts/syntax"; import sizeOf from "image-size"; @@ -484,6 +485,25 @@ export abstract class ValOps { path as string as SourcePath, source, ); + // For `.jsonValues()` records, executeValidate only checks the entry + // markers; load + validate each entry's backing `*.val.json` content here. + const jsonValuesErrors = await validateJsonValuesEntries( + schema, + source, + path, + ); + for (const [sourcePathS, entryErrors] of Object.entries( + jsonValuesErrors, + )) { + const sourcePath = sourcePathS as SourcePath; + if (!errors[path]) { + errors[path] = { validations: {} }; + } + if (!errors[path].validations[sourcePath]) { + errors[path].validations[sourcePath] = []; + } + errors[path].validations[sourcePath].push(...entryErrors); + } if (res === false) { continue; } diff --git a/packages/server/src/validateJsonValues.test.ts b/packages/server/src/validateJsonValues.test.ts new file mode 100644 index 000000000..60afa0fab --- /dev/null +++ b/packages/server/src/validateJsonValues.test.ts @@ -0,0 +1,72 @@ +import { initVal, ModuleFilePath } from "@valbuild/core"; +import { validateJsonValuesEntries } from "./validateJsonValues"; + +const { s, c } = initVal(); +const modulePath = "/blogs.val.ts" as ModuleFilePath; + +describe("validateJsonValuesEntries", () => { + const schema = s.record(s.object({ title: s.string() })).jsonValues(); + + test("returns no errors when all entry content is valid", async () => { + const source = { + "/a": c.json( + () => Promise.resolve({ default: { title: "ok" } }), + "sha-a", + ), + "/b": c.json( + () => Promise.resolve({ default: { title: "ok2" } }), + "sha-b", + ), + }; + const errors = await validateJsonValuesEntries(schema, source, modulePath); + expect(errors).toEqual({}); + }); + + test("reports validation errors for invalid entry content", async () => { + const source = { + "/a": c.json( + () => Promise.resolve({ default: { title: "ok" } }), + "sha-a", + ), + // wrong leaf type for title — caught by the deferred content validation + "/bad": c.json( + () => Promise.resolve({ default: { title: 123 } }), + "sha-bad", + ), + }; + const errors = await validateJsonValuesEntries(schema, source, modulePath); + const keys = Object.keys(errors); + expect(keys.length).toBeGreaterThan(0); + expect(keys.some((k) => k.includes("/bad"))).toBe(true); + }); + + test("reports a load error when the entry thunk rejects", async () => { + const source = { + "/boom": c.json(() => Promise.reject(new Error("disk gone")), "sha-boom"), + }; + const errors = await validateJsonValuesEntries(schema, source, modulePath); + const keys = Object.keys(errors); + expect(keys.length).toBe(1); + expect(errors[keys[0] as keyof typeof errors][0].message).toContain( + "Could not load JSON entry", + ); + }); + + test("skips non-jsonValues records (no content loading)", async () => { + const plainSchema = s.record(s.object({ title: s.string() })); + let loaded = false; + const source = { + "/a": c.json(() => { + loaded = true; + return Promise.resolve({ default: { title: "ok" } }); + }, "sha-a"), + }; + const errors = await validateJsonValuesEntries( + plainSchema, + source, + modulePath, + ); + expect(errors).toEqual({}); + expect(loaded).toBe(false); + }); +}); diff --git a/packages/server/src/validateJsonValues.ts b/packages/server/src/validateJsonValues.ts new file mode 100644 index 000000000..4cd3c2fec --- /dev/null +++ b/packages/server/src/validateJsonValues.ts @@ -0,0 +1,78 @@ +import { + Internal, + ModuleFilePath, + RecordSchema, + Schema, + SelectorSource, + SourcePath, +} from "@valbuild/core"; +import { ValidationError } from "@valbuild/core"; + +/** + * Validates the content of every `.jsonValues()` entry in a module by loading + * each backing `*.val.json` (via its lazy import thunk) and checking it against + * the record's item schema. + * + * The record-level `executeValidate` only asserts the marker shape — content + * validation is deferred (the content isn't inlined). This performs that + * deferred deep validation server-side. Every loadable entry is validated; + * validation is allowed to be slower at scale, so a sha-keyed skip-cache is left + * as a later optimization. + * + * Entries without a runtime import thunk (transport markers / draft entries + * whose content lives in a patch) are skipped here — their content is validated + * where it is loaded (the single-entry fetch path). + */ +export async function validateJsonValuesEntries( + schema: Schema, + source: unknown, + modulePath: ModuleFilePath, +): Promise> { + const out: Record = {}; + if (!(schema instanceof RecordSchema)) { + return out; + } + if (!schema["executeSerialize"]().jsonValues) { + return out; + } + if (source === null || typeof source !== "object" || Array.isArray(source)) { + return out; + } + for (const [key, marker] of Object.entries(source)) { + const entryPath = Internal.createValPathOfItem( + modulePath as string as SourcePath, + key, + ); + if (!entryPath) { + continue; + } + if (!Internal.isJson(marker)) { + // a non-marker entry is already reported by the record-level validation + continue; + } + const thunk = Internal.getJsonImport(marker); + if (!thunk) { + continue; + } + let content: SelectorSource; + try { + content = (await thunk()).default as SelectorSource; + } catch (err) { + out[entryPath] = [ + { + message: `Could not load JSON entry '${key}': ${ + err instanceof Error ? err.message : String(err) + }`, + }, + ]; + continue; + } + const entryErrors = schema.validateJsonEntryContent(entryPath, content); + if (entryErrors) { + for (const [p, errs] of Object.entries(entryErrors)) { + out[p as SourcePath] = errs; + } + } + } + return out; +} From 708a7ed1a6eb6b113e476d07fb15d80b08cc6778 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Sun, 28 Jun 2026 20:43:31 +0200 Subject: [PATCH 12/35] Add jsonValues validation sha helper (schema-hash prefix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The c.json validation sha is composed as "-" with the item-schema hash as a readable PREFIX (not hashed together). This lets us decide an entry needs revalidation by comparing the prefix to the current schema hash WITHOUT reading the *.val.json content — a schema change flags affected entries by scanning only the shas already present in the .val.ts. - computeJsonEntrySha(itemSchema, content): the composite sha - getJsonItemSchemaHash / getSchemaHashFromJsonSha: schema-hash prefix - jsonEntrySchemaHashIsStale(storedSha, itemSchema): content-free staleness check - isJsonEntryShaCurrent(storedSha, itemSchema, content): full freshness check Co-Authored-By: Claude Opus 4.8 --- docs/plans/jsonValues.md | 46 +++++++++++-- packages/server/src/jsonValuesSha.test.ts | 64 ++++++++++++++++++ packages/server/src/jsonValuesSha.ts | 79 +++++++++++++++++++++++ 3 files changed, 185 insertions(+), 4 deletions(-) create mode 100644 packages/server/src/jsonValuesSha.test.ts create mode 100644 packages/server/src/jsonValuesSha.ts diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md index a6be4d02f..ae0de0980 100644 --- a/docs/plans/jsonValues.md +++ b/docs/plans/jsonValues.md @@ -27,9 +27,12 @@ runtime-only — read it with `Internal.getJsonImport(source)`, never `source._import` in typed code. - **Done since**: server-side per-entry json validation (`validateJsonValues.ts`, wired into `ValOps.validateSources`); core `Internal.resolveJsonValues(source)` (eager resolver for the - `fetchVal`/`useVal` path). **Discovered gap**: even eager `fetchVal` must resolve markers before - stega-encoding (a jsonValues module's local source is markers, not content) — use - `resolveJsonValues` there. + `fetchVal`/`useVal` path); **sha helper `jsonValuesSha.ts`** (`computeJsonEntrySha` = + `-`, `isJsonEntryShaCurrent`, `getSchemaHashFromJsonSha`) implementing the + revalidation-signal design — the commit flow should write this sha; the sha-keyed validation + skip-cache should use `isJsonEntryShaCurrent`. **Discovered gap**: even eager `fetchVal` must + resolve markers before stega-encoding (a jsonValues module's local source is markers, not content) + — use `resolveJsonValues` there. - **Runtime integration notes (for fetchValKey/fetchVal in next/react)**: disabled/production path reads the local module (`Internal.getSource`) whose markers still carry thunks → resolve locally (`getJsonImport` for one key, `resolveJsonValues` for all). Enabled/Studio path gets shallow @@ -50,7 +53,14 @@ entries; runtime/Studio/validation work one entry at a time; zero overhead when 1. `fetchVal`/`useVal` stay eager; new `fetchValKey`/`useValKey` + `fetchValRoute`/`useValRoute` load a single entry. 2. Hybrid authoring: Val generates/maintains json files + thunks; hand-edits re-validated. -3. The sha = content hash → validation-cache key (auto-generated). +3. The sha = validation-cache key (auto-generated), formatted as **`"-"`** + with the **schema hash as a PREFIX** (NOT hashed together). The prefix is the key property: you + can compare the current item-schema hash against each entry's sha prefix to know the entry needs + revalidation **without reading the entry's content** — the `.val.ts` already lists every entry's + sha. Revalidate when EITHER the prefix differs from the current schema hash (schema changed → + whole record) OR the content hash differs (that entry changed). Implemented in + `jsonValuesSha.ts` (`computeJsonEntrySha`, `getSchemaHashFromJsonSha`, `isJsonEntryShaCurrent`). + See "sha design" below. 4. Type precision: keep object/array structure, widen only what JSON can't carry (literals → base, drop `RawString`/brand, widen `_type` literals). Runtime validation enforces strictness. Val object-unions are always discriminated, so distribute+recurse suffices. @@ -150,10 +160,38 @@ entries; runtime/Studio/validation work one entry at a time; zero overhead when --- +## sha design (revalidation signal) + +The trailing sha in `c.json(thunk, sha)` is the validation-cache key, formatted +**`"-"`** (schema hash as PREFIX), and must answer "must we revalidate this +entry?" cheaply — ideally **without reading the entry's content**: + +- **Schema hash = prefix.** Derived from the serialized item schema (stable serialization) and + written as the part before the `-`. Keep them as separate joined components, NOT hashed together, + so the prefix stays readable. Truncated small (currently 8 hex chars) — it only needs to detect + schema _changes_, not be collision-proof. +- **Cheap staleness scan (the point):** after a schema change, compute the current item-schema hash + once, then walk the record's entry shas (already present in the `.val.ts`) and flag every entry + whose sha PREFIX differs — no `*.val.json` is loaded. Those entries need revalidation. +- **Content hash** (suffix) flips when the backing `*.val.json` content changes ⇒ revalidate just + that entry; the commit flow rewrites the sha. (Computing the content hash _does_ require the + content, but you only reach it for entries whose schema prefix already matches.) +- **Validation skip-cache:** `validateJsonValuesEntries` (today validates every entry) becomes: + if the entry's stored sha prefix ≠ current schema hash ⇒ must revalidate (load + validate, rewrite + sha); else look up `sha → lastValidationResult` and skip on a cached PASS, otherwise load + + validate + cache. Use `isJsonEntryShaCurrent` / `getSchemaHashFromJsonSha` from `jsonValuesSha.ts`. +- **Hybrid hand-edit safety:** a hand-edited json whose content no longer matches the suffix forces + revalidation; Val rewrites the sha on next commit. + ## Open questions / watch-list - `JsonOf` correctness vs `resolveJsonModule` inference (esp. images inside json: `_type` widens to `string`, so json must NOT keep the literal `"file"` brand — `JsonOf` widens it). +- ~~Does `getSources()`/`deepClone` preserve the `_import` thunk?~~ **Resolved**: `deepClone` + ([patch/util.ts](packages/core/src/patch/util.ts)) passes functions through unchanged and + `_import` is enumerable, so base-source thunks survive. Patched/draft json entries become thunkless + markers (over-the-wire value) → `validateJsonValuesEntries` skips them by design (validated via the + draft/endpoint path later). So committed validation is correct. - `vm` loader dynamic `import()` + `.json` resolution. - resolvePath/selector must never throw on a not-yet-loaded entry. - Canonical content hashing so hybrid hand-edits and Val-writes agree on the sha. diff --git a/packages/server/src/jsonValuesSha.test.ts b/packages/server/src/jsonValuesSha.test.ts new file mode 100644 index 000000000..fb30151c9 --- /dev/null +++ b/packages/server/src/jsonValuesSha.test.ts @@ -0,0 +1,64 @@ +import { initVal } from "@valbuild/core"; +import { + computeJsonEntrySha, + getJsonItemSchemaHash, + getSchemaHashFromJsonSha, + isJsonEntryShaCurrent, + jsonEntrySchemaHashIsStale, +} from "./jsonValuesSha"; + +const { s } = initVal(); + +describe("jsonValues validation sha", () => { + const item = s.object({ title: s.string() }); + + test("is deterministic for the same schema + content", () => { + expect(computeJsonEntrySha(item, { title: "a" })).toBe( + computeJsonEntrySha(item, { title: "a" }), + ); + }); + + test("content change flips the sha but keeps the schema component", () => { + const sha1 = computeJsonEntrySha(item, { title: "a" }); + const sha2 = computeJsonEntrySha(item, { title: "b" }); + expect(sha1).not.toBe(sha2); + // same schema → same schema-hash component (so only that entry revalidates) + expect(getSchemaHashFromJsonSha(sha1)).toBe(getSchemaHashFromJsonSha(sha2)); + }); + + test("schema change flips the schema component (forces record revalidation)", () => { + const item2 = s.object({ title: s.string(), extra: s.number() }); + const shaA = computeJsonEntrySha(item, { title: "a" }); + const shaB = computeJsonEntrySha(item2, { title: "a" }); + expect(getJsonItemSchemaHash(item)).not.toBe(getJsonItemSchemaHash(item2)); + expect(getSchemaHashFromJsonSha(shaA)).not.toBe( + getSchemaHashFromJsonSha(shaB), + ); + }); + + test("isJsonEntryShaCurrent detects content and schema drift", () => { + const sha = computeJsonEntrySha(item, { title: "a" }); + expect(isJsonEntryShaCurrent(sha, item, { title: "a" })).toBe(true); + // content drift + expect(isJsonEntryShaCurrent(sha, item, { title: "changed" })).toBe(false); + // schema drift + const item2 = s.object({ title: s.string(), extra: s.number() }); + expect(isJsonEntryShaCurrent(sha, item2, { title: "a" })).toBe(false); + }); + + test("getSchemaHashFromJsonSha returns null for a non-composite sha", () => { + expect(getSchemaHashFromJsonSha("1232132")).toBeNull(); + }); + + test("jsonEntrySchemaHashIsStale detects schema drift WITHOUT reading content", () => { + // sha was written against `item`; we never pass the content here. + const storedSha = computeJsonEntrySha(item, { title: "a" }); + expect(jsonEntrySchemaHashIsStale(storedSha, item)).toBe(false); + + const changedItem = s.object({ title: s.string(), extra: s.number() }); + expect(jsonEntrySchemaHashIsStale(storedSha, changedItem)).toBe(true); + + // a hand-authored / legacy sha with no schema prefix is treated as stale + expect(jsonEntrySchemaHashIsStale("1232132", item)).toBe(true); + }); +}); diff --git a/packages/server/src/jsonValuesSha.ts b/packages/server/src/jsonValuesSha.ts new file mode 100644 index 000000000..c1b63e399 --- /dev/null +++ b/packages/server/src/jsonValuesSha.ts @@ -0,0 +1,79 @@ +import { Internal, Schema, SelectorSource } from "@valbuild/core"; + +const textEncoder = new TextEncoder(); + +function hash(input: string): string { + return Internal.getSHA256Hash(textEncoder.encode(input)); +} + +/** + * A small, stable hash of a record's item schema. It is the *schema component* + * of a json entry's validation sha (see {@link computeJsonEntrySha}). Because it + * is derived from the serialized item schema, it flips whenever the schema + * changes — which is how we know every entry of a `.jsonValues()` record must be + * revalidated after a schema change. + */ +export function getJsonItemSchemaHash( + itemSchema: Schema, +): string { + return hash(JSON.stringify(itemSchema["executeSerialize"]())).slice(0, 8); +} + +/** + * The validation sha stored in `c.json(thunk, sha)`. It is a composite + * `"-"` so that revalidation is required when EITHER: + * - the entry content changes (the content-hash component flips), or + * - the item schema changes (the schema-hash component flips → revalidate the + * whole record). + * + * Validation/commit compares the stored sha against a freshly-computed one and + * only revalidates on mismatch; the commit flow writes this sha into the + * `.val.ts`. + */ +export function computeJsonEntrySha( + itemSchema: Schema, + content: unknown, +): string { + const schemaHash = getJsonItemSchemaHash(itemSchema); + const contentHash = hash(JSON.stringify(content ?? null)); + return `${schemaHash}-${contentHash}`; +} + +/** + * Extracts the schema-hash component (the PREFIX) of a composite json entry sha, + * or `null` if the sha is not in composite form (e.g. a hand-authored sha). + */ +export function getSchemaHashFromJsonSha(sha: string): string | null { + const idx = sha.indexOf("-"); + return idx === -1 ? null : sha.slice(0, idx); +} + +/** + * Cheap, **content-free** staleness check: does this entry need revalidation + * purely because the schema changed? Compares the entry's stored sha PREFIX + * against the current item-schema hash — no `*.val.json` is read. Returns `true` + * when the schema changed under the entry (prefixes differ) or the stored sha + * has no schema prefix (can't prove it's current). + * + * This is what lets a schema change flag the entries needing revalidation by + * scanning only the shas already present in the `.val.ts`. + */ +export function jsonEntrySchemaHashIsStale( + storedSha: string, + itemSchema: Schema, +): boolean { + const prefix = getSchemaHashFromJsonSha(storedSha); + return prefix === null || prefix !== getJsonItemSchemaHash(itemSchema); +} + +/** + * True when `content` + `itemSchema` still match `storedSha`, i.e. no + * revalidation is needed. + */ +export function isJsonEntryShaCurrent( + storedSha: string, + itemSchema: Schema, + content: unknown, +): boolean { + return storedSha === computeJsonEntrySha(itemSchema, content); +} From 69b9324b89d72f2727d250f3ec40836e486d072e Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Sun, 28 Jun 2026 21:40:20 +0200 Subject: [PATCH 13/35] Add single-entry runtime read API for jsonValues (fetchValKey/useValKey) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fetchValKey (RSC) and useValKey (client) resolve a SINGLE .jsonValues() entry by key — loading only that entry's backing *.val.json via one dynamic import — instead of the eager fetchVal/useVal which load the whole record. This is the runtime-scaling counterpart for 10K+ entry routers/records. - fetchValKey: initFetchValKeyStega in next/src/rsc/initValRsc.ts - useValKey: useValKeyStega in next/src/client/initValClient.ts (promise cache + React.use / Suspense) Production path resolves the local module's import thunk (Internal.getJsonImport). Enabled/Studio draft path is a TODO (needs the single-entry fetch endpoint). Co-Authored-By: Claude Opus 4.8 --- docs/plans/jsonValues.md | 13 ++++- packages/next/src/client/initValClient.ts | Bin 5428 -> 7598 bytes packages/next/src/rsc/initValRsc.ts | 62 ++++++++++++++++++++++ 3 files changed, 73 insertions(+), 2 deletions(-) diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md index ae0de0980..dd4002cff 100644 --- a/docs/plans/jsonValues.md +++ b/docs/plans/jsonValues.md @@ -10,8 +10,17 @@ ## Current state / resume here -- **Phase**: 1 ✅ complete. Phase 2 in progress — loader ✅, `createValJsonReference` ✅; - **next: the ValOps commit/serialize flow (task below)**. +- **Phase**: 1 ✅. Phase 2 server mostly done (validation + sha + loader + emit primitive); Phase 4 + started — RSC `fetchValKey` implemented (production path). **Next: example app to validate the + runtime read path end-to-end, then the commit flow + single-entry endpoint.** +- **Single-entry runtime read API (typecheck-validated, runtime-validation via example pending)**: + - RSC `fetchValKey` — `initFetchValKeyStega` in `next/src/rsc/initValRsc.ts` (returned as + `fetchValKeyStega`). Resolves ONE entry by key from the local module's thunk + stega-encodes. + - Client `useValKey` — `useValKeyStega` in `next/src/client/initValClient.ts` (returned as + `useValKeyStega`). Uses a module-level promise cache + `React.use` (Suspense) to load one entry. + - Both: production path resolves the local thunk; Enabled/Studio **draft** path is a TODO (needs + the single-entry endpoint + a sub-selector for stega edit tags). Still TODO: make `fetchValRoute`/ + `useValRoute` use the key path for jsonValues routers (load one). - **Next step (the deep part)**: wire the json commit flow in `ValOps.ts`/`ValOpsFS`/`ValOpsHttp`: (a) shallow-serialize json records to `{ key: { _type:"json", _sha } }` markers for `/sources/~` (the runtime thunk must be dropped — `getJsonImport` reads it at runtime only); (b) on commit, diff --git a/packages/next/src/client/initValClient.ts b/packages/next/src/client/initValClient.ts index 7dcfcf6502d2b9a8f24f1d296b6e6f87b043c96d..bb7816ef5431f6c78b4cd648a94321a0d81a5800 100644 GIT binary patch delta 1741 zcmZ`(O^+Kz5JhkxJ4m@C7a~O}R#7rz{2^93Ag{+hn1B`%7G+l}MF?ryGc|T++uc@o z&o0Aa`34dPgfu690}=wkuRucl6MjJsRQJr-3kklByZhCv_g>ZSou3bXI0!yAN}i}f zr}%4M2eZKdP8SHFP|BdhEP_-?i_(HER|pIYmve!T^q!;4()IXX60;Zvtn4UsC1Fz>edn zzBO%c0Zd@pyZ{)2#0%6Fd+Y9o8hN6!LSRqOe*#jLQeP^kI=x)*`N9qwwpl}7D3+Nn z7zN2wWU1{J1{0X0HNXObGnUP$^OaUhZb*t$D`Ti-nV~iio@0p_%t~0gvKzX{>;m3x z!09yV2c)I3;FR;4d{#qEJ++v~U&0f%+A7i(WVQsW%CJ^$#7=~gOj6AKYIM=)y6qMVooHx;A9{}KuJ-8T`*gB@W8-~ zrRVM>>GUs{@Bv65R1zU8C0p_o_;RJRrAA@Xfs?|lynW#XXV*22fH-qO;P{gC+`_UT zjb_*=i5>E-9erKg#-=#W5gPOxllAXoaDR7{f$4lP$gsLNil zf>0vTl+bJzvz|dmP<1tsbKBN^7@aSjL68^HYXp>UN(u#~`s3!~_sIMqmG#3IP^&Lb z?&x$3Uz2wk=A2eT_5JIU8^*5=h^zIjw>R}32^$KU(h9C6vUADwIZb!$N40r#|GwWT zG?Fby9K*hvJtzMA+bs|_hs?l#7DhW3(dR_{3wp3|i(|MW=*kOpGvYUxJMn`A@xSlz z9Ks_vMUqm@jcYut2}coJc_wuibz)-JUZ&;XZXfyBhYy=at}cK>^$%K%Rtvw0NOhu- z+`Fp2yLER1lL_qauV1@&MXjup8lWcMuABJPYokvT!WPjB-qwDndATbO+uapYGDwUt z>Qh2w_$;KI6+t|9@$ZwptPpk!2LDspT&vt|p|`KrUWDqG8y`M!j2aK9=k0C{TY`1F s>*}YyJ8w4ksz3J*e)}>> = + T extends ValModule + ? S extends Record + ? V extends JsonSource + ? C + : never + : never + : never; + +/** + * Resolves a SINGLE entry of a `.jsonValues()` record/router by key, loading + * only that entry's backing `*.val.json` (one dynamic import) instead of the + * whole record. This is the runtime-scaling counterpart to the eager `fetchVal`. + * + * Production (Val disabled): resolves the entry's lazy import thunk from the + * local module and returns its content. + * + * TODO(enabled/Studio): when Val is enabled, draft edits to the entry live in + * patches on the server. Until the single-entry fetch endpoint is wired, this + * resolves the locally-bundled (committed) content. Visual-editing tags for the + * resolved entry are also a follow-up (needs a sub-selector at the entry path). + */ +const initFetchValKeyStega = + // NOTE: only needs `isEnabled` for the production read path. The + // enabled/Studio draft path (TODO) will also need the server deps the sibling + // init* factories take. + (isEnabled: () => Promise) => + async >>( + selector: T, + key: string, + ): Promise | undefined> => { + let enabled = false; + try { + enabled = await isEnabled(); + } catch { + // not in a server context where draftMode is readable — treat as disabled + } + const source = selector && Internal.getSource(selector); + if (!source || typeof source !== "object") { + return undefined; + } + const marker = (source as Record)[key]; + if (!Internal.isJson(marker)) { + return undefined; + } + const thunk = Internal.getJsonImport(marker); + if (!thunk) { + // transport marker without a runtime thunk — see TODO above + return undefined; + } + const content = (await thunk()).default; + return stegaEncode(content, { + disabled: !enabled, + }); + }; + const initFetchValRouteUrl = ( config: ValConfig, @@ -276,6 +334,7 @@ export function initValRsc( rscNextConfig: ValNextRscConfig, ): { fetchValStega: ReturnType; + fetchValKeyStega: ReturnType; fetchValRouteStega: ReturnType; fetchValRouteUrl: ReturnType; } { @@ -326,6 +385,9 @@ export function initValRsc( return await rscNextConfig.cookies(); }, ), + fetchValKeyStega: initFetchValKeyStega(async () => { + return (await rscNextConfig.draftMode()).isEnabled; + }), fetchValRouteStega: initFetchValRouteStega( config, valApiEndpoints, From 2b57c429314409632e5f85bbee689d9e475968cc Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Sun, 28 Jun 2026 20:36:08 +0000 Subject: [PATCH 14/35] Add example using jsonValues --- .../app/support/[slug]/content/faq.val.json | 5 ++++ .../[slug]/content/getting-started.val.json | 5 ++++ examples/next/app/support/[slug]/page.tsx | 22 ++++++++++++++ examples/next/app/support/[slug]/page.val.ts | 30 +++++++++++++++++++ examples/next/val.modules.ts | 1 + examples/next/val/rsc.ts | 3 +- 6 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 examples/next/app/support/[slug]/content/faq.val.json create mode 100644 examples/next/app/support/[slug]/content/getting-started.val.json create mode 100644 examples/next/app/support/[slug]/page.tsx create mode 100644 examples/next/app/support/[slug]/page.val.ts diff --git a/examples/next/app/support/[slug]/content/faq.val.json b/examples/next/app/support/[slug]/content/faq.val.json new file mode 100644 index 000000000..a929eb5ba --- /dev/null +++ b/examples/next/app/support/[slug]/content/faq.val.json @@ -0,0 +1,5 @@ +{ + "title": "FAQ", + "body": "Frequently asked questions about Val. Each support page is its own *.val.json so the router scales to many pages.", + "order": 2 +} diff --git a/examples/next/app/support/[slug]/content/getting-started.val.json b/examples/next/app/support/[slug]/content/getting-started.val.json new file mode 100644 index 000000000..5eda7f822 --- /dev/null +++ b/examples/next/app/support/[slug]/content/getting-started.val.json @@ -0,0 +1,5 @@ +{ + "title": "Getting started", + "body": "Welcome to Val support. This page is stored as a separate JSON file and loaded lazily via .jsonValues().", + "order": 1 +} diff --git a/examples/next/app/support/[slug]/page.tsx b/examples/next/app/support/[slug]/page.tsx new file mode 100644 index 000000000..60668a5cd --- /dev/null +++ b/examples/next/app/support/[slug]/page.tsx @@ -0,0 +1,22 @@ +import { fetchValKey } from "../../../val/rsc"; +import supportVal from "./page.val"; + +export default async function SupportPage({ + params, +}: { + params: Promise<{ slug: string }>; +}) { + const { slug } = await params; + // Loads ONLY this slug's backing *.val.json (one dynamic import), not the + // whole support-pages record. + const page = await fetchValKey(supportVal, `/support/${slug}`); + if (!page) { + return
Support page not found.
; + } + return ( +
+

{page.title}

+

{page.body}

+
+ ); +} diff --git a/examples/next/app/support/[slug]/page.val.ts b/examples/next/app/support/[slug]/page.val.ts new file mode 100644 index 000000000..006d222b0 --- /dev/null +++ b/examples/next/app/support/[slug]/page.val.ts @@ -0,0 +1,30 @@ +import { s, c, type t, nextAppRouter } from "../../../val.config"; + +/** + * Support pages use `.jsonValues()`: each page's content lives in its own + * `*.val.json` file and is loaded lazily via `c.json(() => import(...), sha)`. + * This keeps `page.val.ts` tiny even with thousands of support pages. + * + * NOTE: the trailing sha is the validation-cache key (`-`). + * These are placeholders for now — Val tooling (commit flow) will generate and + * maintain them; they are not checked on the runtime read path. + */ +export const supportPageSchema = s.object({ + title: s.string().minLength(2), + body: s.string(), + order: s.number(), +}); + +export type SupportPage = t.inferSchema; + +export default c.define( + "/app/support/[slug]/page.val.ts", + s.router(nextAppRouter, supportPageSchema).jsonValues(), + { + "/support/getting-started": c.json( + () => import("./content/getting-started.val.json"), + "v1", + ), + "/support/faq": c.json(() => import("./content/faq.val.json"), "v1"), + }, +); diff --git a/examples/next/val.modules.ts b/examples/next/val.modules.ts index 88a1873ba..6c5e8d1f1 100644 --- a/examples/next/val.modules.ts +++ b/examples/next/val.modules.ts @@ -4,6 +4,7 @@ import { config } from "./val.config"; export default modules(config, [ { def: () => import("./content/authors.val") }, { def: () => import("./app/blogs/[blog]/page.val") }, + { def: () => import("./app/support/[slug]/page.val") }, { def: () => import("./app/generic/[[...path]]/page.val") }, { def: () => import("./content/media.val") }, { def: () => import("./app/page.val") }, diff --git a/examples/next/val/rsc.ts b/examples/next/val/rsc.ts index 122752d11..0d2431e86 100644 --- a/examples/next/val/rsc.ts +++ b/examples/next/val/rsc.ts @@ -6,6 +6,7 @@ import { cookies, draftMode, headers } from "next/headers"; const { fetchValStega: fetchVal, + fetchValKeyStega: fetchValKey, fetchValRouteStega: fetchValRoute, fetchValRouteUrl, } = initValRsc(config, valModules, { @@ -14,4 +15,4 @@ const { cookies, }); -export { fetchVal, fetchValRoute, fetchValRouteUrl }; +export { fetchVal, fetchValKey, fetchValRoute, fetchValRouteUrl }; From 55a15e872de3aa734c1ebb98a92326ff4da854f7 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Sun, 28 Jun 2026 20:36:37 +0000 Subject: [PATCH 15/35] Add back project --- examples/next/val.config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/next/val.config.ts b/examples/next/val.config.ts index ed2762cdb..cfde56f86 100644 --- a/examples/next/val.config.ts +++ b/examples/next/val.config.ts @@ -1,7 +1,7 @@ import { initVal } from "@valbuild/next"; const { s, c, val, config, nextAppRouter, externalPageRouter } = initVal({ - // project: "valbuild/val-examples-next", + project: "valbuild/val-examples-next", root: "/examples/next", defaultTheme: "dark", ai: { From cd37b0ab14cc7b76de373a9dfcd9592adad23e7d Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Mon, 29 Jun 2026 16:05:24 +0200 Subject: [PATCH 16/35] Add /json single-entry endpoint + jsonValues example (support pages) - /json endpoint (ApiRoutes contract + ValServer handler): loads the committed content of one .jsonValues() entry by key (getBaseSources -> marker -> getJsonImport -> thunk). Generic ValRouter dispatch routes it automatically. Unblocks the Studio lazy-load and the enabled/draft runtime path. - examples/next: app/support/[slug] uses s.router(...).jsonValues() with two *.val.json entries; page.tsx renders one via fetchValKey. Registered in val.modules.ts; fetchValKey exported from val/rsc.ts. Example typechecks clean. Note: the Studio still cannot edit jsonValues entries until the UI lazy-load consumes /json (resolvePath guard correctly surfaces "content not loaded"). Co-Authored-By: Claude Opus 4.8 --- docs/plans/jsonValues.md | 8 +++ packages/server/src/ValServer.ts | 67 +++++++++++++++++++++++ packages/shared/src/internal/ApiRoutes.ts | 32 +++++++++++ 3 files changed, 107 insertions(+) diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md index dd4002cff..af461007f 100644 --- a/docs/plans/jsonValues.md +++ b/docs/plans/jsonValues.md @@ -10,6 +10,14 @@ ## Current state / resume here +> **Expected-behavior note (verified in the example, 2026-06-29):** opening a jsonValues entry in +> the **Studio** (`/val/~/.../page.val.ts?p="/support/getting-started"`) throws +> `Cannot resolve path into a jsonValues entry until its content is loaded` — this is the intentional +> `resolvePath` guard (module.ts) firing because the Studio lazy-load of entry content isn't built +> yet (UI task + single-entry endpoint, both pending). The **production read** path +> (`/support/getting-started` rendered via `fetchValKey`) is wired. So: reading works; Studio editing +> of jsonValues entries is the next milestone (endpoint → UI). + - **Phase**: 1 ✅. Phase 2 server mostly done (validation + sha + loader + emit primitive); Phase 4 started — RSC `fetchValKey` implemented (production path). **Next: example app to validate the runtime read path end-to-end, then the commit flow + single-entry endpoint.** diff --git a/packages/server/src/ValServer.ts b/packages/server/src/ValServer.ts index 4e86832ca..20d8b9e13 100644 --- a/packages/server/src/ValServer.ts +++ b/packages/server/src/ValServer.ts @@ -1361,6 +1361,73 @@ export const ValServer = ( }, }, + // #region json + // Loads the content of a single `.jsonValues()` entry by key, so the Studio + // can lazily load just the entry being opened. Returns the committed content + // (from the base source's import thunk); the client overlays its own patches. + "/json": { + GET: async (req) => { + const auth = getAuth(req.cookies); + if (auth.error) { + return { status: 401, json: { message: auth.error } }; + } + if (serverOps instanceof ValOpsHttp && !("id" in auth)) { + return { status: 401, json: { message: "Unauthorized" } }; + } + const moduleFilePath = req.query.path as ModuleFilePath; + const key = req.query.key; + const sources = await serverOps.getBaseSources(); + const moduleSource = sources[moduleFilePath]; + if (moduleSource === undefined || moduleSource === null) { + return { + status: 404, + json: { message: `Module not found: ${moduleFilePath}` }, + }; + } + if (typeof moduleSource !== "object" || Array.isArray(moduleSource)) { + return { + status: 404, + json: { message: `Module is not a record: ${moduleFilePath}` }, + }; + } + const marker = (moduleSource as Record)[key]; + if (marker === undefined) { + return { + status: 404, + json: { message: `Entry not found: ${key} in ${moduleFilePath}` }, + }; + } + if (!Internal.isJson(marker)) { + // Not a jsonValues entry — return the inlined value as-is (defensive). + return { + status: 200, + json: { path: moduleFilePath, key, content: marker }, + }; + } + const thunk = Internal.getJsonImport(marker); + let content: unknown = null; + if (thunk) { + try { + content = (await thunk()).default; + } catch (e) { + return { + status: 500, + json: { + message: `Failed to load JSON entry '${key}': ${ + e instanceof Error ? e.message : JSON.stringify(e) + }`, + }, + }; + } + } + const sha = (marker as { _sha?: string })._sha; + return { + status: 200, + json: { path: moduleFilePath, key, content, sha }, + }; + }, + }, + // #region sources "/sources/~": { PUT: async (req) => { diff --git a/packages/shared/src/internal/ApiRoutes.ts b/packages/shared/src/internal/ApiRoutes.ts index 7d75f5591..459f068a7 100644 --- a/packages/shared/src/internal/ApiRoutes.ts +++ b/packages/shared/src/internal/ApiRoutes.ts @@ -1010,6 +1010,38 @@ export const Api = { ]), }, }, + // Loads the content of a single `.jsonValues()` record/router entry by key, + // so the Studio can lazily load just the entry being opened (instead of the + // whole record). Returns the committed entry content; client overlays patches. + "/json": { + GET: { + req: { + query: { + path: onlyOneStringQueryParam, + key: onlyOneStringQueryParam, + }, + cookies: { [VAL_SESSION_COOKIE]: z.string().optional() }, + }, + res: z.union([ + unauthorizedResponse, + notFoundResponse, + z.object({ + status: z.literal(200), + json: z.object({ + path: ModuleFilePath, + key: z.string(), + // The entry's JSON content (or null if the entry has no value). + content: z.any(), + sha: z.string().optional(), + }), + }), + z.object({ + status: z.literal(500), + json: GenericError, + }), + ]), + }, + }, "/ai/initialize": { POST: { req: { From 5eef908305a0ce711a30064072bcfa0b6e5888ec Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Mon, 29 Jun 2026 17:17:57 +0200 Subject: [PATCH 17/35] Record Studio UI lazy-load plan + expected resolvePath-guard note Co-Authored-By: Claude Opus 4.8 --- docs/plans/jsonValues.md | 56 ++++++++++++++++++++++++++++------------ 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md index af461007f..efbfeaa3c 100644 --- a/docs/plans/jsonValues.md +++ b/docs/plans/jsonValues.md @@ -18,9 +18,11 @@ > (`/support/getting-started` rendered via `fetchValKey`) is wired. So: reading works; Studio editing > of jsonValues entries is the next milestone (endpoint → UI). -- **Phase**: 1 ✅. Phase 2 server mostly done (validation + sha + loader + emit primitive); Phase 4 - started — RSC `fetchValKey` implemented (production path). **Next: example app to validate the - runtime read path end-to-end, then the commit flow + single-entry endpoint.** +- **Phase**: 1 ✅. Phase 2 server: validation + sha + loader + emit primitive + **/json endpoint** ✅ + (commit flow still pending). Phase 4: `fetchValKey`/`useValKey` ✅ (production path). Example + (support pages) added + typechecks. **NEXT MILESTONE: Phase 3 UI lazy-load** (consume `/json` so + the Studio can open/edit jsonValues entries — see Phase 3 below for exact integration points). The + other remaining big piece is the **commit flow** (write `*.val.json` + thunks/shas on commit). - **Single-entry runtime read API (typecheck-validated, runtime-validation via example pending)**: - RSC `fetchValKey` — `initFetchValKeyStega` in `next/src/rsc/initValRsc.ts` (returned as `fetchValKeyStega`). Resolves ONE entry by key from the local module's thunk + stega-encodes. @@ -150,23 +152,43 @@ entries; runtime/Studio/validation work one entry at a time; zero overhead when - [ ] **Verify**: `pnpm test packages/server/...` green (ops add/replace/remove, loader fixture, incremental validation). -## Phase 3 — UI (`packages/ui/spa`) - -- [ ] `ValSyncEngine.ts`: model json records as `{ key → { sha, patch_id? } }`; lazy fetch + cache - one entry on open; per-entry patches; sha-aware invalidation. -- [ ] `components/ValFieldProvider.tsx`: reuse `useShallowSourceAtPath` for the key list; add a - hook to lazily fetch one entry's content (`useJsonEntrySource` / extend `useSourceAtPath`). -- [ ] `components/fields/RecordFields.tsx` + router nav: render keys without loading content; load - on open; build per-entry add/replace/remove patches. -- [ ] **Verify**: manual Studio pass (list shows keys w/o loading; open fetches one json; - edit→commit writes file + updates sha; add/remove a route inserts/removes thunk + file). +## Phase 3 — UI (`packages/ui/spa`) — NEXT MILESTONE (do with Studio running) + +The Studio currently throws the (intentional) `resolvePath` guard when opening a jsonValues entry, +because entry content is never loaded into the client source tree. Concrete integration points +(all in the 3376-line `ValSyncEngine.ts` unless noted): + +- [ ] **Content cache + fetch**: add `private jsonEntryContents: Record>` and `private loadingJsonEntries: Set<"mfp\0key">`. Add `requestJsonEntry(mfp, key)` + that, if not loaded/loading, calls `this.client("/json", "GET", { query: { path: mfp, key } })`, + stores `content`, then `invalidatePatchedSourcesCache(mfp)` + clears `cachedSourceSnapshots` + for the module + `emit(this.listeners["source"]?.[mfp])` / `["sources"]` so subscribers + re-render. (Mirror the existing `requestModuleValidation` side-effect pattern.) +- [ ] **Substitution before patches**: in `getPatchedSource(mfp)`, build the effective base by + replacing each loaded json marker at `baseSource[key]` with `jsonEntryContents[mfp][key]` + BEFORE applying patches (so field-level patches at `?p="key"."field"` apply on top). Markers + without loaded content stay as-is (list view only reads keys). Invalidate the patched cache on + load (above) since the effective base changed. +- [ ] **Trigger on open**: the field component that renders a navigated-to entry path must call + `requestJsonEntry(mfp, key)` (effect on mount, keyed by path). Find the entry detail renderer + (AnyField/Field at a path); when the path's parent schema is a `jsonValues` record and the + marker isn't loaded, request it and render a loading state until `getSourceSnapshot` returns + content. `useShallowSourceAtPath(path, "record")` already returns keys only (list is fine). +- [ ] **Per-entry patches**: editing an entry field produces a normal patch at the entry path; on + commit the server commit-flow writes the `*.val.json` + updates the thunk/sha (Phase 2 commit + flow). Add/remove entry = add/remove the record key (commit flow emits/removes thunk + file). +- [ ] **Verify** (Studio running): list shows keys w/o loading; opening an entry fetches one `/json`; + fields render + edit; commit writes file + updates sha; add/remove a route inserts/removes + thunk + file. ## Phase 4 — Runtime APIs (`packages/next`, `packages/react`) -- [ ] `rsc/initValRsc.ts`: `fetchValKey` / `fetchValRoute` (route matching reuses `ValRouter`). -- [ ] `client/initValClient.ts`: `useValKey` / `useValRoute`. -- [ ] Apply existing stega/transform per resolved entry. -- [ ] **Verify**: `fetchVal` still returns all; `fetchValRoute` imports only the requested entry. +- [x] `rsc/initValRsc.ts`: `fetchValKey` (`initFetchValKeyStega`, returned as `fetchValKeyStega`). +- [x] `client/initValClient.ts`: `useValKey` (`useValKeyStega`, promise cache + `React.use`). +- [x] Example wires `fetchValKey` (support pages) — typechecks clean. +- [ ] `fetchValRoute` / `useValRoute`: use the key path for jsonValues routers (load one entry by + matching the route), instead of the eager fetchVal. (Reuse `ValRouter` to map route→key.) +- [ ] Enabled/Studio draft path: resolve draft content via `/json` (+ sub-selector stega tags). ## Phase 5 — Example + CI gate From 75909371167e41e2b128f519c9ad6d3333794529 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Mon, 29 Jun 2026 19:16:37 +0000 Subject: [PATCH 18/35] Studio: lazy-load jsonValues entry content (consume /json) When the Studio opens a path that descends into a .jsonValues() entry whose content isn't loaded, fetch it via GET /json and fold it into the source view: - ValSyncEngine: jsonEntryContents cache + requestJsonEntry(mfp, key) (GET /json, then invalidateSource to re-render); applyJsonEntryContents substitutes loaded content for the marker in getPatchedSource (before patches apply, so field edits layer on top). Caches reset on engine reset. - ValFieldProvider: findUnloadedJsonEntryKey detects a path descending into an un-loaded marker; useSourceAtPath + useSchemaAtPathInternal trigger requestJsonEntry and render "loading" until content resolves (then the entry's fields render normally). This resolves the intentional resolvePath "content not loaded" guard in the Studio. Editing shows optimistic updates; persistence to *.val.json needs the commit flow (pending). UI typecheck + ValSyncEngine tests green. Co-Authored-By: Claude Opus 4.8 --- packages/ui/spa/ValSyncEngine.ts | 97 ++++++++++++++++++- .../ui/spa/components/ValFieldProvider.tsx | 90 +++++++++++++++-- 2 files changed, 178 insertions(+), 9 deletions(-) diff --git a/packages/ui/spa/ValSyncEngine.ts b/packages/ui/spa/ValSyncEngine.ts index f2d6d9229..2e0578860 100644 --- a/packages/ui/spa/ValSyncEngine.ts +++ b/packages/ui/spa/ValSyncEngine.ts @@ -135,6 +135,17 @@ export class ValSyncEngine { } | undefined > | null; + /** + * Loaded content for `.jsonValues()` record entries, keyed by module then + * entry key. The on-disk source only carries lazy `{ _type:"json", _sha }` + * markers; the Studio fetches an entry's content on demand via + * `requestJsonEntry` (GET /json) and `getPatchedSource` substitutes it in so + * field resolution/rendering works. + */ + private jsonEntryContents: Record> = + {}; + /** In-flight json entry loads, keyed `${moduleFilePath}\0${key}`. */ + private loadingJsonEntries: Set = new Set(); private renders: Record | null; private schemas: Record | null; private serverSideSchemaSha: string | null; @@ -386,6 +397,8 @@ export class ValSyncEngine { this.sourcesSha = null; this.serverSources = null; this.patchedSourcesCache = null; + this.jsonEntryContents = {}; + this.loadingJsonEntries = new Set(); this.renders = null; this.globalServerSidePatchIds = []; this.syncedServerSidePatchIds = []; @@ -816,11 +829,91 @@ export class ValSyncEngine { * Otherwise we rebuild from `serverSources`, which covers patch deletion, * server-side reorder, and any other non-append change. */ + /** + * Lazily loads the content of a single `.jsonValues()` entry (GET /json) and + * folds it into the source view (re-rendering subscribers). No-op if the entry + * is already loaded or a load is in flight. Called by the field hooks when the + * Studio opens a path that descends into an unloaded json marker. + */ + requestJsonEntry(moduleFilePath: ModuleFilePath, key: string): void { + if (this.jsonEntryContents[moduleFilePath]?.[key] !== undefined) { + return; + } + const loadingKey = `${moduleFilePath}\0${key}`; + if (this.loadingJsonEntries.has(loadingKey)) { + return; + } + this.loadingJsonEntries.add(loadingKey); + this.client("/json", "GET", { + query: { path: moduleFilePath, key }, + }) + .then((res) => { + if (res.status === 200) { + if (this.jsonEntryContents[moduleFilePath] === undefined) { + this.jsonEntryContents[moduleFilePath] = {}; + } + this.jsonEntryContents[moduleFilePath][key] = + res.json.content ?? null; + this.invalidateSource(moduleFilePath); + } else { + console.error("Val: SyncEngine: failed to load json entry", { + moduleFilePath, + key, + res, + }); + } + }) + .catch((err) => { + console.error("Val: SyncEngine: error loading json entry", { + moduleFilePath, + key, + err, + }); + }) + .finally(() => { + this.loadingJsonEntries.delete(loadingKey); + }); + } + + /** + * Returns `baseSource` with any loaded `.jsonValues()` entry content + * substituted in place of its lazy marker, so downstream resolution/patching + * sees real content. Markers without loaded content are left untouched. + */ + private applyJsonEntryContents( + moduleFilePath: ModuleFilePath, + baseSource: JSONValue, + ): JSONValue { + const contents = this.jsonEntryContents[moduleFilePath]; + if ( + contents === undefined || + baseSource === null || + typeof baseSource !== "object" || + Array.isArray(baseSource) + ) { + return baseSource; + } + let result: Record | null = null; + for (const key in contents) { + if (Internal.isJson(baseSource[key])) { + if (result === null) { + result = { ...baseSource }; + } + result[key] = contents[key]; + } + } + return result ?? baseSource; + } + private getPatchedSource( moduleFilePath: ModuleFilePath, ): JSONValue | undefined { - const baseSource = this.serverSources?.[moduleFilePath]; - if (baseSource === undefined) return undefined; + const rawBaseSource = this.serverSources?.[moduleFilePath]; + if (rawBaseSource === undefined) return undefined; + const baseSource = this.applyJsonEntryContents( + moduleFilePath, + rawBaseSource, + ); const nextIds = this.orderedPatchIdsForModule(moduleFilePath); if (nextIds.length === 0) return baseSource; diff --git a/packages/ui/spa/components/ValFieldProvider.tsx b/packages/ui/spa/components/ValFieldProvider.tsx index 0264aef7b..22c41db54 100644 --- a/packages/ui/spa/components/ValFieldProvider.tsx +++ b/packages/ui/spa/components/ValFieldProvider.tsx @@ -533,16 +533,33 @@ function useSchemaAtPathInternal( () => syncEngine.getSourceSnapshot(moduleFilePath), () => syncEngine.getSourceSnapshot(moduleFilePath), ); - const resolvedSchemaAtPathRes = useMemo(() => { - if (schemaRes.status !== "success") { - return schemaRes; - } - const sourceData = + const sourceData = useMemo( + () => sourceOverride && sourceOverride.moduleFilePath === moduleFilePath ? sourceOverride.moduleSource : sourcesRes.status === "success" ? sourcesRes.data - : undefined; + : undefined, + [sourceOverride, moduleFilePath, sourcesRes], + ); + // Lazily load `.jsonValues()` entry content when the path descends into an + // un-loaded marker, and treat the schema as loading until it resolves. + const unloadedJsonKey = useMemo( + () => findUnloadedJsonEntryKey(modulePath, sourceData), + [modulePath, sourceData], + ); + useEffect(() => { + if (unloadedJsonKey !== null) { + syncEngine.requestJsonEntry(moduleFilePath, unloadedJsonKey); + } + }, [syncEngine, moduleFilePath, unloadedJsonKey]); + const resolvedSchemaAtPathRes = useMemo(() => { + if (schemaRes.status !== "success") { + return schemaRes; + } + if (unloadedJsonKey !== null) { + return { status: "loading" as const }; + } if (sourceData === undefined) { if (sourcesRes.status !== "success") { return sourcesRes; @@ -601,7 +618,14 @@ function useSchemaAtPathInternal( }`, }; } - }, [schemaRes, sourcesRes, moduleFilePath, modulePath, sourceOverride]); + }, [ + schemaRes, + sourcesRes, + moduleFilePath, + modulePath, + sourceData, + unloadedJsonKey, + ]); const initializedAt = useSyncEngineInitializedAt(syncEngine); if (initializedAt === null) { return { status: "loading" }; @@ -715,6 +739,40 @@ export function useAllRenders() { return renders; } +/** + * Walks `modulePath` against `sourceData` and returns the record key at which + * the path descends into a `.jsonValues()` entry whose content has NOT been + * loaded yet (the value is still a lazy json marker), or `null` otherwise. + * + * The sync engine substitutes loaded entry content in place of the marker, so a + * marker still present here means the entry isn't loaded — the caller should + * trigger `requestJsonEntry` and render a loading state until it resolves. + */ +function findUnloadedJsonEntryKey( + modulePath: ModulePath, + sourceData: Json | undefined, +): string | null { + if (sourceData === undefined) { + return null; + } + let current: Json = sourceData; + for (const part of Internal.splitModulePath(modulePath)) { + if ( + current === null || + typeof current !== "object" || + isJsonArray(current) + ) { + return null; + } + const next: Json = current[part]; + if (Internal.isJson(next)) { + return part; + } + current = next; + } + return null; +} + function walkSourcePath( modulePath: ModulePath, sources?: Json, @@ -1161,6 +1219,20 @@ export function useSourceAtPath( syncEngine ? () => syncEngine.getInitializedAtSnapshot() : getNull, syncEngine ? () => syncEngine.getInitializedAtSnapshot() : getNull, ); + // A `.jsonValues()` entry's content is loaded lazily: if this path descends + // into an un-loaded marker, request it and render loading until it resolves. + const unloadedJsonKey = useMemo( + () => + sourceSnapshot && sourceSnapshot.status === "success" + ? findUnloadedJsonEntryKey(modulePath, sourceSnapshot.data) + : null, + [modulePath, sourceSnapshot], + ); + useEffect(() => { + if (syncEngine && unloadedJsonKey !== null) { + syncEngine.requestJsonEntry(moduleFilePath, unloadedJsonKey); + } + }, [syncEngine, moduleFilePath, unloadedJsonKey]); return useMemo(() => { if (!syncEngine) { return NOT_FOUND; @@ -1171,6 +1243,9 @@ export function useSourceAtPath( if (sourceOverride && sourceOverride.moduleFilePath === moduleFilePath) { return walkSourcePath(modulePath, sourceOverride.moduleSource); } + if (unloadedJsonKey !== null) { + return { status: "loading" }; + } if (sourceSnapshot && sourceSnapshot.status === "success") { return walkSourcePath(modulePath, sourceSnapshot.data); } @@ -1185,6 +1260,7 @@ export function useSourceAtPath( modulePath, moduleFilePath, sourceOverride, + unloadedJsonKey, ]); } From cf7b0ce44ecce4133c85408be650bea37abe1871 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Mon, 29 Jun 2026 19:16:49 +0000 Subject: [PATCH 19/35] Tracker: Studio lazy-load done; editing needs commit flow Co-Authored-By: Claude Opus 4.8 --- docs/plans/jsonValues.md | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md index efbfeaa3c..51bd59f13 100644 --- a/docs/plans/jsonValues.md +++ b/docs/plans/jsonValues.md @@ -10,13 +10,12 @@ ## Current state / resume here -> **Expected-behavior note (verified in the example, 2026-06-29):** opening a jsonValues entry in -> the **Studio** (`/val/~/.../page.val.ts?p="/support/getting-started"`) throws -> `Cannot resolve path into a jsonValues entry until its content is loaded` — this is the intentional -> `resolvePath` guard (module.ts) firing because the Studio lazy-load of entry content isn't built -> yet (UI task + single-entry endpoint, both pending). The **production read** path -> (`/support/getting-started` rendered via `fetchValKey`) is wired. So: reading works; Studio editing -> of jsonValues entries is the next milestone (endpoint → UI). +> **Studio lazy-load DONE (2026-06-30):** opening a jsonValues entry now fetches its content via +> `GET /json` and renders the fields (was: the `resolvePath` guard error). Read path works in both +> production (`fetchValKey`) and the Studio. **Editing** shows optimistic updates, but **persistence +> to `*.val.json` needs the commit flow (still pending)** — so publishing an edit won't write the +> json file yet. Requires `pnpm --filter @valbuild/ui build` for the Studio bundle to pick up UI +> changes (it's a built bundle, not a live dev-stub). - **Phase**: 1 ✅. Phase 2 server: validation + sha + loader + emit primitive + **/json endpoint** ✅ (commit flow still pending). Phase 4: `fetchValKey`/`useValKey` ✅ (production path). Example @@ -159,7 +158,7 @@ because entry content is never loaded into the client source tree. Concrete inte (all in the 3376-line `ValSyncEngine.ts` unless noted): - [ ] **Content cache + fetch**: add `private jsonEntryContents: Record>` and `private loadingJsonEntries: Set<"mfp\0key">`. Add `requestJsonEntry(mfp, key)` + JSONValue>>` and `private loadingJsonEntries: Set<"mfp\0key">`. Add `requestJsonEntry(mfp, key)` that, if not loaded/loading, calls `this.client("/json", "GET", { query: { path: mfp, key } })`, stores `content`, then `invalidatePatchedSourcesCache(mfp)` + clears `cachedSourceSnapshots` for the module + `emit(this.listeners["source"]?.[mfp])` / `["sources"]` so subscribers From 32f02920e3527552b405cfb4d3af2bf507a7edbe Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Mon, 29 Jun 2026 23:14:25 +0000 Subject: [PATCH 20/35] ix jsonValues markers leaking into Studio source/schema walkers Opening a .jsonValues() entry in the Studio surfaced two errors caused by lazy json markers ({ _type:"json", _sha }) reaching code that expected real content: - "Expected a c.json(...) entry, got 'object'": once an entry's *.val.json is loaded, the UI substitutes its content for the marker, but RecordSchema's executeValidate rejected anything that wasn't a marker. Now it validates inline/loaded content against the item schema and only DEFERS actual markers (so loaded entries validate live; unloaded markers are skipped, still validated server-side once loaded). - "Schema not found for ...entry._type/._sha": source+schema walkers descended into an un-loaded marker and treated _type/_sha as object fields. Skip json markers (opaque until loaded) in traverseSchemas, getRouteReferences, and createSearchIndex (search still indexes the entry's path, not its content). Core + UI typecheck; core jsonValues + server validation + ValSyncEngine tests pass. Co-Authored-By: Claude Opus 4.8 --- packages/core/src/schema/jsonValues.test.ts | 21 ++++++++---- packages/core/src/schema/record.ts | 32 +++++++++++-------- .../ui/spa/components/getRouteReferences.ts | 5 +++ packages/ui/spa/components/traverseSchemas.ts | 7 ++++ packages/ui/spa/search/createSearchIndex.ts | 5 +++ 5 files changed, 49 insertions(+), 21 deletions(-) diff --git a/packages/core/src/schema/jsonValues.test.ts b/packages/core/src/schema/jsonValues.test.ts index fd5287d5c..82de7c981 100644 --- a/packages/core/src/schema/jsonValues.test.ts +++ b/packages/core/src/schema/jsonValues.test.ts @@ -95,13 +95,20 @@ describe("c.json + .jsonValues()", () => { expect(loaded).toBe(false); }); - test("rejects a non-json entry value", () => { - const errors = looseSchema["executeValidate"]( - "/test.val.ts" as SourcePath, - // deliberately an inline value, which is invalid for a jsonValues record - { a: { title: "inline-not-allowed" } }, - ); - expect(errors).not.toBe(false); + test("validates inline (loaded) content against the item schema", () => { + // valid inline content passes (this is what the UI sees once an entry's + // *.val.json is loaded and substituted for its marker) + expect( + looseSchema["executeValidate"]("/test.val.ts" as SourcePath, { + a: { title: "ok" }, + }), + ).toBe(false); + // invalid inline content (wrong leaf type) is caught + expect( + looseSchema["executeValidate"]("/test.val.ts" as SourcePath, { + a: { title: 123 }, + }), + ).not.toBe(false); }); test("validateJsonEntryContent validates loaded content against item", () => { diff --git a/packages/core/src/schema/record.ts b/packages/core/src/schema/record.ts index 63170021d..16e9b9f1a 100644 --- a/packages/core/src/schema/record.ts +++ b/packages/core/src/schema/record.ts @@ -259,21 +259,25 @@ export class RecordSchema< this.markKeyErrorsAtPath(entryErr, subPath); error = error ? { ...error, ...entryErr } : entryErr; } + } else if (this.isJsonValues && isJson(elem)) { + // jsonValues record, entry not loaded: the value is a lazy JsonSource + // marker. Deep validation is deferred and run per-entry once the backing + // `*.val.json` is loaded (server: validateJsonEntryContent; UI: the + // loaded content is substituted and validated by the branch below). } else if (this.isJsonValues) { - // jsonValues record: the value is a lazily-loaded JsonSource marker. - // Only assert the marker shape here; the deep content validation is - // deferred and run per-entry by the server (validateJsonEntryContent) - // once the backing `*.val.json` is loaded. - if (!isJson(elem)) { - error = this.appendValidationError( - error, - subPath, - `Expected a c.json(...) entry, got '${ - elem === null ? "null" : typeof elem - }'`, - elem, - true, - ); + // jsonValues record, entry content is inlined (loaded in the UI, or + // hand-authored content) — validate it against the item schema. + const subError = this.item["executeValidate"]( + subPath, + elem as SelectorSource, + ); + if (subError && error) { + error = { + ...subError, + ...error, + }; + } else if (subError) { + error = subError; } } else { const subError = this.item["executeValidate"]( diff --git a/packages/ui/spa/components/getRouteReferences.ts b/packages/ui/spa/components/getRouteReferences.ts index 16a3db334..34f671e74 100644 --- a/packages/ui/spa/components/getRouteReferences.ts +++ b/packages/ui/spa/components/getRouteReferences.ts @@ -1,4 +1,5 @@ import { + Internal, ModuleFilePath, ModuleFilePathSep, Source, @@ -27,6 +28,10 @@ export function getRouteReferences( if (schema === undefined) { return; } + if (Internal.isJson(source)) { + // Un-loaded `.jsonValues()` entry marker — opaque until loaded. + return; + } if (schema.type === "route") { // Check if the source value matches the route key we're looking for if (typeof source === "string" && source === routeKey) { diff --git a/packages/ui/spa/components/traverseSchemas.ts b/packages/ui/spa/components/traverseSchemas.ts index 2d1a5439d..61568d428 100644 --- a/packages/ui/spa/components/traverseSchemas.ts +++ b/packages/ui/spa/components/traverseSchemas.ts @@ -1,4 +1,5 @@ import { + Internal, ModuleFilePath, ModuleFilePathSep, SerializedArraySchema, @@ -34,6 +35,12 @@ export function traverseSchemas( schema: SerializedSchema | undefined, source: Source, ) => { + if (Internal.isJson(source)) { + // An un-loaded `.jsonValues()` entry is an opaque lazy marker + // ({ _type:"json", _sha }). Skip it: once its content is loaded the + // substituted value is the real content and is traversed normally. + return; + } if (schema === undefined) { console.error(`Schema not found for ${sourcePath}`); return; diff --git a/packages/ui/spa/search/createSearchIndex.ts b/packages/ui/spa/search/createSearchIndex.ts index 6797b37d3..76e9a2b13 100644 --- a/packages/ui/spa/search/createSearchIndex.ts +++ b/packages/ui/spa/search/createSearchIndex.ts @@ -22,6 +22,11 @@ function rec( ) { addTokenizedSourcePath(sourcePathIndex, path); } + if (Internal.isJson(source)) { + // Un-loaded `.jsonValues()` entry marker — its content isn't available to + // index yet (the entry path was indexed above). Indexed once loaded. + return; + } if (!schema?.type) { throw new Error("Schema not found for " + path); } else if (source === null) { From 25f6beb191255680519fba871633562cd4145637 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Thu, 2 Jul 2026 21:27:42 +0000 Subject: [PATCH 21/35] Remove c.json sha parameter...\n... Wanted to have a good way where we could skip validation of json values, however, the sha param would only tell us when we HAD to re-validate, not when we could skip validation --- docs/plans/jsonValues.md | 142 +++++++++--------- examples/next/app/support/[slug]/page.val.ts | 9 +- packages/core/src/schema/jsonValues.test.ts | 37 ++--- packages/core/src/schema/record.ts | 2 +- packages/core/src/source/json.ts | 12 +- packages/server/src/ValServer.ts | 3 +- packages/server/src/jsonValuesSha.test.ts | 64 -------- packages/server/src/jsonValuesSha.ts | 79 ---------- .../src/loadValModules.jsonValues.test.ts | 2 - .../server/src/patch/ts/jsonReference.test.ts | 7 +- .../src/patch/ts/jsonValuesModule.test.ts | 63 ++++++++ .../server/src/patch/ts/jsonValuesModule.ts | 109 ++++++++++++++ packages/server/src/patch/ts/ops.ts | 13 +- .../server/src/validateJsonValues.test.ts | 24 +-- .../test/jsonValues-fixture/blogs.val.ts | 5 +- packages/shared/src/internal/ApiRoutes.ts | 1 - packages/ui/spa/ValSyncEngine.ts | 2 +- packages/ui/spa/components/traverseSchemas.ts | 2 +- 18 files changed, 283 insertions(+), 293 deletions(-) delete mode 100644 packages/server/src/jsonValuesSha.test.ts delete mode 100644 packages/server/src/jsonValuesSha.ts create mode 100644 packages/server/src/patch/ts/jsonValuesModule.test.ts create mode 100644 packages/server/src/patch/ts/jsonValuesModule.ts diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md index 51bd59f13..69a616573 100644 --- a/docs/plans/jsonValues.md +++ b/docs/plans/jsonValues.md @@ -17,11 +17,11 @@ > json file yet. Requires `pnpm --filter @valbuild/ui build` for the Studio bundle to pick up UI > changes (it's a built bundle, not a live dev-stub). -- **Phase**: 1 ✅. Phase 2 server: validation + sha + loader + emit primitive + **/json endpoint** ✅ - (commit flow still pending). Phase 4: `fetchValKey`/`useValKey` ✅ (production path). Example - (support pages) added + typechecks. **NEXT MILESTONE: Phase 3 UI lazy-load** (consume `/json` so - the Studio can open/edit jsonValues entries — see Phase 3 below for exact integration points). The - other remaining big piece is the **commit flow** (write `*.val.json` + thunks/shas on commit). +- **Phase**: 1 ✅. Phase 2 server: validation + loader + emit primitive + **/json endpoint** ✅ + (commit flow still pending). Phase 3 UI lazy-load ✅ (Studio reads jsonValues entries). Phase 4: + `fetchValKey`/`useValKey` ✅ (production path). Example (support pages) added + typechecks. + **The `c.json` sha was removed (2026-07-02).** Remaining big piece: the **commit flow** (write + `*.val.json` for content edits; insert/remove `c.json` thunks for add/remove). - **Single-entry runtime read API (typecheck-validated, runtime-validation via example pending)**: - RSC `fetchValKey` — `initFetchValKeyStega` in `next/src/rsc/initValRsc.ts` (returned as `fetchValKeyStega`). Resolves ONE entry by key from the local module's thunk + stega-encodes. @@ -30,14 +30,38 @@ - Both: production path resolves the local thunk; Enabled/Studio **draft** path is a TODO (needs the single-entry endpoint + a sub-selector for stega edit tags). Still TODO: make `fetchValRoute`/ `useValRoute` use the key path for jsonValues routers (load one). -- **Next step (the deep part)**: wire the json commit flow in `ValOps.ts`/`ValOpsFS`/`ValOpsHttp`: - (a) shallow-serialize json records to `{ key: { _type:"json", _sha } }` markers for `/sources/~` - (the runtime thunk must be dropped — `getJsonImport` reads it at runtime only); (b) on commit, - write/replace/delete `*.val.json` files + emit/update the `c.json(...)` thunk in the `.val.ts` - using `createValJsonReference` (already in `patch/ts/ops.ts`) via `insertAt`/`removeAt`/ - `replaceNodeValue`; (c) per-entry content load (invoke `getJsonImport(marker)?.()` or read the - file by AST-derived path) + sha-keyed incremental validation calling - `RecordSchema.validateJsonEntryContent`. Then the single-entry fetch endpoint in `ValServer.ts`. +- **Next step (the commit flow — IN PROGRESS)**: persist edits to `*.val.json` instead of the + `.val.ts`. Design (now fully mapped): + - **Key enabler**: `ValOps.prepare`'s `patchedSourceFiles: Record` is written by + the commit loop to **arbitrary paths** relative to rootDir (`null` = delete). So `*.val.json` + writes/deletes go through the same map (no new abstract FS method needed). + - **AST analyzer DONE**: `patch/ts/jsonValuesModule.ts` `analyzeJsonValuesEntries(sourceExpr)` → + `Map` (uses `analyzeValModule` to get the source object literal). Tested. + - **Filename convention (LOCKED)**: the `*.val.json` for a new entry mirrors the key under a folder + named after the `.val.ts` (the `.val.ts` suffix becomes the folder). For module + `/app/foo/[...slug]/page.val.ts` and key `/foo/bar/zoo`: + - json file path (relative to rootDir): `/app/foo/[...slug]/page/foo/bar/zoo.val.json` + - import path written in the `.val.ts` thunk: `./page/foo/bar/zoo.val.json` + - i.e. `jsonPath = stripValTsSuffix(moduleFilePath) + "/" + key.replace(/^\//, "") + ".val.json"`; + `importPath = "./" + relative(dirOf(moduleFilePath), jsonPath)`. For EXISTING entries use the + `importPath` from the analyzer (hybrid: devs may have hand-placed files); only NEW entries use + this convention. + - **Routing in `prepare.applySourceFilePatches`**: need the module's serialized schema to know + which records are `jsonValues`. Partition each patch's `sourceFileOps` by op path: + 1. ops descending into a jsonValues entry (`[entryKey, ...sub]`) → CONTENT edits; + 2. add/remove of an entry key on a jsonValues record → STRUCTURAL; + 3. everything else → normal `tsOps` on the `.val.ts` (unchanged). + - **Content edits** (no sha ⇒ the `.val.ts` is NOT touched): load the entry's current `*.val.json` + (path = module dir + `importPath` from the analyzer; content via `getSourceFile`/fs), apply the + rebased sub-ops with `applyPatch` + `@valbuild/core/patch` JSON ops, and set + `patchedSourceFiles[jsonPath] = JSON.stringify(newContent)`. That's it — only the json file + changes. + - **Add entry**: `patchedSourceFiles[newJsonPath] = JSON.stringify(content)` + insert + `c.json(() => import(""))` property via `createValJsonReference(importPath)` + + `insertAt`. **Remove**: `patchedSourceFiles[jsonPath] = null` + `removeAt` the property. + - Then validation already handles inline content (`validateJsonValuesEntries` for base thunks; + `executeValidate` validates inline content). `/sources/~` shallow markers already work + (JSON.stringify drops the thunk). - **Last verified green**: core json suite (14 tests) + server `validateJsonValues`/loader/ `jsonReference` suites; core + server typecheck. (Earlier: whole-monorepo `-r typecheck` clean except the pre-existing unrelated `packages/cli` chokidar failure.) @@ -45,12 +69,10 @@ runtime-only — read it with `Internal.getJsonImport(source)`, never `source._import` in typed code. - **Done since**: server-side per-entry json validation (`validateJsonValues.ts`, wired into `ValOps.validateSources`); core `Internal.resolveJsonValues(source)` (eager resolver for the - `fetchVal`/`useVal` path); **sha helper `jsonValuesSha.ts`** (`computeJsonEntrySha` = - `-`, `isJsonEntryShaCurrent`, `getSchemaHashFromJsonSha`) implementing the - revalidation-signal design — the commit flow should write this sha; the sha-keyed validation - skip-cache should use `isJsonEntryShaCurrent`. **Discovered gap**: even eager `fetchVal` must - resolve markers before stega-encoding (a jsonValues module's local source is markers, not content) - — use `resolveJsonValues` there. + `fetchVal`/`useVal` path). **The sha was dropped (2026-07-02)** — `jsonValuesSha.ts` deleted, sha + removed from `c.json`/`JsonSource`/`/json`/analyzer/example. **Discovered gap**: even eager + `fetchVal` must resolve markers before stega-encoding (a jsonValues module's local source is + markers, not content) — use `resolveJsonValues` there. - **Runtime integration notes (for fetchValKey/fetchVal in next/react)**: disabled/production path reads the local module (`Internal.getSource`) whose markers still carry thunks → resolve locally (`getJsonImport` for one key, `resolveJsonValues` for all). Enabled/Studio path gets shallow @@ -63,7 +85,7 @@ Let `s.record(...)` and `s.router(...)` (NOT `s.images()` / `s.files()` galleries) opt into `.jsonValues()`, so each entry's value lives in its own `*.val.json` file referenced by a lazy -thunk `c.json(() => import("./x.val.json"), "")`. Keeps `.val.ts` tiny at 10K+ +thunk `c.json(() => import("./x.val.json"))`. Keeps `.val.ts` tiny at 10K+ entries; runtime/Studio/validation work one entry at a time; zero overhead when Val is disabled. ## Locked decisions (do not relitigate) @@ -71,14 +93,11 @@ entries; runtime/Studio/validation work one entry at a time; zero overhead when 1. `fetchVal`/`useVal` stay eager; new `fetchValKey`/`useValKey` + `fetchValRoute`/`useValRoute` load a single entry. 2. Hybrid authoring: Val generates/maintains json files + thunks; hand-edits re-validated. -3. The sha = validation-cache key (auto-generated), formatted as **`"-"`** - with the **schema hash as a PREFIX** (NOT hashed together). The prefix is the key property: you - can compare the current item-schema hash against each entry's sha prefix to know the entry needs - revalidation **without reading the entry's content** — the `.val.ts` already lists every entry's - sha. Revalidate when EITHER the prefix differs from the current schema hash (schema changed → - whole record) OR the content hash differs (that entry changed). Implemented in - `jsonValuesSha.ts` (`computeJsonEntrySha`, `getSchemaHashFromJsonSha`, `isJsonEntryShaCurrent`). - See "sha design" below. +3. **No sha (dropped 2026-07-02).** `c.json` takes ONLY the thunk: + `c.json(() => import("./x.val.json"))`. The earlier sha / validation-cache-key idea (a + `-` token) was removed — not worth the complexity. There is no + revalidation token: validation just runs on an entry's content when it is loaded (we accept that + validation/builds take more time). `jsonValuesSha.ts` was deleted. 4. Type precision: keep object/array structure, widen only what JSON can't carry (literals → base, drop `RawString`/brand, widen `_type` literals). Runtime validation enforces strictness. Val object-unions are always discriminated, so distribute+recurse suffices. @@ -87,18 +106,19 @@ entries; runtime/Studio/validation work one entry at a time; zero overhead when ## Key runtime shapes -`JsonSource` is a phantom-typed **pure-JSON marker** so `Source` stays JSON-serializable: +`JsonSource` is a phantom-typed **pure-JSON marker** so `Source` stays JSON-serializable +(no `_sha`): ```ts // JsonSource TYPE (what flows through Source/SelectorSource): -{ _type: "json", _sha: string, patch_id?: string } & PhantomType +{ _type: "json", patch_id?: string } & PhantomType -// RUNTIME value produced by c.json(thunk, sha) — also carries the thunk, which is +// RUNTIME value produced by c.json(thunk) — also carries the thunk, which is // NOT in the type; read it via Internal.getJsonImport(source): -{ _type: "json", _import: () => Promise<{ default: T }>, _sha: string } +{ _type: "json", _import: () => Promise<{ default: T }> } // Over the wire (/sources/~): the marker only (thunk dropped): -{ _type: "json", _sha: string, patch_id?: string } +{ _type: "json", patch_id?: string } ``` --- @@ -130,21 +150,20 @@ entries; runtime/Studio/validation work one entry at a time; zero overhead when to `RESOLVE_EXTENSIONS`; dynamic `import()` transpiles to a lazy `require` via `customRequire` so thunks stay lazy. Fixture: `test/jsonValues-fixture/` + `loadValModules.jsonValues.test.ts` (verifies marker shape, laziness, and thunk-loads-json). ✅ -- [x] `patch/ts/ops.ts`: `createValJsonReference(importPath, sha)` — emits - `c.json(() => import("..."), "sha")` (factory-built; uses `createIdentifier("import")` to print - a dynamic import without casting the ImportKeyword token). Tested in `jsonReference.test.ts`. ✅ -- [ ] `patch/ts/ops.ts` (remaining): wire add/replace/remove of json entries through - `insertAt`/`removeAt`/`replaceNodeValue` (+ write/replace-sha/delete `*.val.json`) — done with - the ValOps commit flow below. +- [x] `patch/ts/ops.ts`: `createValJsonReference(importPath)` — emits `c.json(() => import("..."))` + (factory-built; uses `createIdentifier("import")` to print a dynamic import without casting the + ImportKeyword token). Tested in `jsonReference.test.ts`. ✅ +- [ ] `patch/ts/ops.ts` (remaining): wire add/remove of json entries through `insertAt`/`removeAt` + (+ write/delete `*.val.json`) — done with the ValOps commit flow below. (Content edits don't + touch the `.val.ts` — no sha to update.) - [x] **Per-entry validation**: `validateJsonValues.ts` (`validateJsonValuesEntries`) loads each entry's content via `getJsonImport` and validates against the item schema; wired into `ValOps.validateSources` (runs before the `res === false` early-continue). Tested in `validateJsonValues.test.ts` (valid/invalid/load-error/non-jsonValues-skip). ✅ - [ ] `ValOps.ts` / `ValOpsFS.ts` / `ValOpsHttp.ts` (remaining): confirm shallow source - serialization on `/sources/~` (JSON.stringify already drops the thunk → `{_type,_sha}`); - commit writes `*.val.json` + updates `.val.ts` shas/thunks (use `createValJsonReference` + - `insertAt`/`removeAt`/`replaceNodeValue`); sha-keyed incremental validation (optimization); - recompute SHAs. + serialization on `/sources/~` (JSON.stringify already drops the thunk → `{_type}`); commit + writes `*.val.json` (content edits) + inserts/removes `c.json(...)` thunks for add/remove (use + `createValJsonReference` + `insertAt`/`removeAt`). - [x] Core eager resolver `Internal.resolveJsonValues(source)` (for `fetchVal`/`useVal`). ✅ - [ ] `ValServer.ts`: endpoint to fetch one entry's content (draft-aware via `patch_id`); `/sources/~` returns shallow markers for json records. @@ -158,7 +177,7 @@ because entry content is never loaded into the client source tree. Concrete inte (all in the 3376-line `ValSyncEngine.ts` unless noted): - [ ] **Content cache + fetch**: add `private jsonEntryContents: Record>` and `private loadingJsonEntries: Set<"mfp\0key">`. Add `requestJsonEntry(mfp, key)` +JSONValue>>` and `private loadingJsonEntries: Set<"mfp\0key">`. Add `requestJsonEntry(mfp, key)` that, if not loaded/loading, calls `this.client("/json", "GET", { query: { path: mfp, key } })`, stores `content`, then `invalidatePatchedSourcesCache(mfp)` + clears `cachedSourceSnapshots` for the module + `emit(this.listeners["source"]?.[mfp])` / `["sources"]` so subscribers @@ -174,10 +193,10 @@ because entry content is never loaded into the client source tree. Concrete inte marker isn't loaded, request it and render a loading state until `getSourceSnapshot` returns content. `useShallowSourceAtPath(path, "record")` already returns keys only (list is fine). - [ ] **Per-entry patches**: editing an entry field produces a normal patch at the entry path; on - commit the server commit-flow writes the `*.val.json` + updates the thunk/sha (Phase 2 commit - flow). Add/remove entry = add/remove the record key (commit flow emits/removes thunk + file). + commit the server commit-flow writes the `*.val.json` (Phase 2 commit flow). Add/remove entry = + add/remove the record key (commit flow emits/removes thunk + file). - [ ] **Verify** (Studio running): list shows keys w/o loading; opening an entry fetches one `/json`; - fields render + edit; commit writes file + updates sha; add/remove a route inserts/removes + fields render + edit; commit writes the `*.val.json`; add/remove a route inserts/removes thunk + file. ## Phase 4 — Runtime APIs (`packages/next`, `packages/react`) @@ -198,28 +217,12 @@ because entry content is never loaded into the client source tree. Concrete inte --- -## sha design (revalidation signal) - -The trailing sha in `c.json(thunk, sha)` is the validation-cache key, formatted -**`"-"`** (schema hash as PREFIX), and must answer "must we revalidate this -entry?" cheaply — ideally **without reading the entry's content**: - -- **Schema hash = prefix.** Derived from the serialized item schema (stable serialization) and - written as the part before the `-`. Keep them as separate joined components, NOT hashed together, - so the prefix stays readable. Truncated small (currently 8 hex chars) — it only needs to detect - schema _changes_, not be collision-proof. -- **Cheap staleness scan (the point):** after a schema change, compute the current item-schema hash - once, then walk the record's entry shas (already present in the `.val.ts`) and flag every entry - whose sha PREFIX differs — no `*.val.json` is loaded. Those entries need revalidation. -- **Content hash** (suffix) flips when the backing `*.val.json` content changes ⇒ revalidate just - that entry; the commit flow rewrites the sha. (Computing the content hash _does_ require the - content, but you only reach it for entries whose schema prefix already matches.) -- **Validation skip-cache:** `validateJsonValuesEntries` (today validates every entry) becomes: - if the entry's stored sha prefix ≠ current schema hash ⇒ must revalidate (load + validate, rewrite - sha); else look up `sha → lastValidationResult` and skip on a cached PASS, otherwise load + - validate + cache. Use `isJsonEntryShaCurrent` / `getSchemaHashFromJsonSha` from `jsonValuesSha.ts`. -- **Hybrid hand-edit safety:** a hand-edited json whose content no longer matches the suffix forces - revalidation; Val rewrites the sha on next commit. +## sha design — DROPPED (2026-07-02) + +The `c.json` sha (a `-` revalidation-cache key) was **removed** — it wasn't +worth the complexity. `c.json` takes only the thunk; `jsonValuesSha.ts` was deleted. Without a cache +key there is no skip-cache: `validateJsonValuesEntries` validates each loaded entry's content +unconditionally (accepted "validation takes more time" tradeoff). ## Open questions / watch-list @@ -232,7 +235,6 @@ entry?" cheaply — ideally **without reading the entry's content**: draft/endpoint path later). So committed validation is correct. - `vm` loader dynamic `import()` + `.json` resolution. - resolvePath/selector must never throw on a not-yet-loaded entry. -- Canonical content hashing so hybrid hand-edits and Val-writes agree on the sha. ## Changelog diff --git a/examples/next/app/support/[slug]/page.val.ts b/examples/next/app/support/[slug]/page.val.ts index 006d222b0..d55ad50ce 100644 --- a/examples/next/app/support/[slug]/page.val.ts +++ b/examples/next/app/support/[slug]/page.val.ts @@ -2,12 +2,8 @@ import { s, c, type t, nextAppRouter } from "../../../val.config"; /** * Support pages use `.jsonValues()`: each page's content lives in its own - * `*.val.json` file and is loaded lazily via `c.json(() => import(...), sha)`. + * `*.val.json` file and is loaded lazily via `c.json(() => import(...))`. * This keeps `page.val.ts` tiny even with thousands of support pages. - * - * NOTE: the trailing sha is the validation-cache key (`-`). - * These are placeholders for now — Val tooling (commit flow) will generate and - * maintain them; they are not checked on the runtime read path. */ export const supportPageSchema = s.object({ title: s.string().minLength(2), @@ -23,8 +19,7 @@ export default c.define( { "/support/getting-started": c.json( () => import("./content/getting-started.val.json"), - "v1", ), - "/support/faq": c.json(() => import("./content/faq.val.json"), "v1"), + "/support/faq": c.json(() => import("./content/faq.val.json")), }, ); diff --git a/packages/core/src/schema/jsonValues.test.ts b/packages/core/src/schema/jsonValues.test.ts index 82de7c981..6322113af 100644 --- a/packages/core/src/schema/jsonValues.test.ts +++ b/packages/core/src/schema/jsonValues.test.ts @@ -20,11 +20,10 @@ import { nextAppRouter } from "../router"; import { initVal } from "../initVal"; describe("c.json + .jsonValues()", () => { - test("json() returns a JsonSource marker with thunk + sha", () => { + test("json() returns a JsonSource marker with a thunk", () => { const thunk = () => Promise.resolve({ default: { title: "hi" } }); - const src = json(thunk, "abc123"); + const src = json(thunk); expect(src[VAL_EXTENSION]).toBe("json"); - expect(src._sha).toBe("abc123"); expect(getJsonImport(src)).toBe(thunk); expect(isJson(src)).toBe(true); expect(isJson({ title: "hi" })).toBe(false); @@ -85,7 +84,7 @@ describe("c.json + .jsonValues()", () => { // validateJsonEntryContent once the file is loaded. loaded = true; return Promise.resolve({ default: { title: "ok" } }); - }, "s1"), + }), }; const errors = schema["executeValidate"]( "/test.val.ts" as SourcePath, @@ -131,8 +130,8 @@ describe("c.json + .jsonValues()", () => { describe("resolveJsonValues", () => { test("resolves a record of markers into inlined content", async () => { const source: Source = { - "/a": json(() => Promise.resolve({ default: { title: "A" } }), "sa"), - "/b": json(() => Promise.resolve({ default: { title: "B" } }), "sb"), + "/a": json(() => Promise.resolve({ default: { title: "A" } })), + "/b": json(() => Promise.resolve({ default: { title: "B" } })), }; const resolved = await resolveJsonValues(source); expect(resolved).toEqual({ "/a": { title: "A" }, "/b": { title: "B" } }); @@ -140,17 +139,12 @@ describe("resolveJsonValues", () => { test("resolves nested markers recursively", async () => { const source: Source = { - "/outer": json( - () => - Promise.resolve({ - default: { - inner: json( - () => Promise.resolve({ default: { deep: "value" } }), - "si", - ), - }, - }), - "so", + "/outer": json(() => + Promise.resolve({ + default: { + inner: json(() => Promise.resolve({ default: { deep: "value" } })), + }, + }), ), }; const resolved = await resolveJsonValues(source); @@ -158,9 +152,9 @@ describe("resolveJsonValues", () => { }); test("leaves transport markers (no thunk) as-is", async () => { - const marker = { _type: "json", _sha: "x" } as unknown as Source; + const marker = { _type: "json" } as unknown as Source; const resolved = await resolveJsonValues({ "/a": marker }); - expect(resolved).toEqual({ "/a": { _type: "json", _sha: "x" } }); + expect(resolved).toEqual({ "/a": { _type: "json" } }); }); }); @@ -173,9 +167,8 @@ describe("c.define authoring surface (compile-time)", () => { "/blogs/[slug]/page.val.ts", s.router(nextAppRouter, s.object({ title: s.string() })).jsonValues(), { - "/blogs/test": c.json( - () => Promise.resolve({ default: { title: "Hello" } }), - "1232132", + "/blogs/test": c.json(() => + Promise.resolve({ default: { title: "Hello" } }), ), }, ); diff --git a/packages/core/src/schema/record.ts b/packages/core/src/schema/record.ts index 16e9b9f1a..93c83fa99 100644 --- a/packages/core/src/schema/record.ts +++ b/packages/core/src/schema/record.ts @@ -640,7 +640,7 @@ export class RecordSchema< /** * Store each entry's value in its own lazily-loaded `*.val.json` file instead * of inlining it in the `.val.ts` module. Entry values become - * {@link JsonSource} thunks (`c.json(() => import("./entry.val.json"), sha)`), + * {@link JsonSource} thunks (`c.json(() => import("./entry.val.json"))`), * which lets the runtime, the Studio and validation work one entry at a time * so a record/router can scale to many thousands of entries. * diff --git a/packages/core/src/source/json.ts b/packages/core/src/source/json.ts index 8c10ca1ce..6494f5220 100644 --- a/packages/core/src/source/json.ts +++ b/packages/core/src/source/json.ts @@ -18,9 +18,9 @@ export type JsonImportThunk = () => Promise<{ default: T }>; * A JSON source represents a record/router entry whose value is stored in a * separate `*.val.json` file and loaded lazily via a dynamic `import()` thunk. * - * This is what `c.json(() => import("./entry.val.json"), "")` returns. + * This is what `c.json(() => import("./entry.val.json"))` returns. * - * The *type* is a pure-JSON marker (`{ _type: "json", _sha, patch_id? }`) so that + * The *type* is a pure-JSON marker (`{ _type: "json", patch_id? }`) so that * `Source` stays JSON-serializable. At runtime the value additionally carries a * lazy import thunk (read via {@link getJsonImport}); over the wire only the * marker is sent. @@ -31,21 +31,15 @@ export type JsonImportThunk = () => Promise<{ default: T }>; */ export type JsonSource = { readonly [VAL_EXTENSION]: typeof JSON_VAL_EXTENSION_TAG; - /** Content hash of the backing JSON, used as a validation-cache key. */ - readonly _sha: string; /** Set on uncommitted/draft entries (mirrors FileSource/RemoteSource). */ readonly patch_id?: string; } & PhantomType; -export function json( - importThunk: JsonImportThunk, - sha: string, -): JsonSource { +export function json(importThunk: JsonImportThunk): JsonSource { return { [VAL_EXTENSION]: JSON_VAL_EXTENSION_TAG, // runtime-only: not described by JsonSource so Source stays JSON-serializable _import: importThunk, - _sha: sha, } as unknown as JsonSource; } diff --git a/packages/server/src/ValServer.ts b/packages/server/src/ValServer.ts index 20d8b9e13..08641c2fb 100644 --- a/packages/server/src/ValServer.ts +++ b/packages/server/src/ValServer.ts @@ -1420,10 +1420,9 @@ export const ValServer = ( }; } } - const sha = (marker as { _sha?: string })._sha; return { status: 200, - json: { path: moduleFilePath, key, content, sha }, + json: { path: moduleFilePath, key, content }, }; }, }, diff --git a/packages/server/src/jsonValuesSha.test.ts b/packages/server/src/jsonValuesSha.test.ts deleted file mode 100644 index fb30151c9..000000000 --- a/packages/server/src/jsonValuesSha.test.ts +++ /dev/null @@ -1,64 +0,0 @@ -import { initVal } from "@valbuild/core"; -import { - computeJsonEntrySha, - getJsonItemSchemaHash, - getSchemaHashFromJsonSha, - isJsonEntryShaCurrent, - jsonEntrySchemaHashIsStale, -} from "./jsonValuesSha"; - -const { s } = initVal(); - -describe("jsonValues validation sha", () => { - const item = s.object({ title: s.string() }); - - test("is deterministic for the same schema + content", () => { - expect(computeJsonEntrySha(item, { title: "a" })).toBe( - computeJsonEntrySha(item, { title: "a" }), - ); - }); - - test("content change flips the sha but keeps the schema component", () => { - const sha1 = computeJsonEntrySha(item, { title: "a" }); - const sha2 = computeJsonEntrySha(item, { title: "b" }); - expect(sha1).not.toBe(sha2); - // same schema → same schema-hash component (so only that entry revalidates) - expect(getSchemaHashFromJsonSha(sha1)).toBe(getSchemaHashFromJsonSha(sha2)); - }); - - test("schema change flips the schema component (forces record revalidation)", () => { - const item2 = s.object({ title: s.string(), extra: s.number() }); - const shaA = computeJsonEntrySha(item, { title: "a" }); - const shaB = computeJsonEntrySha(item2, { title: "a" }); - expect(getJsonItemSchemaHash(item)).not.toBe(getJsonItemSchemaHash(item2)); - expect(getSchemaHashFromJsonSha(shaA)).not.toBe( - getSchemaHashFromJsonSha(shaB), - ); - }); - - test("isJsonEntryShaCurrent detects content and schema drift", () => { - const sha = computeJsonEntrySha(item, { title: "a" }); - expect(isJsonEntryShaCurrent(sha, item, { title: "a" })).toBe(true); - // content drift - expect(isJsonEntryShaCurrent(sha, item, { title: "changed" })).toBe(false); - // schema drift - const item2 = s.object({ title: s.string(), extra: s.number() }); - expect(isJsonEntryShaCurrent(sha, item2, { title: "a" })).toBe(false); - }); - - test("getSchemaHashFromJsonSha returns null for a non-composite sha", () => { - expect(getSchemaHashFromJsonSha("1232132")).toBeNull(); - }); - - test("jsonEntrySchemaHashIsStale detects schema drift WITHOUT reading content", () => { - // sha was written against `item`; we never pass the content here. - const storedSha = computeJsonEntrySha(item, { title: "a" }); - expect(jsonEntrySchemaHashIsStale(storedSha, item)).toBe(false); - - const changedItem = s.object({ title: s.string(), extra: s.number() }); - expect(jsonEntrySchemaHashIsStale(storedSha, changedItem)).toBe(true); - - // a hand-authored / legacy sha with no schema prefix is treated as stale - expect(jsonEntrySchemaHashIsStale("1232132", item)).toBe(true); - }); -}); diff --git a/packages/server/src/jsonValuesSha.ts b/packages/server/src/jsonValuesSha.ts deleted file mode 100644 index c1b63e399..000000000 --- a/packages/server/src/jsonValuesSha.ts +++ /dev/null @@ -1,79 +0,0 @@ -import { Internal, Schema, SelectorSource } from "@valbuild/core"; - -const textEncoder = new TextEncoder(); - -function hash(input: string): string { - return Internal.getSHA256Hash(textEncoder.encode(input)); -} - -/** - * A small, stable hash of a record's item schema. It is the *schema component* - * of a json entry's validation sha (see {@link computeJsonEntrySha}). Because it - * is derived from the serialized item schema, it flips whenever the schema - * changes — which is how we know every entry of a `.jsonValues()` record must be - * revalidated after a schema change. - */ -export function getJsonItemSchemaHash( - itemSchema: Schema, -): string { - return hash(JSON.stringify(itemSchema["executeSerialize"]())).slice(0, 8); -} - -/** - * The validation sha stored in `c.json(thunk, sha)`. It is a composite - * `"-"` so that revalidation is required when EITHER: - * - the entry content changes (the content-hash component flips), or - * - the item schema changes (the schema-hash component flips → revalidate the - * whole record). - * - * Validation/commit compares the stored sha against a freshly-computed one and - * only revalidates on mismatch; the commit flow writes this sha into the - * `.val.ts`. - */ -export function computeJsonEntrySha( - itemSchema: Schema, - content: unknown, -): string { - const schemaHash = getJsonItemSchemaHash(itemSchema); - const contentHash = hash(JSON.stringify(content ?? null)); - return `${schemaHash}-${contentHash}`; -} - -/** - * Extracts the schema-hash component (the PREFIX) of a composite json entry sha, - * or `null` if the sha is not in composite form (e.g. a hand-authored sha). - */ -export function getSchemaHashFromJsonSha(sha: string): string | null { - const idx = sha.indexOf("-"); - return idx === -1 ? null : sha.slice(0, idx); -} - -/** - * Cheap, **content-free** staleness check: does this entry need revalidation - * purely because the schema changed? Compares the entry's stored sha PREFIX - * against the current item-schema hash — no `*.val.json` is read. Returns `true` - * when the schema changed under the entry (prefixes differ) or the stored sha - * has no schema prefix (can't prove it's current). - * - * This is what lets a schema change flag the entries needing revalidation by - * scanning only the shas already present in the `.val.ts`. - */ -export function jsonEntrySchemaHashIsStale( - storedSha: string, - itemSchema: Schema, -): boolean { - const prefix = getSchemaHashFromJsonSha(storedSha); - return prefix === null || prefix !== getJsonItemSchemaHash(itemSchema); -} - -/** - * True when `content` + `itemSchema` still match `storedSha`, i.e. no - * revalidation is needed. - */ -export function isJsonEntryShaCurrent( - storedSha: string, - itemSchema: Schema, - content: unknown, -): boolean { - return storedSha === computeJsonEntrySha(itemSchema, content); -} diff --git a/packages/server/src/loadValModules.jsonValues.test.ts b/packages/server/src/loadValModules.jsonValues.test.ts index b36235001..4fb27c030 100644 --- a/packages/server/src/loadValModules.jsonValues.test.ts +++ b/packages/server/src/loadValModules.jsonValues.test.ts @@ -13,14 +13,12 @@ describe("loadValModules with .jsonValues() + c.json", () => { const source = Internal.getSource(mod) as Record; const entry = source["/blogs/test"] as { _type: string; - _sha: string; _import: () => Promise<{ default: unknown }>; }; // The entry is a json marker, not the loaded content. expect(Internal.isJson(entry)).toBe(true); expect(entry._type).toBe("json"); - expect(entry._sha).toBe("testsha123"); expect(typeof entry._import).toBe("function"); }); diff --git a/packages/server/src/patch/ts/jsonReference.test.ts b/packages/server/src/patch/ts/jsonReference.test.ts index 46415e424..f11d3c3c6 100644 --- a/packages/server/src/patch/ts/jsonReference.test.ts +++ b/packages/server/src/patch/ts/jsonReference.test.ts @@ -9,17 +9,16 @@ function print(node: ts.Node): string { } describe("createValJsonReference", () => { - test("prints c.json(() => import(...), sha)", () => { - const node = createValJsonReference("./page/blogs/test.val.json", "abc123"); + test("prints c.json(() => import(...))", () => { + const node = createValJsonReference("./page/blogs/test.val.json"); const out = print(node); expect(out).toContain("c.json("); expect(out).toContain('import("./page/blogs/test.val.json")'); - expect(out).toContain('"abc123"'); expect(out).toContain("() =>"); }); test("produces valid, re-parseable output", () => { - const node = createValJsonReference("./a.val.json", "sha"); + const node = createValJsonReference("./a.val.json"); const out = print(node); const reparsed = ts.createSourceFile( "reparse.ts", diff --git a/packages/server/src/patch/ts/jsonValuesModule.test.ts b/packages/server/src/patch/ts/jsonValuesModule.test.ts new file mode 100644 index 000000000..0053a5bb2 --- /dev/null +++ b/packages/server/src/patch/ts/jsonValuesModule.test.ts @@ -0,0 +1,63 @@ +import ts from "typescript"; +import { analyzeValModule } from "./valModule"; +import { analyzeJsonValuesEntries } from "./jsonValuesModule"; +import { result } from "@valbuild/core/fp"; + +function analyze(code: string) { + const sf = ts.createSourceFile( + "test.val.ts", + code, + ts.ScriptTarget.ES2020, + true, + ); + const mod = analyzeValModule(sf); + if (result.isErr(mod)) { + throw new Error("analyzeValModule failed: " + JSON.stringify(mod.error)); + } + return analyzeJsonValuesEntries(mod.value.source); +} + +describe("analyzeJsonValuesEntries", () => { + test("extracts import path for each c.json entry", () => { + const entries = analyze(` + import { s, c } from "./val.config"; + export default c.define( + "/blogs.val.ts", + s.router(r, schema).jsonValues(), + { + "/blogs/a": c.json(() => import("./content/a.val.json")), + "/blogs/b": c.json(() => import("./content/b.val.json")), + } + ); + `); + expect(entries.size).toBe(2); + expect(entries.get("/blogs/a")).toEqual({ + importPath: "./content/a.val.json", + }); + expect(entries.get("/blogs/b")).toEqual({ + importPath: "./content/b.val.json", + }); + }); + + test("skips non-c.json entries", () => { + const entries = analyze(` + import { s, c } from "./val.config"; + export default c.define("/x.val.ts", s.record(schema), { + "/a": { title: "inline" }, + }); + `); + expect(entries.size).toBe(0); + }); + + test("handles a block-body thunk", () => { + const entries = analyze(` + import { s, c } from "./val.config"; + export default c.define("/x.val.ts", s.record(schema).jsonValues(), { + "/a": c.json(() => { return import("./a.val.json"); }), + }); + `); + expect(entries.get("/a")).toEqual({ + importPath: "./a.val.json", + }); + }); +}); diff --git a/packages/server/src/patch/ts/jsonValuesModule.ts b/packages/server/src/patch/ts/jsonValuesModule.ts new file mode 100644 index 000000000..7c9d17c45 --- /dev/null +++ b/packages/server/src/patch/ts/jsonValuesModule.ts @@ -0,0 +1,109 @@ +import ts from "typescript"; + +export type JsonValuesEntry = { + /** The string literal inside `import("...")` in the entry's thunk. */ + importPath: string; +}; + +/** + * Given the `source` object-literal expression of a `.jsonValues()` record / + * router (the 3rd argument to `c.define`, via {@link analyzeValModule}), + * extracts each entry's `c.json(() => import("..."))` import path, keyed by + * entry key. Entries whose value is not a `c.json(...)` call are skipped. + */ +export function analyzeJsonValuesEntries( + source: ts.Expression, +): Map { + const entries = new Map(); + if (!ts.isObjectLiteralExpression(source)) { + return entries; + } + for (const prop of source.properties) { + if (!ts.isPropertyAssignment(prop)) { + continue; + } + const key = getPropertyKey(prop.name); + if (key === null) { + continue; + } + const entry = analyzeCJsonCall(prop.initializer); + if (entry) { + entries.set(key, entry); + } + } + return entries; +} + +function getPropertyKey(name: ts.PropertyName): string | null { + if (ts.isStringLiteralLike(name)) { + return name.text; + } + if (ts.isIdentifier(name)) { + return name.text; + } + return null; +} + +function analyzeCJsonCall(node: ts.Expression): JsonValuesEntry | null { + // c.json(() => import("path")) + if (!ts.isCallExpression(node) || !isCJson(node.expression)) { + return null; + } + const [thunk] = node.arguments; + if (!thunk) { + return null; + } + const importPath = getImportPathFromThunk(thunk); + if (importPath === null) { + return null; + } + return { importPath }; +} + +function isCJson(expr: ts.Expression): boolean { + // c.json + return ( + ts.isPropertyAccessExpression(expr) && + ts.isIdentifier(expr.expression) && + expr.expression.text === "c" && + ts.isIdentifier(expr.name) && + expr.name.text === "json" + ); +} + +function getImportPathFromThunk(thunk: ts.Expression): string | null { + // () => import("path") (concise body) OR () => { return import("path"); } + if (!ts.isArrowFunction(thunk)) { + return null; + } + let importCall: ts.Expression | undefined; + if (ts.isCallExpression(thunk.body)) { + importCall = thunk.body; + } else if (ts.isBlock(thunk.body)) { + for (const stmt of thunk.body.statements) { + if ( + ts.isReturnStatement(stmt) && + stmt.expression && + ts.isCallExpression(stmt.expression) + ) { + importCall = stmt.expression; + break; + } + } + } + if (!importCall || !ts.isCallExpression(importCall)) { + return null; + } + const isImport = + importCall.expression.kind === ts.SyntaxKind.ImportKeyword || + (ts.isIdentifier(importCall.expression) && + importCall.expression.text === "import"); + if (!isImport) { + return null; + } + const arg = importCall.arguments[0]; + if (arg && ts.isStringLiteralLike(arg)) { + return arg.text; + } + return null; +} diff --git a/packages/server/src/patch/ts/ops.ts b/packages/server/src/patch/ts/ops.ts index 064cfa4e1..6bae146c3 100644 --- a/packages/server/src/patch/ts/ops.ts +++ b/packages/server/src/patch/ts/ops.ts @@ -127,13 +127,10 @@ function createValRemoteReference(value: RemoteSource) { } /** - * Builds the expression `c.json(() => import(""), "")` used to - * reference a lazily-loaded `*.val.json` entry of a `.jsonValues()` record. + * Builds the expression `c.json(() => import(""))` used to reference + * a lazily-loaded `*.val.json` entry of a `.jsonValues()` record. */ -export function createValJsonReference( - importPath: string, - sha: string, -): ts.Expression { +export function createValJsonReference(importPath: string): ts.Expression { // () => import("") // NOTE: an `import` identifier prints as the dynamic-import keyword call, // which avoids casting the ImportKeyword token (not typed as an Expression). @@ -150,14 +147,14 @@ export function createValJsonReference( ts.factory.createToken(ts.SyntaxKind.EqualsGreaterThanToken), importCall, ); - // c.json(, "") + // c.json() return ts.factory.createCallExpression( ts.factory.createPropertyAccessExpression( ts.factory.createIdentifier("c"), ts.factory.createIdentifier("json"), ), undefined, - [thunk, ts.factory.createStringLiteral(sha)], + [thunk], ); } diff --git a/packages/server/src/validateJsonValues.test.ts b/packages/server/src/validateJsonValues.test.ts index 60afa0fab..8010f0f39 100644 --- a/packages/server/src/validateJsonValues.test.ts +++ b/packages/server/src/validateJsonValues.test.ts @@ -9,14 +9,8 @@ describe("validateJsonValuesEntries", () => { test("returns no errors when all entry content is valid", async () => { const source = { - "/a": c.json( - () => Promise.resolve({ default: { title: "ok" } }), - "sha-a", - ), - "/b": c.json( - () => Promise.resolve({ default: { title: "ok2" } }), - "sha-b", - ), + "/a": c.json(() => Promise.resolve({ default: { title: "ok" } })), + "/b": c.json(() => Promise.resolve({ default: { title: "ok2" } })), }; const errors = await validateJsonValuesEntries(schema, source, modulePath); expect(errors).toEqual({}); @@ -24,15 +18,9 @@ describe("validateJsonValuesEntries", () => { test("reports validation errors for invalid entry content", async () => { const source = { - "/a": c.json( - () => Promise.resolve({ default: { title: "ok" } }), - "sha-a", - ), + "/a": c.json(() => Promise.resolve({ default: { title: "ok" } })), // wrong leaf type for title — caught by the deferred content validation - "/bad": c.json( - () => Promise.resolve({ default: { title: 123 } }), - "sha-bad", - ), + "/bad": c.json(() => Promise.resolve({ default: { title: 123 } })), }; const errors = await validateJsonValuesEntries(schema, source, modulePath); const keys = Object.keys(errors); @@ -42,7 +30,7 @@ describe("validateJsonValuesEntries", () => { test("reports a load error when the entry thunk rejects", async () => { const source = { - "/boom": c.json(() => Promise.reject(new Error("disk gone")), "sha-boom"), + "/boom": c.json(() => Promise.reject(new Error("disk gone"))), }; const errors = await validateJsonValuesEntries(schema, source, modulePath); const keys = Object.keys(errors); @@ -59,7 +47,7 @@ describe("validateJsonValuesEntries", () => { "/a": c.json(() => { loaded = true; return Promise.resolve({ default: { title: "ok" } }); - }, "sha-a"), + }), }; const errors = await validateJsonValuesEntries( plainSchema, diff --git a/packages/server/test/jsonValues-fixture/blogs.val.ts b/packages/server/test/jsonValues-fixture/blogs.val.ts index 392e8fae2..5a7c0d2ca 100644 --- a/packages/server/test/jsonValues-fixture/blogs.val.ts +++ b/packages/server/test/jsonValues-fixture/blogs.val.ts @@ -4,9 +4,6 @@ export default c.define( "/blogs.val.ts", s.record(s.object({ title: s.string() })).jsonValues(), { - "/blogs/test": c.json( - () => import("./page/blogs/test.val.json"), - "testsha123", - ), + "/blogs/test": c.json(() => import("./page/blogs/test.val.json")), }, ); diff --git a/packages/shared/src/internal/ApiRoutes.ts b/packages/shared/src/internal/ApiRoutes.ts index 459f068a7..ba350a339 100644 --- a/packages/shared/src/internal/ApiRoutes.ts +++ b/packages/shared/src/internal/ApiRoutes.ts @@ -1032,7 +1032,6 @@ export const Api = { key: z.string(), // The entry's JSON content (or null if the entry has no value). content: z.any(), - sha: z.string().optional(), }), }), z.object({ diff --git a/packages/ui/spa/ValSyncEngine.ts b/packages/ui/spa/ValSyncEngine.ts index 2e0578860..bbb5e803c 100644 --- a/packages/ui/spa/ValSyncEngine.ts +++ b/packages/ui/spa/ValSyncEngine.ts @@ -137,7 +137,7 @@ export class ValSyncEngine { > | null; /** * Loaded content for `.jsonValues()` record entries, keyed by module then - * entry key. The on-disk source only carries lazy `{ _type:"json", _sha }` + * entry key. The on-disk source only carries lazy `{ _type:"json" }` * markers; the Studio fetches an entry's content on demand via * `requestJsonEntry` (GET /json) and `getPatchedSource` substitutes it in so * field resolution/rendering works. diff --git a/packages/ui/spa/components/traverseSchemas.ts b/packages/ui/spa/components/traverseSchemas.ts index 61568d428..8508f1a09 100644 --- a/packages/ui/spa/components/traverseSchemas.ts +++ b/packages/ui/spa/components/traverseSchemas.ts @@ -37,7 +37,7 @@ export function traverseSchemas( ) => { if (Internal.isJson(source)) { // An un-loaded `.jsonValues()` entry is an opaque lazy marker - // ({ _type:"json", _sha }). Skip it: once its content is loaded the + // ({ _type:"json" }). Skip it: once its content is loaded the // substituted value is the real content and is traversed normally. return; } From 1ac3b159df7fb6828c93e1b76d34ee56360c1cc5 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Thu, 2 Jul 2026 22:41:50 +0000 Subject: [PATCH 22/35] Persist json values patch edits --- docs/plans/jsonValues.md | 106 +++-- packages/server/src/ValOps.ts | 382 +++++++++++++++--- .../server/src/ValOpsFS.jsonValues.test.ts | 152 +++++++ .../server/src/patch/jsonValuesPatch.test.ts | 93 +++++ packages/server/src/patch/jsonValuesPatch.ts | 121 ++++++ .../src/patch/ts/jsonValuesEntry.test.ts | 72 ++++ packages/server/src/patch/ts/ops.ts | 105 +++++ 7 files changed, 923 insertions(+), 108 deletions(-) create mode 100644 packages/server/src/ValOpsFS.jsonValues.test.ts create mode 100644 packages/server/src/patch/jsonValuesPatch.test.ts create mode 100644 packages/server/src/patch/jsonValuesPatch.ts create mode 100644 packages/server/src/patch/ts/jsonValuesEntry.test.ts diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md index 69a616573..03b3ad36c 100644 --- a/docs/plans/jsonValues.md +++ b/docs/plans/jsonValues.md @@ -10,18 +10,26 @@ ## Current state / resume here +> **Commit flow DONE (2026-07-03):** `ValOps.prepare` now routes patch ops for `.jsonValues()` +> records. Content edits write only the entry's `*.val.json` (the `.val.ts` is NOT touched); adding +> an entry writes a new `*.val.json` + inserts a `c.json(() => import("..."))` thunk into the +> `.val.ts`; removing an entry deletes the `*.val.json` (via a `patchedSourceFiles[path] = null`) + +> drops the thunk. All three go through the existing `patchedSourceFiles` map, so both `ValOpsFS` +> and `ValOpsHttp` commit them with no new abstraction. Server suite green (166), whole monorepo +> `pnpm test` green (1056), `-r typecheck` clean. + > **Studio lazy-load DONE (2026-06-30):** opening a jsonValues entry now fetches its content via > `GET /json` and renders the fields (was: the `resolvePath` guard error). Read path works in both -> production (`fetchValKey`) and the Studio. **Editing** shows optimistic updates, but **persistence -> to `*.val.json` needs the commit flow (still pending)** — so publishing an edit won't write the -> json file yet. Requires `pnpm --filter @valbuild/ui build` for the Studio bundle to pick up UI -> changes (it's a built bundle, not a live dev-stub). +> production (`fetchValKey`) and the Studio. **Editing** now persists on commit (see above). +> Requires `pnpm --filter @valbuild/ui build` for the Studio bundle to pick up UI changes (it's a +> built bundle, not a live dev-stub). -- **Phase**: 1 ✅. Phase 2 server: validation + loader + emit primitive + **/json endpoint** ✅ - (commit flow still pending). Phase 3 UI lazy-load ✅ (Studio reads jsonValues entries). Phase 4: +- **Phase**: 1 ✅. Phase 2 server: validation + loader + emit primitive + **/json endpoint** + + **commit flow** ✅. Phase 3 UI lazy-load ✅ (Studio reads jsonValues entries). Phase 4: `fetchValKey`/`useValKey` ✅ (production path). Example (support pages) added + typechecks. - **The `c.json` sha was removed (2026-07-02).** Remaining big piece: the **commit flow** (write - `*.val.json` for content edits; insert/remove `c.json` thunks for add/remove). + **The `c.json` sha was removed (2026-07-02).** Remaining: end-to-end Studio verify of add/remove/ + edit against a running dev server; `fetchValRoute`/`useValRoute` key path; Enabled/Studio draft + runtime path; Phase 5 CI gate (example `.jsonValues()` router + `examples/next` build). - **Single-entry runtime read API (typecheck-validated, runtime-validation via example pending)**: - RSC `fetchValKey` — `initFetchValKeyStega` in `next/src/rsc/initValRsc.ts` (returned as `fetchValKeyStega`). Resolves ONE entry by key from the local module's thunk + stega-encodes. @@ -30,36 +38,42 @@ - Both: production path resolves the local thunk; Enabled/Studio **draft** path is a TODO (needs the single-entry endpoint + a sub-selector for stega edit tags). Still TODO: make `fetchValRoute`/ `useValRoute` use the key path for jsonValues routers (load one). -- **Next step (the commit flow — IN PROGRESS)**: persist edits to `*.val.json` instead of the - `.val.ts`. Design (now fully mapped): +- **Commit flow — DONE (2026-07-03)**. Persists edits to `*.val.json` instead of the `.val.ts`. + Implementation landed: - **Key enabler**: `ValOps.prepare`'s `patchedSourceFiles: Record` is written by - the commit loop to **arbitrary paths** relative to rootDir (`null` = delete). So `*.val.json` + the commit loop to **arbitrary paths** relative to rootDir (`null` = delete). `*.val.json` writes/deletes go through the same map (no new abstract FS method needed). - - **AST analyzer DONE**: `patch/ts/jsonValuesModule.ts` `analyzeJsonValuesEntries(sourceExpr)` → + - **AST analyzer**: `patch/ts/jsonValuesModule.ts` `analyzeJsonValuesEntries(sourceExpr)` → `Map` (uses `analyzeValModule` to get the source object literal). Tested. - - **Filename convention (LOCKED)**: the `*.val.json` for a new entry mirrors the key under a folder - named after the `.val.ts` (the `.val.ts` suffix becomes the folder). For module - `/app/foo/[...slug]/page.val.ts` and key `/foo/bar/zoo`: - - json file path (relative to rootDir): `/app/foo/[...slug]/page/foo/bar/zoo.val.json` - - import path written in the `.val.ts` thunk: `./page/foo/bar/zoo.val.json` - - i.e. `jsonPath = stripValTsSuffix(moduleFilePath) + "/" + key.replace(/^\//, "") + ".val.json"`; - `importPath = "./" + relative(dirOf(moduleFilePath), jsonPath)`. For EXISTING entries use the - `importPath` from the analyzer (hybrid: devs may have hand-placed files); only NEW entries use - this convention. - - **Routing in `prepare.applySourceFilePatches`**: need the module's serialized schema to know - which records are `jsonValues`. Partition each patch's `sourceFileOps` by op path: - 1. ops descending into a jsonValues entry (`[entryKey, ...sub]`) → CONTENT edits; - 2. add/remove of an entry key on a jsonValues record → STRUCTURAL; - 3. everything else → normal `tsOps` on the `.val.ts` (unchanged). - - **Content edits** (no sha ⇒ the `.val.ts` is NOT touched): load the entry's current `*.val.json` - (path = module dir + `importPath` from the analyzer; content via `getSourceFile`/fs), apply the - rebased sub-ops with `applyPatch` + `@valbuild/core/patch` JSON ops, and set - `patchedSourceFiles[jsonPath] = JSON.stringify(newContent)`. That's it — only the json file - changes. - - **Add entry**: `patchedSourceFiles[newJsonPath] = JSON.stringify(content)` + insert - `c.json(() => import(""))` property via `createValJsonReference(importPath)` + - `insertAt`. **Remove**: `patchedSourceFiles[jsonPath] = null` + `removeAt` the property. - - Then validation already handles inline content (`validateJsonValuesEntries` for base thunks; + - **Path helpers**: `patch/jsonValuesPatch.ts` — `getNewJsonEntryPaths(mfp, key)` (LOCKED + convention below) + `resolveExistingJsonPath(mfp, importPath)` (existing/hand-placed files use + the analyzer's importPath). Tested (`jsonValuesPatch.test.ts`). + - **Op classifier**: `patch/jsonValuesPatch.ts` `classifyJsonValuesOp(serializedSchema, opPath)` + walks the serialized schema; returns `{kind:"entry", recordPath, entryKey, subPath}` or + `{kind:"normal"}`. Handles root records/routers (recordPath `[]`) AND nested jsonValues records. + - **ts-ops**: `patch/ts/ops.ts` `insertValJsonEntry` / `removeValJsonEntry` insert/remove the + `c.json(() => import(...))` property on the record's object literal (built with + `createValJsonReference`, spliced with the internal `insertAt`/`removeAt`). Tested + (`jsonValuesEntry.test.ts`). + - **Routing in `prepare.applySourceFilePatches`**: fetches `this.getSchemas()` once, serializes per + module, and processes each patch's `sourceFileOps` op-by-op (in order). Per op: + 1. `normal` → `applyPatch(tsSourceFile, tsOps, [op])` (unchanged behavior; `.val.ts` reformatted). + 2. `entry` + empty subPath → STRUCTURAL: `add` = new `*.val.json` + `insertValJsonEntry`; + `remove` = `null` json + `removeValJsonEntry`; `replace` = whole-entry json content. + 3. `entry` + non-empty subPath → CONTENT: load current `*.val.json` (`getSourceFile` + + `JSON.parse`, cached in a per-module map), then replay the **rebased** op via `jsonOps` + (`rebaseContentOp` drops the record+entryKey prefix and rebases any move/copy `from`). + - **`.val.ts` untouched on pure content edits**: `applySourceFilePatches` now returns + `result: string | null` (`null` = ts unchanged) + `extraFiles: Record`. The + caller only writes `patchedSourceFiles[mfp]` when `result !== null`, and always merges + `extraFiles`. So a content-only commit writes ONLY the `*.val.json`. + - **Filename convention (LOCKED)**: new entry mirrors the key under a folder named after the + `.val.ts` (its `.val.ts` suffix becomes the folder). Module `/app/foo/[...slug]/page.val.ts`, + key `/foo/bar/zoo` → jsonPath `/app/foo/[...slug]/page/foo/bar/zoo.val.json`, importPath + `./page/foo/bar/zoo.val.json`. EXISTING entries use the analyzer's importPath (hybrid authoring). + - **Tested**: `ValOpsFS.jsonValues.test.ts` (content edit writes only json; add writes json + + thunk; remove nulls json + drops thunk). Whole `pnpm test` green (1056). + - Validation already handles inline content (`validateJsonValuesEntries` for base thunks; `executeValidate` validates inline content). `/sources/~` shallow markers already work (JSON.stringify drops the thunk). - **Last verified green**: core json suite (14 tests) + server `validateJsonValues`/loader/ @@ -153,17 +167,19 @@ entries; runtime/Studio/validation work one entry at a time; zero overhead when - [x] `patch/ts/ops.ts`: `createValJsonReference(importPath)` — emits `c.json(() => import("..."))` (factory-built; uses `createIdentifier("import")` to print a dynamic import without casting the ImportKeyword token). Tested in `jsonReference.test.ts`. ✅ -- [ ] `patch/ts/ops.ts` (remaining): wire add/remove of json entries through `insertAt`/`removeAt` - (+ write/delete `*.val.json`) — done with the ValOps commit flow below. (Content edits don't - touch the `.val.ts` — no sha to update.) +- [x] `patch/ts/ops.ts`: `insertValJsonEntry` / `removeValJsonEntry` insert/remove a + `c.json(() => import(...))` entry property on the record object literal (via + `insertAt`/`removeAt` + `createValJsonReference`). Tested (`jsonValuesEntry.test.ts`). ✅ - [x] **Per-entry validation**: `validateJsonValues.ts` (`validateJsonValuesEntries`) loads each entry's content via `getJsonImport` and validates against the item schema; wired into `ValOps.validateSources` (runs before the `res === false` early-continue). Tested in `validateJsonValues.test.ts` (valid/invalid/load-error/non-jsonValues-skip). ✅ -- [ ] `ValOps.ts` / `ValOpsFS.ts` / `ValOpsHttp.ts` (remaining): confirm shallow source - serialization on `/sources/~` (JSON.stringify already drops the thunk → `{_type}`); commit - writes `*.val.json` (content edits) + inserts/removes `c.json(...)` thunks for add/remove (use - `createValJsonReference` + `insertAt`/`removeAt`). +- [x] `ValOps.ts` commit flow: `prepare.applySourceFilePatches` routes ops via `classifyJsonValuesOp` + and writes `*.val.json` (content edits) + inserts/removes `c.json(...)` thunks for add/remove + through the existing `patchedSourceFiles` map (so `ValOpsFS`/`ValOpsHttp` commit them + unchanged). `.val.ts` is not written on pure content edits. Tested + (`ValOpsFS.jsonValues.test.ts`). ✅ (Still to confirm: shallow `/sources/~` serialization end to + end against the Studio.) - [x] Core eager resolver `Internal.resolveJsonValues(source)` (for `fetchVal`/`useVal`). ✅ - [ ] `ValServer.ts`: endpoint to fetch one entry's content (draft-aware via `patch_id`); `/sources/~` returns shallow markers for json records. @@ -238,6 +254,12 @@ unconditionally (accepted "validation takes more time" tradeoff). ## Changelog +- **Session 2 (2026-07-03)**: Commit flow landed. New `patch/jsonValuesPatch.ts` (op classifier + + path helpers), new `insertValJsonEntry`/`removeValJsonEntry` in `patch/ts/ops.ts`, and a rewritten + `ValOps.prepare.applySourceFilePatches` that routes ops into `*.val.json` writes/deletes + + `.val.ts` thunk insert/remove, skipping the `.val.ts` on pure content edits. Tests: + `jsonValuesPatch.test.ts`, `jsonValuesEntry.test.ts`, `ValOpsFS.jsonValues.test.ts`. Whole + `pnpm test` (1056) + `-r typecheck` green. - **Session 1**: Phase 1 (core) complete + tested; Phase 2 loader done + tested; `createValJsonReference` primitive done + tested. Whole monorepo typechecks (except pre-existing unrelated `packages/cli` chokidar error). `JsonSource` redesigned to a phantom-typed pure-JSON diff --git a/packages/server/src/ValOps.ts b/packages/server/src/ValOps.ts index 9c9f0b947..0a4e8e469 100644 --- a/packages/server/src/ValOps.ts +++ b/packages/server/src/ValOps.ts @@ -12,6 +12,7 @@ import { RemoteSource, Schema, SelectorSource, + SerializedSchema, Source, SourcePath, VAL_EXTENSION, @@ -21,10 +22,11 @@ import { ValidationErrors, extractValModules, } from "@valbuild/core"; -import { pipe, result } from "@valbuild/core/fp"; +import { array, pipe, result } from "@valbuild/core/fp"; import { JSONOps, JSONValue, + Operation, ParentRef, Patch, PatchError, @@ -32,8 +34,14 @@ import { applyPatch, deepClone, } from "@valbuild/core/patch"; -import { TSOps } from "./patch/ts/ops"; +import { TSOps, insertValJsonEntry, removeValJsonEntry } from "./patch/ts/ops"; import { analyzeValModule } from "./patch/ts/valModule"; +import { analyzeJsonValuesEntries } from "./patch/ts/jsonValuesModule"; +import { + classifyJsonValuesOp, + getNewJsonEntryPaths, + resolveExistingJsonPath, +} from "./patch/jsonValuesPatch"; import { validateJsonValuesEntries } from "./validateJsonValues"; import ts from "typescript"; import { ValSyntaxError, ValSyntaxErrorTree } from "./patch/ts/syntax"; @@ -71,6 +79,47 @@ const tsOps = new TSOps((document) => { ); }); +/** + * Rebases a patch op that targets a `.jsonValues()` entry's content so its paths + * are relative to the entry's `*.val.json` root (drops the record + entry-key + * prefix). Used to replay the op against the backing JSON file. + */ +function rebaseContentOp( + op: Operation, + prefixLen: number, +): result.Result { + const path = op.path.slice(prefixLen); + switch (op.op) { + case "add": + case "replace": + case "test": + return result.ok({ ...op, path }); + case "remove": { + if (!array.isNonEmpty(path)) { + return result.err( + new PatchError("Cannot remove the root of a jsonValues entry"), + ); + } + return result.ok({ ...op, path }); + } + case "move": { + const from = op.from.slice(prefixLen); + if (!array.isNonEmpty(from)) { + return result.err( + new PatchError("Cannot move from the root of a jsonValues entry"), + ); + } + return result.ok({ ...op, path, from }); + } + case "copy": + return result.ok({ ...op, path, from: op.from.slice(prefixLen) }); + case "file": + return result.err( + new PatchError("Cannot apply a file op to a jsonValues entry"), + ); + } +} + export type ValOpsOptions = { formatter?: (code: string, filePath: string) => string | Promise; statPollingInterval?: number; @@ -815,13 +864,21 @@ export abstract class ValOps { const patchedSourceFiles: Record = {}; const previousSourceFiles: Record = {}; + // Serialized schemas are needed to route ops that target `.jsonValues()` + // entries (their content lives in `*.val.json`, not the `.val.ts`). + const schemas = await this.getSchemas(); + const applySourceFilePatches = async ( path: ModuleFilePath, patches: { patchId: PatchId }[], ): Promise< | { path: ModuleFilePath; - result: string; + // `null` means the `.val.ts` itself was not changed (e.g. pure + // jsonValues content edits only touch `*.val.json`). + result: string | null; + // Extra files to write/delete (jsonValues `*.val.json` entries). + extraFiles: Record; appliedPatches: PatchId[]; errors?: undefined; } @@ -849,11 +906,93 @@ export abstract class ValOps { } const sourceFile = sourceFileRes.data; previousSourceFiles[path] = sourceFile; - let tsSourceFile = ts.createSourceFile( + const originalSourceFile = ts.createSourceFile( "", sourceFile, ts.ScriptTarget.ES2015, ); + let tsSourceFile = originalSourceFile; + let tsChanged = false; + const serializedSchema = schemas[path]?.["executeSerialize"](); + + // jsonValues entry content, keyed by `*.val.json` path. `null` = delete. + const jsonEntryContents = new Map(); + // Entries added in this commit → their new `*.val.json` path, so later + // content ops in the same commit resolve to the freshly-created file. + const entryKeyToJsonPath = new Map(); + + // Lazily analyzed `c.json(() => import("..."))` entries of the ORIGINAL + // `.val.ts` (import paths are authoritative for existing/hand-placed files). + let analyzerEntries: Map | null = null; + const resolveEntryJsonPath = ( + entryKey: string, + ): result.Result => { + const added = entryKeyToJsonPath.get(entryKey); + if (added !== undefined) { + return result.ok(added); + } + if (analyzerEntries === null) { + const analysis = analyzeValModule(originalSourceFile); + if (result.isErr(analysis)) { + return result.err(analysis.error); + } + analyzerEntries = analyzeJsonValuesEntries(analysis.value.source); + } + const entry = analyzerEntries.get(entryKey); + if (!entry) { + return result.err({ + message: `Could not find jsonValues entry '${entryKey}' in ${path}`, + filePath: path, + }); + } + return result.ok(resolveExistingJsonPath(path, entry.importPath)); + }; + const loadEntryContent = async ( + jsonPath: string, + ): Promise> => { + const current = jsonEntryContents.get(jsonPath); + if (current !== undefined) { + if (current === null) { + return result.err({ + message: `Cannot edit a removed jsonValues entry: ${jsonPath}`, + filePath: jsonPath, + }); + } + return result.ok(current); + } + const res = await this.getSourceFile(jsonPath as ModuleFilePath); + if (res.error) { + return result.err({ message: res.error.message, filePath: jsonPath }); + } + try { + const parsed: JSONValue = JSON.parse(res.data); + jsonEntryContents.set(jsonPath, parsed); + return result.ok(parsed); + } catch (err) { + return result.err({ + message: `Could not parse jsonValues entry ${jsonPath}: ${ + err instanceof Error ? err.message : String(err) + }`, + filePath: jsonPath, + }); + } + }; + const collectPatchError = ( + err: PatchError | ValSyntaxErrorTree, + patchId: PatchId, + op: unknown, + ) => { + console.error( + "Could not patch", + JSON.stringify({ path, patchId, error: err, op }, null, 2), + ); + if (Array.isArray(err)) { + errors.push(...err); + } else { + errors.push(err); + } + }; + const appliedPatches: PatchId[] = []; const triedPatches: PatchId[] = []; for (const { patchId } of patches) { @@ -868,81 +1007,185 @@ export abstract class ValOps { } const patch = patchData.patch; const sourceFileOps = patch.filter((op) => op.op !== "file"); // file is not a valid source file op - const patchRes = applyPatch(tsSourceFile, tsOps, sourceFileOps); - if (result.isErr(patchRes)) { - if (Array.isArray(patchRes.error)) { - for (const error of patchRes.error) { - console.error( - "Could not patch", - JSON.stringify( - { - path, - patchId, - error, - sourceFileOps, - }, - null, - 2, - ), + let patchHadError = false; + for (const op of sourceFileOps) { + const cls = serializedSchema + ? classifyJsonValuesOp(serializedSchema, op.path) + : ({ kind: "normal" } as const); + if (cls.kind === "normal") { + const patchRes = applyPatch(tsSourceFile, tsOps, [op]); + if (result.isErr(patchRes)) { + collectPatchError(patchRes.error, patchId, op); + patchHadError = true; + break; + } + tsSourceFile = patchRes.value; + tsChanged = true; + continue; + } + // The op targets a `.jsonValues()` entry. + if (cls.subPath.length === 0) { + // Structural / whole-entry op. + if (op.op === "add") { + const { jsonPath, importPath } = getNewJsonEntryPaths( + path, + cls.entryKey, + ); + const insRes = insertValJsonEntry( + tsSourceFile, + cls.recordPath, + cls.entryKey, + importPath, + ); + if (result.isErr(insRes)) { + collectPatchError(insRes.error, patchId, op); + patchHadError = true; + break; + } + tsSourceFile = insRes.value; + tsChanged = true; + jsonEntryContents.set(jsonPath, op.value); + entryKeyToJsonPath.set(cls.entryKey, jsonPath); + } else if (op.op === "remove") { + const jsonPathRes = resolveEntryJsonPath(cls.entryKey); + if (result.isErr(jsonPathRes)) { + errors.push(jsonPathRes.error); + patchHadError = true; + break; + } + const remRes = removeValJsonEntry( + tsSourceFile, + cls.recordPath, + cls.entryKey, ); + if (result.isErr(remRes)) { + collectPatchError(remRes.error, patchId, op); + patchHadError = true; + break; + } + tsSourceFile = remRes.value; + tsChanged = true; + jsonEntryContents.set(jsonPathRes.value, null); + } else if (op.op === "replace") { + const jsonPathRes = resolveEntryJsonPath(cls.entryKey); + if (result.isErr(jsonPathRes)) { + errors.push(jsonPathRes.error); + patchHadError = true; + break; + } + jsonEntryContents.set(jsonPathRes.value, op.value); + } else { + errors.push({ + message: `Unsupported op '${op.op}' on jsonValues entry '${cls.entryKey}'`, + filePath: path, + }); + patchHadError = true; + break; } - errors.push(...patchRes.error); } else { - console.error( - "Could not patch", - JSON.stringify( - { - path, - patchId, - error: patchRes.error, - sourceFileOps, - }, - null, - 2, - ), - ); - errors.push(patchRes.error); + // Content sub-op: replay against the entry's `*.val.json`. + const jsonPathRes = resolveEntryJsonPath(cls.entryKey); + if (result.isErr(jsonPathRes)) { + errors.push(jsonPathRes.error); + patchHadError = true; + break; + } + const jsonPath = jsonPathRes.value; + const contentRes = await loadEntryContent(jsonPath); + if (result.isErr(contentRes)) { + errors.push(contentRes.error); + patchHadError = true; + break; + } + const rebasedRes = rebaseContentOp(op, cls.recordPath.length + 1); + if (result.isErr(rebasedRes)) { + errors.push({ + message: rebasedRes.error.message, + filePath: jsonPath, + }); + patchHadError = true; + break; + } + const applied = applyPatch(deepClone(contentRes.value), jsonOps, [ + rebasedRes.value, + ]); + if (result.isErr(applied)) { + collectPatchError(applied.error, patchId, op); + patchHadError = true; + break; + } + jsonEntryContents.set(jsonPath, applied.value); } + } + if (patchHadError) { triedPatches.push(patchId); break; } appliedPatches.push(patchId); - tsSourceFile = patchRes.value; } if (errors.length === 0) { // https://github.com/microsoft/TypeScript/issues/36174 - let sourceFileText = unescape( - tsSourceFile.getText(tsSourceFile).replace(/\\u/g, "%u"), - ); - if (this.options?.formatter) { - try { - sourceFileText = await this.options.formatter(sourceFileText, path); - } catch (err) { - errors.push({ - message: - "Could not format source file: " + - (err instanceof Error ? err.message : "Unknown error"), - }); + let sourceFileText: string | null = null; + if (tsChanged) { + sourceFileText = unescape( + tsSourceFile.getText(tsSourceFile).replace(/\\u/g, "%u"), + ); + if (this.options?.formatter) { + try { + sourceFileText = await this.options.formatter( + sourceFileText, + path, + ); + } catch (err) { + errors.push({ + message: + "Could not format source file: " + + (err instanceof Error ? err.message : "Unknown error"), + }); + } } } - return { - path, - appliedPatches, - result: sourceFileText, - }; - } else { - const skippedPatches = patches - .slice(appliedPatches.length + triedPatches.length) - .map((p) => p.patchId); - - return { - path, - appliedPatches, - triedPatches, - skippedPatches, - errors, - }; + const extraFiles: Record = {}; + for (const [jsonPath, content] of Array.from(jsonEntryContents)) { + if (content === null) { + extraFiles[jsonPath] = null; + continue; + } + let jsonText = JSON.stringify(content, null, 2); + if (this.options?.formatter) { + try { + jsonText = await this.options.formatter(jsonText, jsonPath); + } catch (err) { + errors.push({ + message: + "Could not format jsonValues entry: " + + (err instanceof Error ? err.message : "Unknown error"), + filePath: jsonPath, + }); + } + } + extraFiles[jsonPath] = jsonText; + } + if (errors.length === 0) { + return { + path, + appliedPatches, + result: sourceFileText, + extraFiles, + }; + } } + const skippedPatches = patches + .slice(appliedPatches.length + triedPatches.length) + .map((p) => p.patchId); + + return { + path, + appliedPatches, + triedPatches, + skippedPatches, + errors, + }; }; const allResults = await Promise.all( Object.entries(patchesByModule).map(([path, patches]) => @@ -966,7 +1209,14 @@ export abstract class ValOps { triedPatches[res.path] = res.triedPatches ?? []; skippedPatches[res.path] = res.skippedPatches ?? []; } else { - patchedSourceFiles[res.path] = res.result; + // `result` is null when the `.val.ts` itself was not changed (pure + // jsonValues content edits only write `*.val.json` extraFiles). + if (res.result !== null) { + patchedSourceFiles[res.path] = res.result; + } + for (const [extraPath, data] of Object.entries(res.extraFiles)) { + patchedSourceFiles[extraPath] = data; + } appliedPatches[res.path] = res.appliedPatches ?? []; } for (const patchId of res.appliedPatches ?? []) { diff --git a/packages/server/src/ValOpsFS.jsonValues.test.ts b/packages/server/src/ValOpsFS.jsonValues.test.ts new file mode 100644 index 000000000..336f8ffae --- /dev/null +++ b/packages/server/src/ValOpsFS.jsonValues.test.ts @@ -0,0 +1,152 @@ +import { ModuleFilePath, PatchId, initVal } from "@valbuild/core"; +import { Script } from "node:vm"; +import { transform } from "sucrase"; +import { ValOpsFS } from "./ValOpsFS"; +import fs from "fs"; +import os from "node:os"; +import path from "node:path"; +import synchronizedPrettier from "@prettier/sync"; +import type { OrderedPatches } from "./ValOps"; + +const MODULE_PATH = "/test/pages.val.ts" as ModuleFilePath; + +const MODULE_CODE = ` +import { s, c } from "val.config"; + +export default c.define( + "/test/pages.val.ts", + s.record(s.object({ title: s.string(), order: s.number() })).jsonValues(), + { + "/blog/hello": c.json(() => import("./content/hello.val.json")), + "/blog/world": c.json(() => import("./content/world.val.json")), + } +); +`; + +const JSON_FILES: Record = { + "/test/content/hello.val.json": { title: "Hello", order: 1 }, + "/test/content/world.val.json": { title: "World", order: 2 }, +}; + +function setup() { + const { s, c, config } = initVal(); + const evalModule = (code: string) => + new Script( + transform(code, { transforms: ["imports"] }).code, + ).runInNewContext({ + exports: {}, + require: (p: string) => { + if (p === "val.config") { + return { s, c, config }; + } + // eslint-disable-next-line @typescript-eslint/no-require-imports + return require(p); + }, + module: { exports: {} }, + }); + + // Use the OS temp dir (NOT the repo-local ".tmp", which other suites wipe). + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "val-jsonvalues-test")); + + const moduleAbs = path.join(rootDir, MODULE_PATH); + fs.mkdirSync(path.dirname(moduleAbs), { recursive: true }); + fs.writeFileSync( + moduleAbs, + synchronizedPrettier.format(MODULE_CODE, { parser: "typescript" }), + ); + for (const [filePath, content] of Object.entries(JSON_FILES)) { + const absPath = path.join(rootDir, filePath); + fs.mkdirSync(path.dirname(absPath), { recursive: true }); + fs.writeFileSync(absPath, JSON.stringify(content, null, 2)); + } + + const contentHost = process.env.VAL_CONTENT_URL || "http://localhost:4000"; + const ops = new ValOpsFS( + contentHost, + rootDir, + { + config, + modules: [{ def: async () => ({ default: evalModule(MODULE_CODE) }) }], + }, + { + formatter: (code, filePath) => + synchronizedPrettier.format(code, { filepath: filePath }), + config, + }, + ); + return { ops, rootDir }; +} + +async function prepareSingle( + ops: ValOpsFS, + patch: OrderedPatches["patches"][number]["patch"], +) { + const patches: OrderedPatches["patches"] = [ + { + path: MODULE_PATH, + patchId: crypto.randomUUID() as PatchId, + patch, + createdAt: new Date().toISOString(), + authorId: null, + baseSha: await ops.getBaseSha(), + appliedAt: null, + }, + ]; + const analysis = ops.analyzePatches(patches); + return ops.prepare({ ...analysis, patches }); +} + +describe("ValOpsFS jsonValues commit flow", () => { + test("content edit writes only the *.val.json (not the .val.ts)", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { op: "replace", path: ["/blog/hello", "title"], value: "Hello!" }, + ]); + expect(pc.hasErrors).toBe(false); + // .val.ts is untouched on a pure content edit + expect(pc.patchedSourceFiles[MODULE_PATH]).toBeUndefined(); + const written = pc.patchedSourceFiles["/test/content/hello.val.json"]; + expect(typeof written).toBe("string"); + expect(JSON.parse(written as string)).toEqual({ + title: "Hello!", + order: 1, + }); + // the untouched entry is not written + expect( + pc.patchedSourceFiles["/test/content/world.val.json"], + ).toBeUndefined(); + }); + + test("add entry writes a new *.val.json and inserts a c.json thunk", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { + op: "add", + path: ["/blog/new"], + value: { title: "New", order: 3 }, + }, + ]); + expect(pc.hasErrors).toBe(false); + const newJson = pc.patchedSourceFiles["/test/pages/blog/new.val.json"]; + expect(typeof newJson).toBe("string"); + expect(JSON.parse(newJson as string)).toEqual({ title: "New", order: 3 }); + const ts = pc.patchedSourceFiles[MODULE_PATH]; + expect(typeof ts).toBe("string"); + expect(ts).toContain(`import("./pages/blog/new.val.json")`); + expect(ts).toContain(`"/blog/new"`); + }); + + test("remove entry deletes the *.val.json and drops the thunk", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { op: "remove", path: ["/blog/world"] }, + ]); + expect(pc.hasErrors).toBe(false); + // null signals a file deletion in the commit loop + expect(pc.patchedSourceFiles["/test/content/world.val.json"]).toBeNull(); + const ts = pc.patchedSourceFiles[MODULE_PATH]; + expect(typeof ts).toBe("string"); + expect(ts).not.toContain(`"/blog/world"`); + expect(ts).toContain(`"/blog/hello"`); + }); +}); diff --git a/packages/server/src/patch/jsonValuesPatch.test.ts b/packages/server/src/patch/jsonValuesPatch.test.ts new file mode 100644 index 000000000..467cf1a6d --- /dev/null +++ b/packages/server/src/patch/jsonValuesPatch.test.ts @@ -0,0 +1,93 @@ +import { initVal, type SerializedSchema } from "@valbuild/core"; +import { + classifyJsonValuesOp, + getNewJsonEntryPaths, + resolveExistingJsonPath, +} from "./jsonValuesPatch"; + +const { s } = initVal(); + +describe("classifyJsonValuesOp", () => { + const rootJsonValues: SerializedSchema = s + .record(s.object({ title: s.string(), order: s.number() })) + .jsonValues() + ["executeSerialize"](); + + test("field edit inside a root jsonValues entry → content sub-op", () => { + const cls = classifyJsonValuesOp(rootJsonValues, ["/blog/hello", "title"]); + expect(cls).toEqual({ + kind: "entry", + recordPath: [], + entryKey: "/blog/hello", + subPath: ["title"], + }); + }); + + test("add/remove of an entry key → entry op with empty subPath", () => { + expect(classifyJsonValuesOp(rootJsonValues, ["/blog/hello"])).toEqual({ + kind: "entry", + recordPath: [], + entryKey: "/blog/hello", + subPath: [], + }); + }); + + test("nested jsonValues record under an object carries recordPath", () => { + const nested: SerializedSchema = s + .object({ + pages: s.record(s.object({ title: s.string() })).jsonValues(), + }) + ["executeSerialize"](); + expect(classifyJsonValuesOp(nested, ["pages", "/a/b", "title"])).toEqual({ + kind: "entry", + recordPath: ["pages"], + entryKey: "/a/b", + subPath: ["title"], + }); + }); + + test("plain (non-jsonValues) record → normal", () => { + const plain: SerializedSchema = s + .record(s.object({ title: s.string() })) + ["executeSerialize"](); + expect(classifyJsonValuesOp(plain, ["/blog/hello", "title"])).toEqual({ + kind: "normal", + }); + }); + + test("op that does not reach the jsonValues record → normal", () => { + const nested: SerializedSchema = s + .object({ + title: s.string(), + pages: s.record(s.object({ title: s.string() })).jsonValues(), + }) + ["executeSerialize"](); + expect(classifyJsonValuesOp(nested, ["title"])).toEqual({ kind: "normal" }); + }); +}); + +describe("getNewJsonEntryPaths", () => { + test("mirrors the entry key under a folder named after the .val.ts", () => { + expect(getNewJsonEntryPaths("/test/pages.val.ts", "/blog/hello")).toEqual({ + jsonPath: "/test/pages/blog/hello.val.json", + importPath: "./pages/blog/hello.val.json", + }); + }); + + test("handles nested module directories", () => { + expect( + getNewJsonEntryPaths("/app/support/[slug]/page.val.ts", "/support/faq"), + ).toEqual({ + jsonPath: "/app/support/[slug]/page/support/faq.val.json", + importPath: "./page/support/faq.val.json", + }); + }); +}); + +describe("resolveExistingJsonPath", () => { + test("resolves a hand-placed import path relative to the module dir", () => { + expect( + resolveExistingJsonPath("/test/pages.val.ts", "./content/hello.val.json"), + ).toBe("/test/content/hello.val.json"); + }); +}); diff --git a/packages/server/src/patch/jsonValuesPatch.ts b/packages/server/src/patch/jsonValuesPatch.ts new file mode 100644 index 000000000..b902c3461 --- /dev/null +++ b/packages/server/src/patch/jsonValuesPatch.ts @@ -0,0 +1,121 @@ +import * as path from "path"; +import type { SerializedSchema } from "@valbuild/core"; + +/** + * Classification of a single patch op against a module's serialized schema, + * used by the commit flow to route ops for `.jsonValues()` records: + * + * - `normal`: the op does not descend into a `.jsonValues()` entry; apply it to + * the `.val.ts` as usual. + * - `entry`: the op targets a `.jsonValues()` entry. `recordPath` is the path to + * the record within the module source (empty for a root record/router), + * `entryKey` is the entry key, and `subPath` is the remaining path inside the + * entry (empty when the op targets the entry value itself, e.g. add/remove of + * the whole entry). + */ +export type JsonValuesOpClass = + | { kind: "normal" } + | { + kind: "entry"; + recordPath: string[]; + entryKey: string; + subPath: string[]; + }; + +/** + * Walks the serialized schema following the op path. When a `.jsonValues()` + * record is encountered, the next path segment is the entry key and everything + * after it lives inside the entry's `*.val.json` (so it does not touch the + * `.val.ts`). Returns `{ kind: "normal" }` when the op never enters a + * `.jsonValues()` record. + */ +export function classifyJsonValuesOp( + schema: SerializedSchema, + opPath: string[], +): JsonValuesOpClass { + let current: SerializedSchema | undefined = schema; + const recordPath: string[] = []; + for (let i = 0; i < opPath.length; i++) { + if (!current) { + return { kind: "normal" }; + } + if (current.type === "record" && current.jsonValues) { + return { + kind: "entry", + recordPath: recordPath.slice(), + entryKey: opPath[i], + subPath: opPath.slice(i + 1), + }; + } + const seg = opPath[i]; + current = descend(current, seg); + recordPath.push(seg); + } + return { kind: "normal" }; +} + +function descend( + schema: SerializedSchema, + key: string, +): SerializedSchema | undefined { + switch (schema.type) { + case "object": + return schema.items[key]; + case "record": + return schema.item; + case "array": + return schema.item; + default: + // Unions / primitives / leaf schemas: we cannot (or need not) descend + // further to find a jsonValues record. Anything below is a normal + // `.val.ts` edit. + return undefined; + } +} + +/** + * The `.val.ts` suffix a module file path ends with. Stripping it yields the + * folder that a new entry's `*.val.json` files are nested under. + */ +const VAL_TS_SUFFIX = ".val.ts"; + +/** + * Computes the `*.val.json` file path (relative to rootDir) and the `import(...)` + * path (relative to the module's directory) for a NEW `.jsonValues()` entry, + * following the locked filename convention: the file mirrors the entry key under + * a folder named after the `.val.ts` (its `.val.ts` suffix becomes the folder). + * + * For module `/app/support/[slug]/page.val.ts` and key `/support/faq`: + * - jsonPath: `/app/support/[slug]/page/support/faq.val.json` + * - importPath: `./page/support/faq.val.json` + */ +export function getNewJsonEntryPaths( + moduleFilePath: string, + entryKey: string, +): { jsonPath: string; importPath: string } { + const base = moduleFilePath.endsWith(VAL_TS_SUFFIX) + ? moduleFilePath.slice(0, -VAL_TS_SUFFIX.length) + : moduleFilePath; + const keyRel = entryKey.replace(/^\//, ""); + const jsonPath = `${base}/${keyRel}.val.json`; + const moduleDir = path.posix.dirname(moduleFilePath); + let importPath = path.posix.relative(moduleDir, jsonPath); + if (!importPath.startsWith(".")) { + importPath = `./${importPath}`; + } + return { jsonPath, importPath }; +} + +/** + * Resolves an EXISTING entry's `*.val.json` path (relative to rootDir) from the + * `import(...)` path recorded in the `.val.ts` thunk (from + * {@link analyzeJsonValuesEntries}). Existing files may have been hand-placed, + * so the import path is authoritative (hybrid authoring). + */ +export function resolveExistingJsonPath( + moduleFilePath: string, + importPath: string, +): string { + const moduleDir = path.posix.dirname(moduleFilePath); + return path.posix.join(moduleDir, importPath); +} diff --git a/packages/server/src/patch/ts/jsonValuesEntry.test.ts b/packages/server/src/patch/ts/jsonValuesEntry.test.ts new file mode 100644 index 000000000..37227a4b9 --- /dev/null +++ b/packages/server/src/patch/ts/jsonValuesEntry.test.ts @@ -0,0 +1,72 @@ +import ts from "typescript"; +import { result } from "@valbuild/core/fp"; +import { insertValJsonEntry, removeValJsonEntry } from "./ops"; + +const MODULE = `import { s, c } from "../val.config"; + +export default c.define( + "/test/pages.val.ts", + s.record(s.object({ title: s.string() })).jsonValues(), + { + "/blog/hello": c.json(() => import("./content/hello.val.json")), + "/blog/world": c.json(() => import("./content/world.val.json")), + }, +); +`; + +function parse(src: string): ts.SourceFile { + return ts.createSourceFile("", src, ts.ScriptTarget.ES2015); +} + +function print(node: ts.SourceFile): string { + return node.getText(node); +} + +describe("insertValJsonEntry", () => { + test("appends a c.json(() => import(...)) property to the root record", () => { + const res = insertValJsonEntry( + parse(MODULE), + [], + "/blog/new", + "./pages/blog/new.val.json", + ); + if (result.isErr(res)) { + throw res.error; + } + const out = print(res.value); + expect(out).toContain( + `"/blog/new": c.json(() => import("./pages/blog/new.val.json"))`, + ); + // existing entries are untouched + expect(out).toContain( + `"/blog/hello": c.json(() => import("./content/hello.val.json"))`, + ); + }); + + test("fails when the entry key already exists", () => { + const res = insertValJsonEntry( + parse(MODULE), + [], + "/blog/hello", + "./pages/blog/hello.val.json", + ); + expect(result.isErr(res)).toBe(true); + }); +}); + +describe("removeValJsonEntry", () => { + test("removes an existing entry property", () => { + const res = removeValJsonEntry(parse(MODULE), [], "/blog/hello"); + if (result.isErr(res)) { + throw res.error; + } + const out = print(res.value); + expect(out).not.toContain(`"/blog/hello"`); + expect(out).toContain(`"/blog/world"`); + }); + + test("fails when the entry key does not exist", () => { + const res = removeValJsonEntry(parse(MODULE), [], "/blog/missing"); + expect(result.isErr(res)).toBe(true); + }); +}); diff --git a/packages/server/src/patch/ts/ops.ts b/packages/server/src/patch/ts/ops.ts index 6bae146c3..a4c9d3f51 100644 --- a/packages/server/src/patch/ts/ops.ts +++ b/packages/server/src/patch/ts/ops.ts @@ -16,6 +16,7 @@ import { findValRemoteNodeArg, findValRemoteMetadataArg, } from "./syntax"; +import { analyzeValModule } from "./valModule"; import { deepEqual, isNotRoot, @@ -158,6 +159,110 @@ export function createValJsonReference(importPath: string): ts.Expression { ); } +/** + * Inserts a new `.jsonValues()` entry `"": c.json(() => import(""))` + * into the record's object literal in a `.val.ts` module. `recordPath` is the + * path from the module source to the record (empty for a root record/router). + * Fails if the entry key already exists. + */ +export function insertValJsonEntry( + document: ts.SourceFile, + recordPath: string[], + key: string, + importPath: string, +): TSOpsResult { + return pipe( + analyzeValModule(document), + result.flatMap(({ source }) => getAtPath(source, recordPath)), + result.flatMap((recordNode: ts.Expression): TSOpsResult => { + if (!ts.isObjectLiteralExpression(recordNode)) { + return result.err( + new PatchError( + "Cannot add jsonValues entry: record source is not an object literal", + ), + ); + } + return pipe( + findObjectPropertyAssignment(recordNode, key), + result.flatMap( + ( + assignment: ts.PropertyAssignment | undefined, + ): TSOpsResult => { + if (assignment) { + return result.err( + new PatchError( + `Cannot add jsonValues entry '${key}': it already exists`, + ), + ); + } + const property = ts.factory.createPropertyAssignment( + isValidIdentifier(key) + ? ts.factory.createIdentifier(key) + : ts.factory.createStringLiteral(key), + createValJsonReference(importPath), + ); + return result.ok( + insertAt( + document, + recordNode.properties, + recordNode.properties.length, + property, + ), + ); + }, + ), + ); + }), + ); +} + +/** + * Removes the `.jsonValues()` entry `""` from the record's object literal + * in a `.val.ts` module. `recordPath` is the path from the module source to the + * record (empty for a root record/router). Fails if the entry key is missing. + */ +export function removeValJsonEntry( + document: ts.SourceFile, + recordPath: string[], + key: string, +): TSOpsResult { + return pipe( + analyzeValModule(document), + result.flatMap(({ source }) => getAtPath(source, recordPath)), + result.flatMap((recordNode: ts.Expression): TSOpsResult => { + if (!ts.isObjectLiteralExpression(recordNode)) { + return result.err( + new PatchError( + "Cannot remove jsonValues entry: record source is not an object literal", + ), + ); + } + return pipe( + findObjectPropertyAssignment(recordNode, key), + result.flatMap( + ( + assignment: ts.PropertyAssignment | undefined, + ): TSOpsResult => { + if (!assignment) { + return result.err( + new PatchError( + `Cannot remove jsonValues entry '${key}': it does not exist`, + ), + ); + } + const [doc] = removeAt( + document, + recordNode.properties, + recordNode.properties.indexOf(assignment), + ); + return result.ok(doc); + }, + ), + ); + }), + ); +} + function toExpression(value: JSONValue): ts.Expression { if (typeof value === "string") { // TODO: Use configuration/heuristics to determine use of single quote or double quote From 22729d053c28c3ace022102810b5b58f0082a360 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Sun, 5 Jul 2026 11:20:46 +0200 Subject: [PATCH 23/35] Enhance jsonValues support: implement fetchValRoute for dynamic entry loading and add isJsonValuesRecordSchema check --- docs/plans/jsonValues.md | 41 +++++++---- examples/next/app/support/[slug]/page.tsx | 9 ++- packages/next/src/client/initValClient.ts | Bin 7598 -> 9364 bytes packages/next/src/routeFromVal.ts | 12 ++++ packages/next/src/rsc/initValRsc.ts | 81 +++++++++++++++++----- 5 files changed, 110 insertions(+), 33 deletions(-) diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md index 03b3ad36c..5bd5172aa 100644 --- a/docs/plans/jsonValues.md +++ b/docs/plans/jsonValues.md @@ -26,18 +26,20 @@ - **Phase**: 1 ✅. Phase 2 server: validation + loader + emit primitive + **/json endpoint** + **commit flow** ✅. Phase 3 UI lazy-load ✅ (Studio reads jsonValues entries). Phase 4: - `fetchValKey`/`useValKey` ✅ (production path). Example (support pages) added + typechecks. + `fetchValKey`/`useValKey` + **`fetchValRoute`/`useValRoute`** ✅ (production path). Example + (support pages) now uses `fetchValRoute`; `examples/next` `next build` green. **The `c.json` sha was removed (2026-07-02).** Remaining: end-to-end Studio verify of add/remove/ - edit against a running dev server; `fetchValRoute`/`useValRoute` key path; Enabled/Studio draft - runtime path; Phase 5 CI gate (example `.jsonValues()` router + `examples/next` build). + edit against a running dev server; **Enabled/Studio draft runtime path** (all the single-entry read + APIs still read the committed local thunk, not draft edits); full Phase 5 CI gate run. - **Single-entry runtime read API (typecheck-validated, runtime-validation via example pending)**: - RSC `fetchValKey` — `initFetchValKeyStega` in `next/src/rsc/initValRsc.ts` (returned as `fetchValKeyStega`). Resolves ONE entry by key from the local module's thunk + stega-encodes. - Client `useValKey` — `useValKeyStega` in `next/src/client/initValClient.ts` (returned as `useValKeyStega`). Uses a module-level promise cache + `React.use` (Suspense) to load one entry. + - `fetchValRoute`/`useValRoute` now also load a single entry for `.jsonValues()` routers (map + params → key, then resolve one entry) — see Phase 4. ✅ - Both: production path resolves the local thunk; Enabled/Studio **draft** path is a TODO (needs - the single-entry endpoint + a sub-selector for stega edit tags). Still TODO: make `fetchValRoute`/ - `useValRoute` use the key path for jsonValues routers (load one). + the single-entry endpoint + a sub-selector for stega edit tags). - **Commit flow — DONE (2026-07-03)**. Persists edits to `*.val.json` instead of the `.val.ts`. Implementation landed: - **Key enabler**: `ValOps.prepare`'s `patchedSourceFiles: Record` is written by @@ -219,17 +221,26 @@ JSONValue>>` and `private loadingJsonEntries: Set<"mfp\0key">`. Add `requestJson - [x] `rsc/initValRsc.ts`: `fetchValKey` (`initFetchValKeyStega`, returned as `fetchValKeyStega`). - [x] `client/initValClient.ts`: `useValKey` (`useValKeyStega`, promise cache + `React.use`). -- [x] Example wires `fetchValKey` (support pages) — typechecks clean. -- [ ] `fetchValRoute` / `useValRoute`: use the key path for jsonValues routers (load one entry by - matching the route), instead of the eager fetchVal. (Reuse `ValRouter` to map route→key.) +- [x] `fetchValRoute` / `useValRoute`: jsonValues-aware. When the module schema is a `.jsonValues()` + record (`isJsonValuesRecordSchema` in `routeFromVal.ts`), map params → the entry key via + `getValRouteUrlFromVal` (passing the LOCAL source markers as the guard `val`), then load ONLY + that entry (RSC: `loadJsonEntryContent`; client: same `React.use` + `jsonEntryPromiseCache` as + `useValKey`). Non-jsonValues routers keep the eager `fetchVal`/`useValStega` path. Return types + gained a `NonNullable[string] extends JsonSource ? C | null : …` branch. ✅ +- [x] Example wires `fetchValRoute` (support pages) — `next build` green; `/support/[slug]` is a + single-entry dynamic route. (Was `fetchValKey`.) ✅ - [ ] Enabled/Studio draft path: resolve draft content via `/json` (+ sub-selector stega tags). + (fetchValKey/useValKey + fetchValRoute/useValRoute all still read the LOCAL committed thunk on + the enabled/draft path — draft edits aren't reflected until commit.) ## Phase 5 — Example + CI gate -- [ ] Add a `.jsonValues()` router to `examples/next` with a few `*.val.json` entries. -- [ ] `cd examples/next && pnpm run build` green; confirm single-entry import in output. -- [ ] Full CI: `pnpm run lint`, `pnpm -w run format`, `pnpm run -r typecheck`, `pnpm test`, - `pnpm run build`, `cd examples/next && pnpm run build`. +- [x] Add a `.jsonValues()` router to `examples/next` with a few `*.val.json` entries + (`app/support/[slug]/page.val.ts` + `content/*.val.json`); consumed via `fetchValRoute`. +- [x] `cd examples/next && pnpm run build` green (`/support/[slug]` dynamic route builds). +- [ ] Full CI in one pass: `pnpm run lint`, `pnpm -w run format`, `pnpm run -r typecheck`, + `pnpm test`, `pnpm run build` (root preconstruct+ui; remember `pnpm preconstruct dev` after), + `cd examples/next && pnpm run build`. --- @@ -254,6 +265,12 @@ unconditionally (accepted "validation takes more time" tradeoff). ## Changelog +- **Session 3 (2026-07-03)**: `fetchValRoute`/`useValRoute` made jsonValues-aware — they now map + route params → the entry key and load ONLY the matched entry (RSC `loadJsonEntryContent`; client + reuses the `useValKey` `React.use` cache), instead of eagerly resolving the whole record. Added + `isJsonValuesRecordSchema` to `routeFromVal.ts`; return types gained the `JsonSource ? C` branch. + Example support page switched to `fetchValRoute`; `examples/next` `next build` green. `-r typecheck` + - next tests green. - **Session 2 (2026-07-03)**: Commit flow landed. New `patch/jsonValuesPatch.ts` (op classifier + path helpers), new `insertValJsonEntry`/`removeValJsonEntry` in `patch/ts/ops.ts`, and a rewritten `ValOps.prepare.applySourceFilePatches` that routes ops into `*.val.json` writes/deletes + diff --git a/examples/next/app/support/[slug]/page.tsx b/examples/next/app/support/[slug]/page.tsx index 60668a5cd..044196b35 100644 --- a/examples/next/app/support/[slug]/page.tsx +++ b/examples/next/app/support/[slug]/page.tsx @@ -1,4 +1,4 @@ -import { fetchValKey } from "../../../val/rsc"; +import { fetchValRoute } from "../../../val/rsc"; import supportVal from "./page.val"; export default async function SupportPage({ @@ -6,10 +6,9 @@ export default async function SupportPage({ }: { params: Promise<{ slug: string }>; }) { - const { slug } = await params; - // Loads ONLY this slug's backing *.val.json (one dynamic import), not the - // whole support-pages record. - const page = await fetchValKey(supportVal, `/support/${slug}`); + // Maps the route params to the matching entry and loads ONLY that entry's + // backing *.val.json (one dynamic import), not the whole support-pages record. + const page = await fetchValRoute(supportVal, params); if (!page) { return
Support page not found.
; } diff --git a/packages/next/src/client/initValClient.ts b/packages/next/src/client/initValClient.ts index bb7816ef5431f6c78b4cd648a94321a0d81a5800..1f7fa1fe072412c9004ba50c468843ae5e70131d 100644 GIT binary patch delta 838 zcmaKq&ubGw6vs(hYHii-raw%ZQl8MtuEqqUh@_+ugn~b?U_r1bWSX72Yi4)i&a5TW zP){Q0MW5iwlb%J$L8#)fcdy>`*8jpc*)#=3=Q7NjH=pl!zB3<}-(36jA~P@wDD%^U z1)9I-Cg*8}KwVcdy2C`m)oosvvA16LdB_T*ozIys_IjxVwc^8mi{sWHmWk$Z8MMzK zWSZJN;E|475OXEPW3E6;2($uo4(c+}v|S{zbl}s8X?#28*zg3RM2P-Ix$|fu9Kv_w z=XXi=DieapNkjlI(1DDo&@$)-=S}dXYyjEiF(}P81uw#?jDh`Wm*@mbJ$Fr!$wX+n zIH40o3eGk^bWX=r5Iqp5dGAJ>T*z$*BBeR=+#>SX6|{1D!%e1rsNi1yTy~ih$fDAq zy2(-bAn&f5@_)h$SreKpW}$*#&t7-4Ofzio7O= import(...))` marker. Used by the + * route helpers to load ONLY the matched entry instead of the whole record. + */ +export function isJsonValuesRecordSchema(schema: unknown): boolean { + return ( + schema instanceof RecordSchema && + schema["executeSerialize"]().jsonValues === true + ); +} + export function getValRouteUrlFromVal( resolvedParams: Record | unknown, methodName: string, diff --git a/packages/next/src/rsc/initValRsc.ts b/packages/next/src/rsc/initValRsc.ts index 1fe1c9789..c4c548bbc 100644 --- a/packages/next/src/rsc/initValRsc.ts +++ b/packages/next/src/rsc/initValRsc.ts @@ -20,7 +20,11 @@ import { cookies, draftMode, headers } from "next/headers"; import { VAL_SESSION_COOKIE } from "@valbuild/shared/internal"; import { createValServer, ValServer } from "@valbuild/server"; import { VERSION } from "../version"; -import { getValRouteUrlFromVal, initValRouteFromVal } from "../routeFromVal"; +import { + getValRouteUrlFromVal, + initValRouteFromVal, + isJsonValuesRecordSchema, +} from "../routeFromVal"; SET_RSC(true); const initFetchValStega = @@ -174,7 +178,10 @@ type FetchValRouteReturnType< > = T extends ValModule ? S extends SourceObject - ? StegaOfSource[string]> | null + ? // `.jsonValues()` router: the matched entry resolves to its json content. + NonNullable[string] extends JsonSource + ? C | null + : StegaOfSource[string]> | null : never : never; @@ -198,6 +205,36 @@ const initFetchValRouteStega = | Record | unknown, ): Promise> => { + const resolvedParams = await Promise.resolve(params); + const path = selector && Internal.getValPath(selector); + const schema = selector && Internal.getSchema(selector); + // `.jsonValues()` router: map params → the entry key and load ONLY that + // entry's backing `*.val.json`, instead of eagerly resolving the whole + // record via `fetchVal`. + if (isJsonValuesRecordSchema(schema)) { + const source = selector && Internal.getSource(selector); + const url = getValRouteUrlFromVal( + resolvedParams, + "fetchValRoute", + path, + schema, + source, + ); + if (!url) { + return null as FetchValRouteReturnType; + } + const content = await loadJsonEntryContent(source, url); + if (content === undefined) { + return null as FetchValRouteReturnType; + } + let enabled = false; + try { + enabled = await isEnabled(); + } catch { + // not in a server context where draftMode is readable — treat as disabled + } + return stegaEncode(content, { disabled: !enabled }); + } const fetchVal = initFetchValStega( config, valApiEndpoints, @@ -206,9 +243,6 @@ const initFetchValRouteStega = getHeaders, getCookies, ); - const resolvedParams = await Promise.resolve(params); - const path = selector && Internal.getValPath(selector); - const schema = selector && Internal.getSchema(selector); const val = selector && (await fetchVal(selector)); const route = initValRouteFromVal( resolvedParams, @@ -220,6 +254,29 @@ const initFetchValRouteStega = return route; }; +/** + * Resolves a single `.jsonValues()` entry's content by key from a module's local + * source markers (one dynamic import). Returns `undefined` when the key is + * missing or its marker has no runtime thunk (transport marker / draft entry). + */ +async function loadJsonEntryContent( + source: unknown, + key: string, +): Promise { + if (!source || typeof source !== "object") { + return undefined; + } + const marker = (source as Record)[key]; + if (!Internal.isJson(marker)) { + return undefined; + } + const thunk = Internal.getJsonImport(marker); + if (!thunk) { + return undefined; + } + return (await thunk()).default; +} + // The (loosened) content type a single `.jsonValues()` entry resolves to. type JsonEntryContentOf>> = T extends ValModule @@ -259,19 +316,11 @@ const initFetchValKeyStega = // not in a server context where draftMode is readable — treat as disabled } const source = selector && Internal.getSource(selector); - if (!source || typeof source !== "object") { - return undefined; - } - const marker = (source as Record)[key]; - if (!Internal.isJson(marker)) { - return undefined; - } - const thunk = Internal.getJsonImport(marker); - if (!thunk) { - // transport marker without a runtime thunk — see TODO above + const content = await loadJsonEntryContent(source, key); + if (content === undefined) { + // missing key, or transport marker without a runtime thunk — see TODO above return undefined; } - const content = (await thunk()).default; return stegaEncode(content, { disabled: !enabled, }); From b62b9a2ffd735f8d5992029e0131b0d825e71fea Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Mon, 27 Jul 2026 09:46:16 +0200 Subject: [PATCH 24/35] jsonValues: fix Studio round-trip (rename, publish, error state, nesting) Closes the Phase 3 defects for `.jsonValues()`, none of which were covered by tests. Rename/duplicate now commit. `ValOps.prepare` classified only `op.path`, so the `move` the Studio emits when renaming a record key fell through to "Unsupported op 'move' on jsonValues entry". It now classifies `op.from` too and has `move`/`copy` arms: load the source entry's content, drop the old thunk (move only), insert the new one at the generated path, and null the old file. Cross-record and cross-entry move/copy are rejected rather than silently corrupting -- `rebaseContentOp` sliced `from` by the destination's prefix without checking the two matched. Published edits no longer appear to revert. `jsonEntryContents` was cleared only on init/reset, so the pre-edit content was re-substituted after publish. Invalidating on the /sources/~ refresh is not enough: `publish()` never refetches sources, and a content-only edit leaves the module source (bare markers) byte-identical so `sourcesSha` never changes. Publish now invalidates its own entries. The stale flag is cleared when a request STARTS, not on success, so an invalidation landing mid-flight wins over the in-flight response. A failed /json load no longer spins forever. `jsonEntryErrors` memoizes the failure -- stopping the refetch-on-every-remount -- and the field hooks render an error instead of a permanent loading state. Adds `retryJsonEntry` and `ensureJsonEntry`; the rename flow awaits the latter so the patch carries real content rather than an opaque marker that would 404 on the new key. Nested `.jsonValues()` is now rejected at startup. It was supported by the op classifier but broken in the /json endpoint, the Studio substitution, and validation -- where it silently skipped content validation entirely. `findNestedJsonValuesRecords` feeds a ModulesError from `initSources`, so /sources/~ fails naming the module. The classifier still reports `recordPath` truthfully; the consumers reject. Also fixes a pre-existing bug found on the way: `analyzePatches` pushed one `patchesByModule` entry per non-file op, so `prepare` re-applied the whole patch once per op. Idempotent for `replace` (which is why no test caught it), but it duplicates `add`s and breaks `move`. Locked: a rename relocates to the generated convention path, so renaming a hand-placed file moves it out of its directory. Root-only is decision #7. Tests: +7 ValOpsFS.jsonValues, +6 jsonValuesPatch, +1 validateJsonValues pinning the root-only contract, and a new jsonValues suite in ValSyncEngine.test.ts (previously zero coverage) with /json and /save added to the mock client. Full gate green: lint, format, -r typecheck, 1078 tests, root build, examples/next build. The manual Studio walkthrough (V1-V9 in the tracker) has NOT been run. Co-Authored-By: Claude Opus 5 --- docs/plans/jsonValues.md | 68 ++++- packages/core/src/schema/record.ts | 5 + packages/server/src/ValOps.ts | 179 ++++++++++++- .../server/src/ValOpsFS.jsonValues.test.ts | 116 ++++++++ .../server/src/patch/jsonValuesPatch.test.ts | 62 +++++ packages/server/src/patch/jsonValuesPatch.ts | 51 ++++ .../server/src/validateJsonValues.test.ts | 27 ++ packages/server/src/validateJsonValues.ts | 6 + packages/ui/spa/ValSyncEngine.test.ts | 253 ++++++++++++++++++ packages/ui/spa/ValSyncEngine.ts | 146 +++++++++- .../ui/spa/components/ChangeRecordPopover.tsx | 26 +- .../ui/spa/components/ValFieldProvider.tsx | 38 +++ 12 files changed, 954 insertions(+), 23 deletions(-) diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md index 5bd5172aa..01cdcb689 100644 --- a/docs/plans/jsonValues.md +++ b/docs/plans/jsonValues.md @@ -10,6 +10,14 @@ ## Current state / resume here +> **Studio hardening DONE (2026-07-27):** the Phase 3 defects are fixed. Renaming an entry now +> commits (`move`/`copy` arms in `ValOps.prepare`), a published edit no longer appears to revert +> (json entry cache is invalidated on publish), a failed `/json` load renders an error instead of a +> forever-spinner, and nested `.jsonValues()` is rejected at startup. Also fixed a **pre-existing** +> bug found on the way: `analyzePatches` pushed one `patchesByModule` entry per op, so `prepare` +> re-applied the whole patch once per op (harmless for `replace`, corrupting for `move`). +> Automated tests green; the **manual Studio walkthrough (V1–V9 below) has NOT been run yet.** + > **Commit flow DONE (2026-07-03):** `ValOps.prepare` now routes patch ops for `.jsonValues()` > records. Content edits write only the entry's `*.val.json` (the `.val.ts` is NOT touched); adding > an entry writes a new `*.val.json` + inserts a `c.json(() => import("..."))` thunk into the @@ -119,6 +127,18 @@ entries; runtime/Studio/validation work one entry at a time; zero overhead when strictness. Val object-unions are always discriminated, so distribute+recurse suffices. 5. i18n deferred; design json format locale-agnostic. 6. All-or-nothing: every entry of a `.jsonValues()` record is a `c.json` thunk (no mixing). +7. **Root-only (LOCKED 2026-07-27).** `.jsonValues()` is only supported on a module's ROOT + record/router. A nested one is rejected at startup: `ValOps.initSources` reports a `ModulesError` + per offender (via `findNestedJsonValuesRecords`), so `/sources/~` fails with "Val is not correctly + setup" naming the module; the commit flow rejects nested entry ops as defense in depth. + Rationale: the `/json` endpoint keys entries by a single string, the Studio substitutes content at + the top level of the module source, and `validateJsonValuesEntries` only visits a root record — + nested entries would silently get NO content validation. `classifyJsonValuesOp` still reports + `recordPath` truthfully so the door stays open. +8. **A rename relocates the file (LOCKED 2026-07-27).** A `move` writes the destination via + `getNewJsonEntryPaths` — the generated convention path — and deletes the source file. So renaming + a hand-placed `content/faq.val.json` produces `page/support/faq2.val.json`. One invariant; the + accepted cost is that renaming empties a hand-authored directory. ## Key runtime shapes @@ -210,12 +230,29 @@ JSONValue>>` and `private loadingJsonEntries: Set<"mfp\0key">`. Add `requestJson (AnyField/Field at a path); when the path's parent schema is a `jsonValues` record and the marker isn't loaded, request it and render a loading state until `getSourceSnapshot` returns content. `useShallowSourceAtPath(path, "record")` already returns keys only (list is fine). -- [ ] **Per-entry patches**: editing an entry field produces a normal patch at the entry path; on +- [x] **Per-entry patches**: editing an entry field produces a normal patch at the entry path; on commit the server commit-flow writes the `*.val.json` (Phase 2 commit flow). Add/remove entry = - add/remove the record key (commit flow emits/removes thunk + file). -- [ ] **Verify** (Studio running): list shows keys w/o loading; opening an entry fetches one `/json`; - fields render + edit; commit writes the `*.val.json`; add/remove a route inserts/removes - thunk + file. + add/remove the record key (commit flow emits/removes thunk + file). **Rename** (`move`) and + duplicate (`copy`) now supported too — see Session 4. +- [x] **Cache lifecycle + error state** (Session 4): `jsonEntryErrors` memoizes a failed load (no + refetch-on-remount loop; the field renders an error, not a forever-spinner) and + `staleJsonEntries` invalidates loaded entries on publish / `/sources/~` refresh. The stale flag + is cleared when a request STARTS, so an invalidation that lands mid-flight wins over the + in-flight response. `retryJsonEntry` / `ensureJsonEntry` added. +- [ ] **Verify** (Studio running) — **NOT YET RUN**. Walkthrough: + - **V1** open the router module → both keys listed, **zero** `/api/val/json` requests. + - **V2** open an entry → exactly one `/json`; revisiting it → no new request. + - **V3** edit `title`, publish → only `content/faq.val.json` modified, `page.val.ts` untouched, and + the new title **survives the publish without a reload**. + - **V4** add `/support/new-page` → new `page/support/new-page.val.json` = `{"title":"","body":"","order":0}` + (from `emptyOf`) + a `c.json` thunk in `page.val.ts`. + - **V5** hand-authored `content/` and generated `page/support/` coexist; re-editing an existing + entry still writes its original path. + - **V6** rename → new file with the same content, old file deleted, thunk key + import path swapped. + - **V7** delete → file deleted, thunk gone (empty dirs are left behind; `deleteFile` doesn't prune). + - **V8** corrupt a `*.val.json` → `/json` 500s → ONE request, an error in the field, no refetch on + navigate-away-and-back; restoring the file + a refresh clears the memo. + - **V9** a nested `.jsonValues()` module → Studio refuses to load, naming the module. ## Phase 4 — Runtime APIs (`packages/next`, `packages/react`) @@ -265,6 +302,27 @@ unconditionally (accepted "validation takes more time" tradeoff). ## Changelog +- **Session 4 (2026-07-27)**: Studio hardening — closes the Phase 3 defects. + - **Rename/duplicate now commit.** `ValOps.prepare` classifies `op.from` as well as `op.path` and + gained `move`/`copy` arms: load the source entry's content, remove the old thunk (move only), + insert the new one at `getNewJsonEntryPaths`, and null the old file. Cross-record and + cross-entry `move`/`copy` are rejected instead of silently corrupting (`rebaseContentOp` sliced + `from` by the destination's prefix without checking they matched). + - **Pre-existing bug fixed**: `analyzePatches` pushed one `patchesByModule` entry per non-file op, + so `prepare` re-applied the _whole_ patch once per op. Idempotent for `replace`, but it + duplicated `add`s and broke `move`. Now at most one entry per (module, patch). + - **Nested `.jsonValues()` rejected** — `findNestedJsonValuesRecords` + a `ModulesError` from + `initSources` (locked decision #7). + - **Studio cache lifecycle**: `jsonEntryErrors` (failure memo + field error state), + `staleJsonEntries` (invalidate on publish and on `/sources/~` refresh), `retryJsonEntry`, + `ensureJsonEntry`. Publish needs its own invalidation because a content-only edit leaves the + module source (bare markers) byte-identical, so `sourcesSha` never changes and nothing refetches. + - **Rename guard**: `ChangeRecordPopover` awaits `ensureJsonEntry` before emitting the `move`, so + the patch carries real content instead of an opaque marker (which would 404 on the new key). + - Tests: `ValOpsFS.jsonValues.test.ts` (+7), `jsonValuesPatch.test.ts` (+6), + `validateJsonValues.test.ts` (+1 pinning the root-only contract), and a new `jsonValues` + describe in `ValSyncEngine.test.ts` (+8, was zero coverage) with `/json` + `/save` added to the + mock client. - **Session 3 (2026-07-03)**: `fetchValRoute`/`useValRoute` made jsonValues-aware — they now map route params → the entry key and load ONLY the matched entry (RSC `loadJsonEntryContent`; client reuses the `useValKey` `React.use` cache), instead of eagerly resolving the whole record. Added diff --git a/packages/core/src/schema/record.ts b/packages/core/src/schema/record.ts index 93c83fa99..259b20e66 100644 --- a/packages/core/src/schema/record.ts +++ b/packages/core/src/schema/record.ts @@ -645,6 +645,11 @@ export class RecordSchema< * so a record/router can scale to many thousands of entries. * * Not supported on image/file galleries (`s.images()` / `s.files()`). + * + * Only supported on a module's ROOT record/router — a `.jsonValues()` record + * nested inside an object/array/record is rejected at startup with a module + * error, because the single-entry fetch endpoint, the Studio's content + * substitution and content validation are all root-only. */ jsonValues(): RecordSchema> { if (this.mediaOptions) { diff --git a/packages/server/src/ValOps.ts b/packages/server/src/ValOps.ts index 0a4e8e469..cea27ed8d 100644 --- a/packages/server/src/ValOps.ts +++ b/packages/server/src/ValOps.ts @@ -39,6 +39,7 @@ import { analyzeValModule } from "./patch/ts/valModule"; import { analyzeJsonValuesEntries } from "./patch/ts/jsonValuesModule"; import { classifyJsonValuesOp, + findNestedJsonValuesRecords, getNewJsonEntryPaths, resolveExistingJsonPath, } from "./patch/jsonValuesPatch"; @@ -120,6 +121,40 @@ function rebaseContentOp( } } +/** + * `.jsonValues()` is only supported on a module's ROOT record/router. A nested + * one is broken end to end (the `/json` endpoint keys entries by a single + * string, the Studio substitutes at the top level, and content validation + * silently skips it), so reject it up front as a module error — `/sources/~` + * then fails with "Val is not correctly setup" naming the module. + */ +function findNestedJsonValuesModuleErrors(schemas: Schemas): ModulesError[] { + const errors: ModulesError[] = []; + for (const moduleFilePathS of Object.keys(schemas)) { + const moduleFilePath = moduleFilePathS as ModuleFilePath; + const schema = schemas[moduleFilePath]; + if (!schema) { + continue; + } + let serialized: SerializedSchema; + try { + serialized = schema["executeSerialize"](); + } catch { + // Serialization errors are reported elsewhere (e.g. by extractValModules). + continue; + } + for (const nestedPath of findNestedJsonValuesRecords(serialized)) { + errors.push({ + path: moduleFilePath, + message: `Nested .jsonValues() records are not supported: '${nestedPath.join( + ".", + )}' in ${moduleFilePath}. Use .jsonValues() only on a module's root record/router.`, + }); + } + } + return errors; +} + export type ValOpsOptions = { formatter?: (code: string, filePath: string) => string | Promise; statPollingInterval?: number; @@ -221,13 +256,16 @@ export abstract class ValOps { this.modulesErrors === null ) { const extracted = await extractValModules(this.valModules); + const moduleErrors = extracted.moduleErrors.concat( + findNestedJsonValuesModuleErrors(extracted.schemas), + ); this.sources = extracted.sources; this.schemas = extracted.schemas; this.baseSha = extracted.baseSha as BaseSha; this.schemaSha = extracted.schemaSha as SchemaSha; this.sourcesSha = extracted.sourcesSha as SourcesSha; this.configSha = extracted.configSha as ConfigSha; - this.modulesErrors = extracted.moduleErrors; + this.modulesErrors = moduleErrors; return { baseSha: this.baseSha, schemaSha: this.schemaSha, @@ -235,7 +273,7 @@ export abstract class ValOps { configSha: this.configSha, sources: extracted.sources, schemas: extracted.schemas, - moduleErrors: extracted.moduleErrors, + moduleErrors, }; } return { @@ -313,9 +351,18 @@ export abstract class ValOps { if (!patchesByModule[path]) { patchesByModule[path] = []; } - patchesByModule[path].push({ - patchId: patch.patchId, - }); + // At most ONE entry per (module, patch): consumers treat each entry as + // "apply this whole patch". Pushing per-op made a patch with N non-file + // ops be applied N times — idempotent for `replace`, but it duplicates + // `add`s and corrupts non-idempotent ops like `move`. + if ( + patchesByModule[path][patchesByModule[path].length - 1]?.patchId !== + patch.patchId + ) { + patchesByModule[path].push({ + patchId: patch.patchId, + }); + } } } return { @@ -1012,7 +1059,14 @@ export abstract class ValOps { const cls = serializedSchema ? classifyJsonValuesOp(serializedSchema, op.path) : ({ kind: "normal" } as const); - if (cls.kind === "normal") { + // `move` / `copy` also READ from a path: classify that too, so an op + // that moves a value out of (or into) a jsonValues entry cannot slip + // through as a plain `.val.ts` op. + const fromCls = + serializedSchema && (op.op === "move" || op.op === "copy") + ? classifyJsonValuesOp(serializedSchema, op.from) + : ({ kind: "normal" } as const); + if (cls.kind === "normal" && fromCls.kind === "normal") { const patchRes = applyPatch(tsSourceFile, tsOps, [op]); if (result.isErr(patchRes)) { collectPatchError(patchRes.error, patchId, op); @@ -1023,6 +1077,30 @@ export abstract class ValOps { tsChanged = true; continue; } + if (cls.kind === "normal") { + errors.push({ + message: `Cannot '${op.op}' a value out of a jsonValues entry and into the module source`, + filePath: path, + }); + patchHadError = true; + break; + } + // Nested `.jsonValues()` records are not supported: only the read path + // for a module's ROOT record/router is implemented end to end. This is + // also rejected up front in `initSources`; this is defense in depth. + if ( + cls.recordPath.length > 0 || + (fromCls.kind === "entry" && fromCls.recordPath.length > 0) + ) { + errors.push({ + message: `Nested .jsonValues() records are not supported: '${cls.recordPath.join( + ".", + )}' in ${path}. Use .jsonValues() only on a module's root record/router.`, + filePath: path, + }); + patchHadError = true; + break; + } // The op targets a `.jsonValues()` entry. if (cls.subPath.length === 0) { // Structural / whole-entry op. @@ -1074,9 +1152,83 @@ export abstract class ValOps { break; } jsonEntryContents.set(jsonPathRes.value, op.value); + } else if (op.op === "move" || op.op === "copy") { + // Rename (move) or duplicate (copy) a whole entry. The new entry + // gets its own `*.val.json` written with the source entry's + // content plus a `c.json(...)` thunk; a move additionally drops + // the old thunk and deletes the old file. + if ( + fromCls.kind !== "entry" || + fromCls.subPath.length !== 0 || + fromCls.recordPath.join("\0") !== cls.recordPath.join("\0") + ) { + errors.push({ + message: `Cannot '${ + op.op + }' a jsonValues entry across records or from a non-entry path (from '${op.from.join( + ".", + )}' to '${op.path.join(".")}')`, + filePath: path, + }); + patchHadError = true; + break; + } + const fromKey = fromCls.entryKey; + const fromPathRes = resolveEntryJsonPath(fromKey); + if (result.isErr(fromPathRes)) { + errors.push(fromPathRes.error); + patchHadError = true; + break; + } + // Load BEFORE marking anything deleted: `loadEntryContent` errors + // on a path that has already been nulled in this commit. + const contentRes = await loadEntryContent(fromPathRes.value); + if (result.isErr(contentRes)) { + errors.push(contentRes.error); + patchHadError = true; + break; + } + const content = deepClone(contentRes.value); + if (op.op === "move") { + const remRes = removeValJsonEntry( + tsSourceFile, + cls.recordPath, + fromKey, + ); + if (result.isErr(remRes)) { + collectPatchError(remRes.error, patchId, op); + patchHadError = true; + break; + } + tsSourceFile = remRes.value; + } + // LOCKED convention: the destination always uses the generated + // path, so renaming a hand-placed file relocates it. + const { jsonPath, importPath } = getNewJsonEntryPaths( + path, + cls.entryKey, + ); + const insRes = insertValJsonEntry( + tsSourceFile, + cls.recordPath, + cls.entryKey, + importPath, + ); + if (result.isErr(insRes)) { + collectPatchError(insRes.error, patchId, op); + patchHadError = true; + break; + } + tsSourceFile = insRes.value; + tsChanged = true; + jsonEntryContents.set(jsonPath, content); + entryKeyToJsonPath.set(cls.entryKey, jsonPath); + if (op.op === "move" && fromPathRes.value !== jsonPath) { + jsonEntryContents.set(fromPathRes.value, null); + } } else { errors.push({ - message: `Unsupported op '${op.op}' on jsonValues entry '${cls.entryKey}'`, + message: `Unsupported op '${op.op}' on jsonValues entry '${cls.entryKey}' (supported: add, remove, replace, move, copy)`, filePath: path, }); patchHadError = true; @@ -1084,6 +1236,19 @@ export abstract class ValOps { } } else { // Content sub-op: replay against the entry's `*.val.json`. + // `rebaseContentOp` slices `from` by the same prefix as `path`, so a + // cross-entry move/copy would silently corrupt the target entry. + if ( + (op.op === "move" || op.op === "copy") && + (fromCls.kind !== "entry" || fromCls.entryKey !== cls.entryKey) + ) { + errors.push({ + message: `Cannot '${op.op}' between different jsonValues entries`, + filePath: path, + }); + patchHadError = true; + break; + } const jsonPathRes = resolveEntryJsonPath(cls.entryKey); if (result.isErr(jsonPathRes)) { errors.push(jsonPathRes.error); diff --git a/packages/server/src/ValOpsFS.jsonValues.test.ts b/packages/server/src/ValOpsFS.jsonValues.test.ts index 336f8ffae..7c9e2d1d4 100644 --- a/packages/server/src/ValOpsFS.jsonValues.test.ts +++ b/packages/server/src/ValOpsFS.jsonValues.test.ts @@ -149,4 +149,120 @@ describe("ValOpsFS jsonValues commit flow", () => { expect(ts).not.toContain(`"/blog/world"`); expect(ts).toContain(`"/blog/hello"`); }); + + test("move renames a hand-authored entry: relocates the file and swaps the thunk", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { op: "move", from: ["/blog/hello"], path: ["/blog/renamed"] }, + ]); + expect(pc.hasErrors).toBe(false); + // LOCKED convention: the destination uses the generated path, so a rename + // relocates a hand-placed file out of its original directory. + const newJson = pc.patchedSourceFiles["/test/pages/blog/renamed.val.json"]; + expect(typeof newJson).toBe("string"); + expect(JSON.parse(newJson as string)).toEqual({ + title: "Hello", + order: 1, + }); + expect(pc.patchedSourceFiles["/test/content/hello.val.json"]).toBeNull(); + const ts = pc.patchedSourceFiles[MODULE_PATH]; + expect(typeof ts).toBe("string"); + expect(ts).not.toContain(`"/blog/hello"`); + expect(ts).not.toContain(`import("./content/hello.val.json")`); + expect(ts).toContain(`"/blog/renamed"`); + expect(ts).toContain(`import("./pages/blog/renamed.val.json")`); + // the untouched entry survives + expect(ts).toContain(`"/blog/world"`); + }); + + test("move then a content edit on the new key resolves to the new file", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { op: "move", from: ["/blog/hello"], path: ["/blog/renamed"] }, + { op: "replace", path: ["/blog/renamed", "title"], value: "Renamed!" }, + ]); + expect(pc.hasErrors).toBe(false); + const newJson = pc.patchedSourceFiles["/test/pages/blog/renamed.val.json"]; + expect(JSON.parse(newJson as string)).toEqual({ + title: "Renamed!", + order: 1, + }); + expect(pc.patchedSourceFiles["/test/content/hello.val.json"]).toBeNull(); + }); + + test("a content edit on the OLD key after a move is an error", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { op: "move", from: ["/blog/hello"], path: ["/blog/renamed"] }, + { op: "replace", path: ["/blog/hello", "title"], value: "Nope" }, + ]); + expect(pc.hasErrors).toBe(true); + }); + + test("copy duplicates an entry without deleting the source", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { op: "copy", from: ["/blog/hello"], path: ["/blog/copy"] }, + ]); + expect(pc.hasErrors).toBe(false); + const newJson = pc.patchedSourceFiles["/test/pages/blog/copy.val.json"]; + expect(JSON.parse(newJson as string)).toEqual({ + title: "Hello", + order: 1, + }); + // the source file is NOT deleted + expect( + pc.patchedSourceFiles["/test/content/hello.val.json"], + ).not.toBeNull(); + const ts = pc.patchedSourceFiles[MODULE_PATH]; + expect(ts).toContain(`"/blog/hello"`); + expect(ts).toContain(`"/blog/copy"`); + }); + + test("move from a non-entry path into an entry is an error", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + // `from` targets a field INSIDE an entry, `path` targets a whole entry + { + op: "move", + from: ["/blog/hello", "title"], + path: ["/blog/renamed"], + }, + ]); + expect(pc.hasErrors).toBe(true); + }); + + test("a multi-op patch is applied exactly once (not once per op)", async () => { + // Regression: analyzePatches used to push one entry per non-file op, so + // `prepare` re-applied the whole patch once per op. + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { op: "replace", path: ["/blog/hello", "order"], value: 10 }, + { op: "replace", path: ["/blog/world", "order"], value: 20 }, + ]); + expect(pc.hasErrors).toBe(false); + expect( + JSON.parse( + pc.patchedSourceFiles["/test/content/hello.val.json"] as string, + ), + ).toEqual({ title: "Hello", order: 10 }); + expect( + JSON.parse( + pc.patchedSourceFiles["/test/content/world.val.json"] as string, + ), + ).toEqual({ title: "World", order: 20 }); + expect(pc.appliedPatches[MODULE_PATH]).toHaveLength(1); + }); + + test("move between different entries' content is an error", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { + op: "move", + from: ["/blog/hello", "title"], + path: ["/blog/world", "title"], + }, + ]); + expect(pc.hasErrors).toBe(true); + }); }); diff --git a/packages/server/src/patch/jsonValuesPatch.test.ts b/packages/server/src/patch/jsonValuesPatch.test.ts index 467cf1a6d..6301c7e40 100644 --- a/packages/server/src/patch/jsonValuesPatch.test.ts +++ b/packages/server/src/patch/jsonValuesPatch.test.ts @@ -1,6 +1,7 @@ import { initVal, type SerializedSchema } from "@valbuild/core"; import { classifyJsonValuesOp, + findNestedJsonValuesRecords, getNewJsonEntryPaths, resolveExistingJsonPath, } from "./jsonValuesPatch"; @@ -84,6 +85,67 @@ describe("getNewJsonEntryPaths", () => { }); }); +describe("findNestedJsonValuesRecords", () => { + test("a root jsonValues record is allowed → no offenders", () => { + const root: SerializedSchema = s + .record(s.object({ title: s.string() })) + .jsonValues() + ["executeSerialize"](); + expect(findNestedJsonValuesRecords(root)).toEqual([]); + }); + + test("a plain schema with no jsonValues → no offenders", () => { + const plain: SerializedSchema = s + .object({ pages: s.record(s.object({ title: s.string() })) }) + ["executeSerialize"](); + expect(findNestedJsonValuesRecords(plain)).toEqual([]); + }); + + test("jsonValues nested under an object is reported", () => { + const nested: SerializedSchema = s + .object({ + title: s.string(), + pages: s.record(s.object({ title: s.string() })).jsonValues(), + }) + ["executeSerialize"](); + expect(findNestedJsonValuesRecords(nested)).toEqual([["pages"]]); + }); + + test("jsonValues nested under an array is reported", () => { + const nested: SerializedSchema = s + .array( + s.object({ + pages: s.record(s.object({ title: s.string() })).jsonValues(), + }), + ) + ["executeSerialize"](); + expect(findNestedJsonValuesRecords(nested)).toEqual([["*", "pages"]]); + }); + + test("jsonValues nested under another record is reported", () => { + const nested: SerializedSchema = s + .record( + s.object({ + pages: s.record(s.object({ title: s.string() })).jsonValues(), + }), + ) + ["executeSerialize"](); + expect(findNestedJsonValuesRecords(nested)).toEqual([["*", "pages"]]); + }); + + test("multiple offenders are all reported", () => { + const nested: SerializedSchema = s + .object({ + a: s.record(s.object({ title: s.string() })).jsonValues(), + b: s.object({ + c: s.record(s.object({ title: s.string() })).jsonValues(), + }), + }) + ["executeSerialize"](); + expect(findNestedJsonValuesRecords(nested)).toEqual([["a"], ["b", "c"]]); + }); +}); + describe("resolveExistingJsonPath", () => { test("resolves a hand-placed import path relative to the module dir", () => { expect( diff --git a/packages/server/src/patch/jsonValuesPatch.ts b/packages/server/src/patch/jsonValuesPatch.ts index b902c3461..6002d9f15 100644 --- a/packages/server/src/patch/jsonValuesPatch.ts +++ b/packages/server/src/patch/jsonValuesPatch.ts @@ -73,6 +73,57 @@ function descend( } } +/** + * Finds every `.jsonValues()` record in a module's schema that is NOT the + * module's root, returning the path to each within the module source. + * + * `.jsonValues()` is only supported on a module's ROOT record/router: the + * `/json` endpoint keys entries by a single string, the Studio substitutes + * loaded content at the top level of the module source, and + * `validateJsonValuesEntries` only visits a root record. A nested one would + * silently skip content validation and hang the Studio on a 404, so we reject + * it up front instead (see {@link ValOps.initSources}). + */ +export function findNestedJsonValuesRecords( + schema: SerializedSchema, + path: string[] = [], +): string[][] { + const found: string[][] = []; + const rec = (current: SerializedSchema, currentPath: string[]) => { + if ( + current.type === "record" && + current.jsonValues && + currentPath.length > 0 + ) { + found.push(currentPath); + // Do not descend: everything below lives in the entry's `*.val.json`. + return; + } + switch (current.type) { + case "object": + for (const key of Object.keys(current.items)) { + rec(current.items[key], currentPath.concat(key)); + } + return; + case "record": + rec(current.item, currentPath.concat("*")); + return; + case "array": + rec(current.item, currentPath.concat("*")); + return; + case "union": + for (let i = 0; i < current.items.length; i++) { + rec(current.items[i], currentPath.concat(`union[${i}]`)); + } + return; + default: + return; + } + }; + rec(schema, path); + return found; +} + /** * The `.val.ts` suffix a module file path ends with. Stripping it yields the * folder that a new entry's `*.val.json` files are nested under. diff --git a/packages/server/src/validateJsonValues.test.ts b/packages/server/src/validateJsonValues.test.ts index 8010f0f39..33fa75118 100644 --- a/packages/server/src/validateJsonValues.test.ts +++ b/packages/server/src/validateJsonValues.test.ts @@ -57,4 +57,31 @@ describe("validateJsonValuesEntries", () => { expect(errors).toEqual({}); expect(loaded).toBe(false); }); + + test("ROOT-ONLY contract: a nested jsonValues record is not visited", async () => { + // Pins the documented limitation. Nested `.jsonValues()` records are + // rejected up front as module errors (findNestedJsonValuesRecords), so they + // never reach here — but if that guard is relaxed without making this a + // recursive visitor, nested entries silently get NO content validation. + const nestedSchema = s.object({ + pages: s.record(s.object({ title: s.string() })).jsonValues(), + }); + let loaded = false; + const source = { + pages: { + // invalid content: would be an error if it were visited + "/a": c.json(() => { + loaded = true; + return Promise.resolve({ default: { title: 123 } }); + }), + }, + }; + const errors = await validateJsonValuesEntries( + nestedSchema, + source, + modulePath, + ); + expect(errors).toEqual({}); + expect(loaded).toBe(false); + }); }); diff --git a/packages/server/src/validateJsonValues.ts b/packages/server/src/validateJsonValues.ts index 4cd3c2fec..c38dbf770 100644 --- a/packages/server/src/validateJsonValues.ts +++ b/packages/server/src/validateJsonValues.ts @@ -22,6 +22,12 @@ import { ValidationError } from "@valbuild/core"; * Entries without a runtime import thunk (transport markers / draft entries * whose content lives in a patch) are skipped here — their content is validated * where it is loaded (the single-entry fetch path). + * + * ROOT-ONLY BY CONTRACT: only a module whose root schema is a `.jsonValues()` + * record is visited. Nested `.jsonValues()` records are rejected up front as + * module errors (see `findNestedJsonValuesRecords`), so they can never reach + * here — if that guard is ever relaxed, this must become a recursive visitor or + * nested entries silently get no content validation at all. */ export async function validateJsonValuesEntries( schema: Schema, diff --git a/packages/ui/spa/ValSyncEngine.test.ts b/packages/ui/spa/ValSyncEngine.test.ts index 3c0e775d8..61e2a7e95 100644 --- a/packages/ui/spa/ValSyncEngine.test.ts +++ b/packages/ui/spa/ValSyncEngine.test.ts @@ -449,6 +449,215 @@ describe("ValSyncEngine", () => { "module-schema-not-found", ); }); + + describe("jsonValues", () => { + const PAGES = "/pages.val.ts" as const; + + /** + * A `.jsonValues()` record module: the source only carries lazy markers, so + * the engine must fetch each entry's content via GET /json. + */ + function setupJsonValues() { + const { s, c, config } = initVal(); + const schema = s + .record(s.object({ title: s.string(), order: s.number() })) + .jsonValues(); + const valModule = c.define(PAGES, schema, { + "/a": c.json(() => + Promise.resolve({ default: { title: "A", order: 1 } }), + ), + "/b": c.json(() => + Promise.resolve({ default: { title: "B", order: 2 } }), + ), + }); + const tester = new SyncEngineTester("fs", [valModule as any], config); + tester.fakeJsonEntries[PAGES] = { + "/a": { title: "A", order: 1 }, + "/b": { title: "B", order: 2 }, + }; + return { tester, config }; + } + + /** Lets the `/json` promise chain settle. */ + const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + + test("requestJsonEntry substitutes the loaded content into the source", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + // Before loading, the entry is an opaque marker. + const before = engine.getSourceSnapshot(toModuleFilePath(PAGES)) + .data as any; + expect(Internal.isJson(before["/a"])).toBe(true); + + engine.requestJsonEntry(toModuleFilePath(PAGES), "/a"); + await flush(); + + const after = engine.getSourceSnapshot(toModuleFilePath(PAGES)) + .data as any; + expect(after["/a"]).toEqual({ title: "A", order: 1 }); + // the un-requested entry is untouched + expect(Internal.isJson(after["/b"])).toBe(true); + expect(tester.jsonRequestCounts[`${PAGES}\0/a`]).toBe(1); + }); + + test("a loaded entry is not refetched", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + engine.requestJsonEntry(toModuleFilePath(PAGES), "/a"); + await flush(); + engine.requestJsonEntry(toModuleFilePath(PAGES), "/a"); + engine.requestJsonEntry(toModuleFilePath(PAGES), "/a"); + await flush(); + expect(tester.jsonRequestCounts[`${PAGES}\0/a`]).toBe(1); + }); + + test("a failed load is memoized: no refetch loop, and an error is exposed", async () => { + // Regression: a failing entry used to be refetched on every remount and + // rendered a spinner forever, because nothing was cached on failure. + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + engine.requestJsonEntry(toModuleFilePath(PAGES), "/missing"); + await flush(); + + expect( + engine.getJsonEntryError(toModuleFilePath(PAGES), "/missing"), + ).toContain("Entry not found"); + expect(tester.jsonRequestCounts[`${PAGES}\0/missing`]).toBe(1); + + // Subsequent requests must NOT hit the endpoint again. + engine.requestJsonEntry(toModuleFilePath(PAGES), "/missing"); + engine.requestJsonEntry(toModuleFilePath(PAGES), "/missing"); + await flush(); + expect(tester.jsonRequestCounts[`${PAGES}\0/missing`]).toBe(1); + }); + + test("retryJsonEntry clears the memo and refetches", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + engine.requestJsonEntry(toModuleFilePath(PAGES), "/late"); + await flush(); + expect( + engine.getJsonEntryError(toModuleFilePath(PAGES), "/late"), + ).not.toBe(null); + + // The entry shows up server-side, then the user retries. + tester.fakeJsonEntries[PAGES]["/late"] = { title: "Late", order: 3 }; + engine.retryJsonEntry(toModuleFilePath(PAGES), "/late"); + await flush(); + + expect(engine.getJsonEntryError(toModuleFilePath(PAGES), "/late")).toBe( + null, + ); + expect(tester.jsonRequestCounts[`${PAGES}\0/late`]).toBe(2); + }); + + test("publish refetches loaded entries (a published edit must not revert)", async () => { + // Regression: jsonEntryContents was only cleared on init/reset. A + // content-only edit does not change the module source (bare markers), so + // sourcesSha is unchanged and nothing refetched — after publish the + // pre-edit content was re-substituted and the edit looked reverted. + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/a"); + expect( + (engine.getSourceSnapshot(toModuleFilePath(PAGES)).data as any)["/a"], + ).toEqual({ title: "A", order: 1 }); + + // Edit the entry, then publish. The fake server commits it: the entry's + // *.val.json now holds the new content, the module source is unchanged. + const res = engine.addPatch( + toSourcePath(PAGES), + "record", + [{ op: "replace", path: ["/a", "title"], value: "A edited" }], + tester.getNextNow(), + ); + expect(res.status).toBe("patch-added"); + tester.simulatePassingOfSeconds(5); + await engine.sync(tester.getNextNow()); + await flush(); + const patchIds = tester.fakePatches.map((p) => p.patchId); + tester.fakeJsonEntries[PAGES]["/a"] = { title: "A edited", order: 1 }; + const requestsBeforePublish = + tester.jsonRequestCounts[`${PAGES}\0/a`] ?? 0; + + expect( + await engine.publish(patchIds, undefined, tester.getNextNow()), + ).toMatchObject({ status: "done" }); + await flush(); + + expect(tester.jsonRequestCounts[`${PAGES}\0/a`]).toBe( + requestsBeforePublish + 1, + ); + expect( + (engine.getSourceSnapshot(toModuleFilePath(PAGES)).data as any)["/a"], + ).toEqual({ title: "A edited", order: 1 }); + }); + + test("an entry invalidated mid-flight is refetched (a stale response must not win)", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/a"); + engine.addPatch( + toSourcePath(PAGES), + "record", + [{ op: "replace", path: ["/a", "title"], value: "A edited" }], + tester.getNextNow(), + ); + tester.simulatePassingOfSeconds(5); + await engine.sync(tester.getNextNow()); + const patchIds = tester.fakePatches.map((p) => p.patchId); + + // Publish WITHOUT letting the sync-triggered refetch settle first, so the + // entry is invalidated while a request is still in flight. That in-flight + // response predates the publish and must not be the final value. + tester.fakeJsonEntries[PAGES]["/a"] = { title: "A edited", order: 1 }; + await engine.publish(patchIds, undefined, tester.getNextNow()); + await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/a"); + await flush(); + + expect( + (engine.getSourceSnapshot(toModuleFilePath(PAGES)).data as any)["/a"], + ).toEqual({ title: "A edited", order: 1 }); + }); + + test("ensureJsonEntry resolves once content is loaded", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/b"); + expect( + (engine.getSourceSnapshot(toModuleFilePath(PAGES)).data as any)["/b"], + ).toEqual({ title: "B", order: 2 }); + + // Already loaded: resolves immediately, no second request. + await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/b"); + expect(tester.jsonRequestCounts[`${PAGES}\0/b`]).toBe(1); + }); + + test("a move patch on a loaded entry carries real content to the new key", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/a"); + engine.addPatch( + toSourcePath(PAGES), + "record", + [{ op: "move", from: ["/a"], path: ["/renamed"] }], + tester.getNextNow(), + ); + + const source = engine.getSourceSnapshot(toModuleFilePath(PAGES)) + .data as any; + expect(source["/a"]).toBeUndefined(); + // real content, NOT an opaque marker + expect(source["/renamed"]).toEqual({ title: "A", order: 1 }); + }); + }); }); function toModuleFilePath(moduleFilePath: `/${string}.val.ts`): ModuleFilePath { @@ -497,6 +706,12 @@ type FakeApi = { "/schema": { GET: z.infer | ClientFetchErrors | null; }; + "/json": { + GET: z.infer | ClientFetchErrors | null; + }; + "/save": { + POST: z.infer | ClientFetchErrors | null; + }; }; class SyncEngineTester { @@ -514,6 +729,10 @@ class SyncEngineTester { fakeSources: Record; now: number; fakeResponses: Partial; + /** Committed `.jsonValues()` entry content, by module then entry key. */ + fakeJsonEntries: Record> = {}; + /** How many times GET /json was called, keyed `${path}\0${key}`. */ + jsonRequestCounts: Record = {}; constructor( private mode: "fs" | "http", @@ -597,6 +816,30 @@ class SyncEngineTester { }; } + getJson( + req: InferReq, + ): z.infer { + const path = req.query.path; + const key = req.query.key; + this.jsonRequestCounts[`${path}\0${key}`] = + (this.jsonRequestCounts[`${path}\0${key}`] ?? 0) + 1; + const entries = this.fakeJsonEntries[path]; + if (entries === undefined || !(key in entries)) { + return { + status: 404, + json: { message: `Entry not found: ${key} in ${path}` }, + }; + } + return { + status: 200, + json: { + path: path as ModuleFilePath, + key, + content: entries[key], + }, + }; + } + putPatches( req: InferReq, ): z.infer { @@ -856,6 +1099,16 @@ class SyncEngineTester { if (route === "/schema" && method === "GET") { return this.getSchema(req); } + if (route === "/json" && method === "GET") { + // NOTE: must be a Promise — the sync engine calls `.then()` on it. + return Promise.resolve(this.getJson(req)); + } + if (route === "/save" && method === "POST") { + // Publishing commits the pending patches; the fake server just drops + // them (fs mode behaviour). + this.fakePatches = []; + return { status: 200, json: {} }; + } return { status: 404, json: { diff --git a/packages/ui/spa/ValSyncEngine.ts b/packages/ui/spa/ValSyncEngine.ts index bbb5e803c..21868a6c6 100644 --- a/packages/ui/spa/ValSyncEngine.ts +++ b/packages/ui/spa/ValSyncEngine.ts @@ -145,7 +145,23 @@ export class ValSyncEngine { private jsonEntryContents: Record> = {}; /** In-flight json entry loads, keyed `${moduleFilePath}\0${key}`. */ - private loadingJsonEntries: Set = new Set(); + private loadingJsonEntries: Map> = new Map(); + /** + * Entries whose last load failed, keyed by module then entry key. Memoizing + * the failure is what stops a permanently-failing entry from being refetched + * on every field remount (and from rendering a spinner forever): the field + * hooks surface this as an error state instead. Cleared by `retryJsonEntry` + * and whenever the module's server source is replaced. + */ + private jsonEntryErrors: Record> = {}; + /** + * Loaded entries whose committed content may now be out of date (the module's + * server source was replaced, e.g. after publish), keyed + * `${moduleFilePath}\0${key}`. They keep rendering their old content until the + * refetch lands, so there is no loading flash — but they MUST be refetched, or + * a published edit appears to revert to the pre-edit content. + */ + private staleJsonEntries: Set = new Set(); private renders: Record | null; private schemas: Record | null; private serverSideSchemaSha: string | null; @@ -398,7 +414,9 @@ export class ValSyncEngine { this.serverSources = null; this.patchedSourcesCache = null; this.jsonEntryContents = {}; - this.loadingJsonEntries = new Set(); + this.loadingJsonEntries = new Map(); + this.jsonEntryErrors = {}; + this.staleJsonEntries = new Set(); this.renders = null; this.globalServerSidePatchIds = []; this.syncedServerSidePatchIds = []; @@ -836,15 +854,53 @@ export class ValSyncEngine { * Studio opens a path that descends into an unloaded json marker. */ requestJsonEntry(moduleFilePath: ModuleFilePath, key: string): void { - if (this.jsonEntryContents[moduleFilePath]?.[key] !== undefined) { - return; - } + void this.ensureJsonEntry(moduleFilePath, key); + } + + /** + * Resolves once a `.jsonValues()` entry's content is loaded — or immediately + * if it already is, or if its load previously failed. Awaited before emitting + * a patch that moves a whole entry, so the patch carries real content rather + * than an opaque marker. {@link requestJsonEntry} is the fire-and-forget + * variant used by the field hooks. + */ + ensureJsonEntry(moduleFilePath: ModuleFilePath, key: string): Promise { const loadingKey = `${moduleFilePath}\0${key}`; - if (this.loadingJsonEntries.has(loadingKey)) { - return; + const inFlight = this.loadingJsonEntries.get(loadingKey); + if (inFlight !== undefined) { + // The stale flag is cleared when a request STARTS, so if it is set again + // now, this in-flight response predates whatever invalidated the entry — + // wait for it, then load again. + return inFlight.then(() => + this.staleJsonEntries.has(loadingKey) + ? this.ensureJsonEntry(moduleFilePath, key) + : undefined, + ); + } + const isStale = this.staleJsonEntries.has(loadingKey); + if ( + !isStale && + this.jsonEntryContents[moduleFilePath]?.[key] !== undefined + ) { + return Promise.resolve(); + } + // A memoized failure stops the refetch-on-every-remount loop. Cleared by + // `retryJsonEntry` or by the module's server source being replaced. + if (this.jsonEntryErrors[moduleFilePath]?.[key] !== undefined) { + return Promise.resolve(); } - this.loadingJsonEntries.add(loadingKey); - this.client("/json", "GET", { + const setJsonEntryError = (message: string) => { + if (this.jsonEntryErrors[moduleFilePath] === undefined) { + this.jsonEntryErrors[moduleFilePath] = {}; + } + this.jsonEntryErrors[moduleFilePath][key] = message; + this.invalidateSource(moduleFilePath); + }; + // Cleared at request START, not on success: anything that marks the entry + // stale while this request is in flight must win, since the response we are + // about to get was produced before that invalidation. + this.staleJsonEntries.delete(loadingKey); + const promise = this.client("/json", "GET", { query: { path: moduleFilePath, key }, }) .then((res) => { @@ -854,6 +910,9 @@ export class ValSyncEngine { } this.jsonEntryContents[moduleFilePath][key] = res.json.content ?? null; + if (this.jsonEntryErrors[moduleFilePath] !== undefined) { + delete this.jsonEntryErrors[moduleFilePath][key]; + } this.invalidateSource(moduleFilePath); } else { console.error("Val: SyncEngine: failed to load json entry", { @@ -861,6 +920,11 @@ export class ValSyncEngine { key, res, }); + setJsonEntryError( + "message" in res.json + ? res.json.message + : `Request failed with status ${res.status}`, + ); } }) .catch((err) => { @@ -869,10 +933,66 @@ export class ValSyncEngine { key, err, }); + setJsonEntryError(err instanceof Error ? err.message : String(err)); }) .finally(() => { this.loadingJsonEntries.delete(loadingKey); }); + this.loadingJsonEntries.set(loadingKey, promise); + return promise; + } + + /** + * The error message of the last failed load of a `.jsonValues()` entry, or + * `null`. Returns a primitive so it is safe to read from a + * `useSyncExternalStore` snapshot getter. + */ + getJsonEntryError( + moduleFilePath: ModuleFilePath, + key: string, + ): string | null { + return this.jsonEntryErrors[moduleFilePath]?.[key] ?? null; + } + + /** Clears a memoized json entry failure and loads it again. */ + retryJsonEntry(moduleFilePath: ModuleFilePath, key: string): void { + if (this.jsonEntryErrors[moduleFilePath] !== undefined) { + delete this.jsonEntryErrors[moduleFilePath][key]; + } + this.requestJsonEntry(moduleFilePath, key); + } + + /** + * Marks every loaded entry of a module stale, so the next request refetches + * its committed content. Called when the module's server source is replaced + * (e.g. after publish): without this the pre-edit content is re-substituted + * and a published edit looks like it reverted. + */ + private markJsonEntriesStale(moduleFilePath: ModuleFilePath): void { + const contents = this.jsonEntryContents[moduleFilePath]; + delete this.jsonEntryErrors[moduleFilePath]; + if (contents === undefined) { + return; + } + for (const key of Object.keys(contents)) { + this.staleJsonEntries.add(`${moduleFilePath}\0${key}`); + this.requestJsonEntry(moduleFilePath, key); + } + } + + /** + * Marks every loaded `.jsonValues()` entry of every module stale. + * + * Needed on publish: a content-only edit rewrites the entry's `*.val.json` + * but NOT the `.val.ts`, so the module source (bare `{_type:"json"}` markers) + * is byte-identical and `sourcesSha` does not change — no `/sources/~` + * refresh is triggered. Without this the just-published content is served + * from the pre-edit cache and the edit looks like it reverted. + */ + private markAllJsonEntriesStale(): void { + for (const moduleFilePathS of Object.keys(this.jsonEntryContents)) { + this.markJsonEntriesStale(moduleFilePathS as ModuleFilePath); + } } /** @@ -3017,6 +3137,11 @@ export class ValSyncEngine { // the un-patched source. The patched view is computed by // getPatchedSource folding the known patch chain on top. this.serverSources[moduleFilePath] = valModule.source; + // The committed content of any loaded `.jsonValues()` entry may + // have changed with it (e.g. after publish) — refetch, or the + // stale pre-edit content is re-substituted and the edit looks + // like it reverted. + this.markJsonEntriesStale(moduleFilePath); // render is always null in the new mode; keep the renders map // up-to-date for any downstream code that still subscribes. if (this.renders === null) { @@ -3211,6 +3336,9 @@ export class ValSyncEngine { this.patchIdsByModuleFilePath = new Map(); this.patchDataByPatchId = {}; this.patchSets = new PatchSets(); + // The published content is now the committed content: any loaded + // `.jsonValues()` entry must be refetched (see markAllJsonEntriesStale). + this.markAllJsonEntriesStale(); const fullReset = true; await this.syncPatches(fullReset, now); this.invalidatePatchSets(); diff --git a/packages/ui/spa/components/ChangeRecordPopover.tsx b/packages/ui/spa/components/ChangeRecordPopover.tsx index 83975d791..ae2e879e8 100644 --- a/packages/ui/spa/components/ChangeRecordPopover.tsx +++ b/packages/ui/spa/components/ChangeRecordPopover.tsx @@ -1,7 +1,12 @@ import { Internal, ModuleFilePath, SourcePath } from "@valbuild/core"; import { useState, useEffect, useCallback, useMemo } from "react"; import { Button } from "./designSystem/button"; -import { useAddPatch, useShallowSourceAtPath } from "./ValFieldProvider"; +import { + useAddPatch, + useSchemaAtPath, + useShallowSourceAtPath, + useSyncEngine, +} from "./ValFieldProvider"; import { useValPortal } from "./ValPortalProvider"; import { useNavigation } from "./ValRouter"; import { @@ -70,8 +75,21 @@ export function ChangeRecordPopover({ } return []; }, [parentSource]); + const syncEngine = useSyncEngine(); + const parentSchema = useSchemaAtPath(parentPath); + // A `.jsonValues()` entry's content is lazily loaded. If we move an entry that + // is still an opaque marker, the marker (not the content) lands on the new key + // and opening it would fetch `/json?key=` — which 404s, since the base + // source still only has the old key. Load it first. + const isJsonValuesRecord = + parentSchema.status === "success" && + parentSchema.data.type === "record" && + parentSchema.data.jsonValues === true; const onSubmit = useCallback( - (key: string) => { + async (key: string) => { + if (isJsonValuesRecord) { + await syncEngine.ensureJsonEntry(moduleFilePath, defaultValue); + } const patchOps: Patch = [ { op: "move", @@ -116,6 +134,10 @@ export function ChangeRecordPopover({ parentPatchPath, navigate, onComplete, + syncEngine, + isJsonValuesRecord, + defaultValue, + existingKeys, ], ); diff --git a/packages/ui/spa/components/ValFieldProvider.tsx b/packages/ui/spa/components/ValFieldProvider.tsx index 22c41db54..6939eabbb 100644 --- a/packages/ui/spa/components/ValFieldProvider.tsx +++ b/packages/ui/spa/components/ValFieldProvider.tsx @@ -553,11 +553,29 @@ function useSchemaAtPathInternal( syncEngine.requestJsonEntry(moduleFilePath, unloadedJsonKey); } }, [syncEngine, moduleFilePath, unloadedJsonKey]); + const jsonEntryError = useSyncExternalStore( + syncEngine.subscribe("source", moduleFilePath), + () => + unloadedJsonKey === null + ? null + : syncEngine.getJsonEntryError(moduleFilePath, unloadedJsonKey), + () => + unloadedJsonKey === null + ? null + : syncEngine.getJsonEntryError(moduleFilePath, unloadedJsonKey), + ); const resolvedSchemaAtPathRes = useMemo(() => { if (schemaRes.status !== "success") { return schemaRes; } if (unloadedJsonKey !== null) { + // A failed load must not render as a perpetual spinner. + if (jsonEntryError !== null) { + return { + status: "error" as const, + error: `Could not load entry '${unloadedJsonKey}': ${jsonEntryError}`, + }; + } return { status: "loading" as const }; } if (sourceData === undefined) { @@ -625,6 +643,7 @@ function useSchemaAtPathInternal( modulePath, sourceData, unloadedJsonKey, + jsonEntryError, ]); const initializedAt = useSyncEngineInitializedAt(syncEngine); if (initializedAt === null) { @@ -1233,6 +1252,17 @@ export function useSourceAtPath( syncEngine.requestJsonEntry(moduleFilePath, unloadedJsonKey); } }, [syncEngine, moduleFilePath, unloadedJsonKey]); + const jsonEntryError = useSyncExternalStore( + syncEngine ? syncEngine.subscribe("source", moduleFilePath) : noopSubscribe, + () => + syncEngine && unloadedJsonKey !== null + ? syncEngine.getJsonEntryError(moduleFilePath, unloadedJsonKey) + : null, + () => + syncEngine && unloadedJsonKey !== null + ? syncEngine.getJsonEntryError(moduleFilePath, unloadedJsonKey) + : null, + ); return useMemo(() => { if (!syncEngine) { return NOT_FOUND; @@ -1244,6 +1274,13 @@ export function useSourceAtPath( return walkSourcePath(modulePath, sourceOverride.moduleSource); } if (unloadedJsonKey !== null) { + // A failed load must not render as a perpetual spinner. + if (jsonEntryError !== null) { + return { + status: "error", + error: `Could not load entry '${unloadedJsonKey}': ${jsonEntryError}`, + }; + } return { status: "loading" }; } if (sourceSnapshot && sourceSnapshot.status === "success") { @@ -1261,6 +1298,7 @@ export function useSourceAtPath( moduleFilePath, sourceOverride, unloadedJsonKey, + jsonEntryError, ]); } From 4124b86997fe18bc68b90e065a2f1eb18ae9db6e Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Mon, 27 Jul 2026 10:17:01 +0200 Subject: [PATCH 25/35] Revert maxLength on blog titles (used for testing validation errors) --- examples/next/app/blogs/[blog]/page.val.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/next/app/blogs/[blog]/page.val.ts b/examples/next/app/blogs/[blog]/page.val.ts index f9544a619..23a2ee8a4 100644 --- a/examples/next/app/blogs/[blog]/page.val.ts +++ b/examples/next/app/blogs/[blog]/page.val.ts @@ -3,7 +3,7 @@ import authorsVal from "../../../content/authors.val"; import { linkSchema } from "../../../components/link.val"; const blogSchema = s.object({ - title: s.string().maxLength(3), + title: s.string(), content: s.richtext({ inline: { a: s.route(), From 7b8b25a219a9f7c6082a58feeb38f29c4c6cab06 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Mon, 27 Jul 2026 10:24:03 +0200 Subject: [PATCH 26/35] jsonValues: load a pending-renamed entry under its base key Renaming an entry and then reloading the Studio left it unopenable. The pending `move` relocates the bare `{_type:"json"}` marker to the new key, so the field requests GET /json for that key -- but /json only resolves keys in the committed source, which still has the old one. It 404s, and (since the previous commit) that renders as a hard error instead of a silent spinner. `resolveBaseJsonEntryKey` walks the pending ops newest-first and undoes whole-entry renames until it lands on a key the base source actually has. Fetching under the base key is also what makes the entry render: `applyJsonEntryContents` substitutes into the base source and the `move` patch then relocates the content to the new key on its own. This is the reload half of the rename fix. The `ensureJsonEntry` guard in ChangeRecordPopover only covers the case where the content happened to be loaded at the moment of the rename; it does not survive a page load. Co-Authored-By: Claude Opus 5 --- packages/ui/spa/ValSyncEngine.test.ts | 38 ++++++++++++++++ packages/ui/spa/ValSyncEngine.ts | 65 +++++++++++++++++++++++++-- 2 files changed, 100 insertions(+), 3 deletions(-) diff --git a/packages/ui/spa/ValSyncEngine.test.ts b/packages/ui/spa/ValSyncEngine.test.ts index 61e2a7e95..e8152194d 100644 --- a/packages/ui/spa/ValSyncEngine.test.ts +++ b/packages/ui/spa/ValSyncEngine.test.ts @@ -639,6 +639,44 @@ describe("ValSyncEngine", () => { expect(tester.jsonRequestCounts[`${PAGES}\0/b`]).toBe(1); }); + test("an UNLOADED entry renamed by a pending patch still loads (rename then reload)", async () => { + // Regression: after a reload, a pending rename leaves the *marker* under + // the new key. Requesting that key from /json 404s, because the committed + // source only knows the old key. The engine must map back to the base key + // — loading it there also lets the move patch relocate the content. + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + // Rename WITHOUT ever loading the entry (as after a fresh page load). + engine.addPatch( + toSourcePath(PAGES), + "record", + [{ op: "move", from: ["/a"], path: ["/renamed"] }], + tester.getNextNow(), + ); + expect( + Internal.isJson( + (engine.getSourceSnapshot(toModuleFilePath(PAGES)).data as any)[ + "/renamed" + ], + ), + ).toBe(true); + + await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/renamed"); + await flush(); + + // Fetched under the BASE key, not the renamed one. + expect(tester.jsonRequestCounts[`${PAGES}\0/a`]).toBe(1); + expect(tester.jsonRequestCounts[`${PAGES}\0/renamed`]).toBeUndefined(); + expect( + engine.getJsonEntryError(toModuleFilePath(PAGES), "/renamed"), + ).toBe(null); + const source = engine.getSourceSnapshot(toModuleFilePath(PAGES)) + .data as any; + expect(source["/a"]).toBeUndefined(); + expect(source["/renamed"]).toEqual({ title: "A", order: 1 }); + }); + test("a move patch on a loaded entry carries real content to the new key", async () => { const { tester } = setupJsonValues(); const engine = await tester.createInitializedSyncEngine(); diff --git a/packages/ui/spa/ValSyncEngine.ts b/packages/ui/spa/ValSyncEngine.ts index 21868a6c6..4d3adc076 100644 --- a/packages/ui/spa/ValSyncEngine.ts +++ b/packages/ui/spa/ValSyncEngine.ts @@ -857,6 +857,59 @@ export class ValSyncEngine { void this.ensureJsonEntry(moduleFilePath, key); } + /** + * Maps an entry key as it appears in the PATCHED source back to the key it + * has in the committed base source, by undoing pending whole-entry renames. + * + * `/json` can only resolve keys that exist in the committed source, so a + * pending rename would 404 on the new key. Loading the content under the + * BASE key instead is also what makes it render: `applyJsonEntryContents` + * substitutes into the base source, and the `move` patch then relocates the + * content to the new key on its own. + */ + private resolveBaseJsonEntryKey( + moduleFilePath: ModuleFilePath, + key: string, + ): string { + const baseSource = this.serverSources?.[moduleFilePath]; + if ( + baseSource === undefined || + baseSource === null || + typeof baseSource !== "object" || + Array.isArray(baseSource) || + key in baseSource + ) { + return key; + } + // Walk the pending ops newest-first, undoing renames until we land on a key + // the base source actually has. + const patchIds = this.orderedPatchIdsForModule(moduleFilePath); + let current = key; + for (let i = patchIds.length - 1; i >= 0; i--) { + const patchData = this.patchDataByPatchId[patchIds[i]]; + if (!patchData) { + continue; + } + const ops = patchData.patch; + for (let j = ops.length - 1; j >= 0; j--) { + const op = ops[j]; + if ( + (op.op === "move" || op.op === "copy") && + op.path.length === 1 && + op.path[0] === current && + op.from.length === 1 + ) { + current = op.from[0]; + if (current in baseSource) { + return current; + } + break; + } + } + } + return current; + } + /** * Resolves once a `.jsonValues()` entry's content is loaded — or immediately * if it already is, or if its load previously failed. Awaited before emitting @@ -864,7 +917,11 @@ export class ValSyncEngine { * than an opaque marker. {@link requestJsonEntry} is the fire-and-forget * variant used by the field hooks. */ - ensureJsonEntry(moduleFilePath: ModuleFilePath, key: string): Promise { + ensureJsonEntry( + moduleFilePath: ModuleFilePath, + requestedKey: string, + ): Promise { + const key = this.resolveBaseJsonEntryKey(moduleFilePath, requestedKey); const loadingKey = `${moduleFilePath}\0${key}`; const inFlight = this.loadingJsonEntries.get(loadingKey); if (inFlight !== undefined) { @@ -949,13 +1006,15 @@ export class ValSyncEngine { */ getJsonEntryError( moduleFilePath: ModuleFilePath, - key: string, + requestedKey: string, ): string | null { + const key = this.resolveBaseJsonEntryKey(moduleFilePath, requestedKey); return this.jsonEntryErrors[moduleFilePath]?.[key] ?? null; } /** Clears a memoized json entry failure and loads it again. */ - retryJsonEntry(moduleFilePath: ModuleFilePath, key: string): void { + retryJsonEntry(moduleFilePath: ModuleFilePath, requestedKey: string): void { + const key = this.resolveBaseJsonEntryKey(moduleFilePath, requestedKey); if (this.jsonEntryErrors[moduleFilePath] !== undefined) { delete this.jsonEntryErrors[moduleFilePath][key]; } From 332efc0d57d9517d3f53bc8f216d359b94eb5c6a Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Tue, 28 Jul 2026 09:17:57 +0200 Subject: [PATCH 27/35] jsonValues: stop entry-content patches poisoning getSources `getSources` applied every non-file op with jsonOps directly against the module source, where a `.jsonValues()` entry is an opaque `{_type:"json"}` marker. A draft content edit at ["","title"] therefore failed with "Cannot replace object element which does not exist", and because the module is then marked poisoned, EVERY later patch for it was skipped with "previous errors exists". Observed live on the example's support router. The Studio dodges this by always passing apply_patches=false, but the eager RSC `fetchVal` sends undefined (which defaults to true) and hits it. Ops are now routed by `classifyJsonValuesOp`: - content sub-ops are skipped -- the module source is genuinely unaffected, since the content lives in the entry's `*.val.json` (draft content is served by the single-entry /json endpoint); - whole-entry add/replace push the MARKER (with patch_id) rather than the content, so the record's key set is right for drafts while record validation still only sees `isJson`; - remove passes through; move/copy become add-marker (+ remove of the source key for a move). Schemas are serialized lazily and only for modules that have patches, so the non-jsonValues case is unaffected. Co-Authored-By: Claude Opus 5 --- packages/server/src/ValOps.ts | 69 ++++++++++++++++++- .../server/src/ValOpsFS.jsonValues.test.ts | 56 ++++++++++++++- 2 files changed, 123 insertions(+), 2 deletions(-) diff --git a/packages/server/src/ValOps.ts b/packages/server/src/ValOps.ts index cea27ed8d..d4cba193f 100644 --- a/packages/server/src/ValOps.ts +++ b/packages/server/src/ValOps.ts @@ -413,6 +413,27 @@ export abstract class ValOps { error: GenericErrorMessage; }[] > = {}; + // Serialized schemas, resolved lazily and only for modules that actually + // have patches, so the common (non-jsonValues) case stays free. + const { schemas } = await this.initSources(); + const serializedSchemaCache = new Map< + ModuleFilePath, + SerializedSchema | undefined + >(); + const jsonValuesSchemaFor = ( + path: ModuleFilePath, + ): SerializedSchema | undefined => { + if (!serializedSchemaCache.has(path)) { + let serialized: SerializedSchema | undefined = undefined; + try { + serialized = schemas[path]?.["executeSerialize"](); + } catch { + // Serialization errors are reported elsewhere; treat as "no schema". + } + serializedSchemaCache.set(path, serialized); + } + return serializedSchemaCache.get(path); + }; for (const patchData of analysis.patches) { const path = patchData.path; if (sources[path] === undefined) { @@ -445,6 +466,13 @@ export abstract class ValOps { } else { const applicableOps: Patch = []; const fileFixOps: Record = {}; + // `.jsonValues()` entry values are opaque `{_type:"json"}` markers in + // the module source — their content lives in the entry's `*.val.json`. + // Ops that reach INTO an entry therefore cannot be applied here (and + // would fail with "Cannot replace object element which does not exist", + // poisoning the rest of this module's patch chain). See the per-op + // routing below. + const serializedSchema = jsonValuesSchemaFor(path); for (const op of patchData.patch) { if (op.op === "file") { if (op.value !== null) { @@ -465,7 +493,46 @@ export abstract class ValOps { // null value = delete: no patch_id to inject; the "remove" op in // the patch already removes the metadata entry from the source } else { - applicableOps.push(op); + const cls = serializedSchema + ? classifyJsonValuesOp(serializedSchema, op.path) + : ({ kind: "normal" } as const); + if (cls.kind === "normal") { + applicableOps.push(op); + } else if (cls.subPath.length > 0) { + // Content edit inside an entry: the module source is genuinely + // unaffected (the content lives in the `*.val.json`), so skip it. + // Draft content is served by the single-entry `/json` endpoint. + } else if (op.op === "add" || op.op === "replace") { + // Whole-entry add/replace: keep the record's KEY SET correct for + // drafts by writing the marker rather than the content. Record + // validation only asserts `isJson`, and + // `validateJsonValuesEntries` skips thunkless markers by design. + applicableOps.push({ + op: op.op, + path: op.path, + value: { + [VAL_EXTENSION]: "json", + patch_id: patchId, + } as JSONValue, + } as Operation); + } else if (op.op === "remove") { + applicableOps.push(op); + } else { + // move/copy of a whole entry: the destination key must appear, and + // for a move the source key must disappear. Both are key-set + // changes we can express with markers. + applicableOps.push({ + op: "add", + path: op.path, + value: { + [VAL_EXTENSION]: "json", + patch_id: patchId, + } as JSONValue, + } as Operation); + if (op.op === "move" && array.isNonEmpty(op.from)) { + applicableOps.push({ op: "remove", path: op.from }); + } + } } } const patchRes = applyPatch( diff --git a/packages/server/src/ValOpsFS.jsonValues.test.ts b/packages/server/src/ValOpsFS.jsonValues.test.ts index 7c9e2d1d4..adff3682d 100644 --- a/packages/server/src/ValOpsFS.jsonValues.test.ts +++ b/packages/server/src/ValOpsFS.jsonValues.test.ts @@ -1,4 +1,4 @@ -import { ModuleFilePath, PatchId, initVal } from "@valbuild/core"; +import { Internal, ModuleFilePath, PatchId, initVal } from "@valbuild/core"; import { Script } from "node:vm"; import { transform } from "sucrase"; import { ValOpsFS } from "./ValOpsFS"; @@ -96,6 +96,25 @@ async function prepareSingle( return ops.prepare({ ...analysis, patches }); } +async function getSourcesWith( + ops: ValOpsFS, + patch: OrderedPatches["patches"][number]["patch"], +) { + const patches: OrderedPatches["patches"] = [ + { + path: MODULE_PATH, + patchId: crypto.randomUUID() as PatchId, + patch, + createdAt: new Date().toISOString(), + authorId: null, + baseSha: await ops.getBaseSha(), + appliedAt: null, + }, + ]; + const analysis = ops.analyzePatches(patches); + return ops.getSources({ ...analysis, patches }); +} + describe("ValOpsFS jsonValues commit flow", () => { test("content edit writes only the *.val.json (not the .val.ts)", async () => { const { ops } = setup(); @@ -254,6 +273,41 @@ describe("ValOpsFS jsonValues commit flow", () => { expect(pc.appliedPatches[MODULE_PATH]).toHaveLength(1); }); + test("getSources: a content edit does not poison the module's patch chain", async () => { + // Regression: getSources applied entry-content ops with jsonOps against the + // opaque `{_type:"json"}` marker. That failed with "Cannot replace object + // element which does not exist", after which EVERY later patch for the + // module was skipped with "previous errors exists". + const { ops } = setup(); + const res = await getSourcesWith(ops, [ + { op: "replace", path: ["/blog/hello", "title"], value: "Hello!" }, + { op: "add", path: ["/blog/new"], value: { title: "New", order: 3 } }, + ]); + expect(res.errors[MODULE_PATH]).toBeUndefined(); + const source = res.sources[MODULE_PATH] as Record; + // Content edits leave the marker alone (content lives in the *.val.json). + expect(Internal.isJson(source["/blog/hello"])).toBe(true); + // A newly added entry appears as a marker, so the record's KEY SET is right. + expect(Internal.isJson(source["/blog/new"])).toBe(true); + expect(Object.keys(source).sort()).toEqual([ + "/blog/hello", + "/blog/new", + "/blog/world", + ]); + }); + + test("getSources: remove and rename update the key set", async () => { + const { ops } = setup(); + const res = await getSourcesWith(ops, [ + { op: "remove", path: ["/blog/world"] }, + { op: "move", from: ["/blog/hello"], path: ["/blog/renamed"] }, + ]); + expect(res.errors[MODULE_PATH]).toBeUndefined(); + const source = res.sources[MODULE_PATH] as Record; + expect(Object.keys(source)).toEqual(["/blog/renamed"]); + expect(Internal.isJson(source["/blog/renamed"])).toBe(true); + }); + test("move between different entries' content is an error", async () => { const { ops } = setup(); const pc = await prepareSingle(ops, [ From f5cdcc973479269ae0dcbd85a351c9e22ddfb5a2 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Tue, 28 Jul 2026 09:23:28 +0200 Subject: [PATCH 28/35] jsonValues: make /json draft-aware Adds the server half of the draft read path: `/json` can now replay pending patches onto an entry, so draft edits are visible to callers that do not own the patch chain themselves (the runtime in draft mode). - `applyJsonValuesEntryPatches` in patch/jsonValuesPatch.ts is the pure replay: route ops with `classifyJsonValuesOp`, rebase content sub-ops with `rebaseContentOp`, apply with jsonOps. Returns content / deleted / error. It is the read-side twin of the commit flow in `ValOps.prepare` -- same routing, but it produces a value instead of files and never touches the `.val.ts`. - `rebaseContentOp` moved out of ValOps.ts into jsonValuesPatch.ts next to the rest of the entry-patch helpers, and is now exported and directly tested. - `ValOps.getJsonEntry(path, key, {applyPatches})` resolves one entry: committed content from the import thunk (works in fs and http mode, no extra I/O), then pending patches on top. The ValServer handler is now a thin adapter over it. - `/json` gains `apply_patches`, defaulting to TRUE to mirror /sources/~. The Studio passes false explicitly: it owns in-flight client patches the server has not seen and applies them itself, so server-side application would double-apply. Also fixes the test harness: this suite evaluates the module in a `vm` whose `require` resolved `c.json(() => import("./x.val.json"))` relative to the test file rather than the temp root, so entry thunks could not load at all. Co-Authored-By: Claude Opus 5 --- packages/server/src/ValOps.ts | 147 ++++++++++++----- .../server/src/ValOpsFS.jsonValues.test.ts | 111 ++++++++++++- packages/server/src/ValServer.ts | 59 ++----- .../server/src/patch/jsonValuesPatch.test.ts | 153 +++++++++++++++++- packages/server/src/patch/jsonValuesPatch.ts | 147 ++++++++++++++++- packages/shared/src/internal/ApiRoutes.ts | 8 +- packages/ui/spa/ValSyncEngine.ts | 5 +- 7 files changed, 538 insertions(+), 92 deletions(-) diff --git a/packages/server/src/ValOps.ts b/packages/server/src/ValOps.ts index d4cba193f..8aff6f82d 100644 --- a/packages/server/src/ValOps.ts +++ b/packages/server/src/ValOps.ts @@ -38,9 +38,11 @@ import { TSOps, insertValJsonEntry, removeValJsonEntry } from "./patch/ts/ops"; import { analyzeValModule } from "./patch/ts/valModule"; import { analyzeJsonValuesEntries } from "./patch/ts/jsonValuesModule"; import { + applyJsonValuesEntryPatches, classifyJsonValuesOp, findNestedJsonValuesRecords, getNewJsonEntryPaths, + rebaseContentOp, resolveExistingJsonPath, } from "./patch/jsonValuesPatch"; import { validateJsonValuesEntries } from "./validateJsonValues"; @@ -80,47 +82,6 @@ const tsOps = new TSOps((document) => { ); }); -/** - * Rebases a patch op that targets a `.jsonValues()` entry's content so its paths - * are relative to the entry's `*.val.json` root (drops the record + entry-key - * prefix). Used to replay the op against the backing JSON file. - */ -function rebaseContentOp( - op: Operation, - prefixLen: number, -): result.Result { - const path = op.path.slice(prefixLen); - switch (op.op) { - case "add": - case "replace": - case "test": - return result.ok({ ...op, path }); - case "remove": { - if (!array.isNonEmpty(path)) { - return result.err( - new PatchError("Cannot remove the root of a jsonValues entry"), - ); - } - return result.ok({ ...op, path }); - } - case "move": { - const from = op.from.slice(prefixLen); - if (!array.isNonEmpty(from)) { - return result.err( - new PatchError("Cannot move from the root of a jsonValues entry"), - ); - } - return result.ok({ ...op, path, from }); - } - case "copy": - return result.ok({ ...op, path, from: op.from.slice(prefixLen) }); - case "file": - return result.err( - new PatchError("Cannot apply a file op to a jsonValues entry"), - ); - } -} - /** * `.jsonValues()` is only supported on a module's ROOT record/router. A nested * one is broken end to end (the `/json` endpoint keys entries by a single @@ -295,6 +256,110 @@ export abstract class ValOps { async getBaseSources(): Promise { return this.initSources().then((result) => result.sources); } + + /** + * Resolves the content of ONE `.jsonValues()` entry. + * + * The committed content comes from the entry's import thunk on the base + * source (so it works in both fs and http mode, with no extra I/O). With + * `applyPatches` (the default) any pending patches for that entry are then + * replayed on top, which is what makes draft edits visible to the runtime. + * + * Callers that apply patches themselves (the Studio, which owns + * in-flight client patches the server has not seen) must pass + * `applyPatches: false` or the same edits would be applied twice. + */ + async getJsonEntry( + moduleFilePath: ModuleFilePath, + entryKey: string, + opts?: { applyPatches?: boolean }, + ): Promise< + | { status: "success"; content: JSONValue | null } + | { status: "not-found"; message: string } + | { status: "error"; message: string } + | { status: "unauthorized"; message: string } + > { + const { sources, schemas } = await this.initSources(); + const moduleSource = sources[moduleFilePath]; + if (moduleSource === undefined || moduleSource === null) { + return { + status: "not-found", + message: `Module not found: ${moduleFilePath}`, + }; + } + if (typeof moduleSource !== "object" || Array.isArray(moduleSource)) { + return { + status: "not-found", + message: `Module is not a record: ${moduleFilePath}`, + }; + } + const marker = (moduleSource as Record)[entryKey]; + let baseContent: JSONValue | undefined = undefined; + if (marker !== undefined) { + if (!Internal.isJson(marker)) { + // Not a jsonValues entry — return the inlined value as-is (defensive). + return { status: "success", content: marker as JSONValue }; + } + const thunk = Internal.getJsonImport(marker); + if (thunk) { + try { + baseContent = ((await thunk()).default ?? null) as JSONValue; + } catch (e) { + return { + status: "error", + message: `Failed to load JSON entry '${entryKey}': ${ + e instanceof Error ? e.message : String(e) + }`, + }; + } + } else { + baseContent = null; + } + } + if (opts?.applyPatches === false) { + if (baseContent === undefined) { + return { + status: "not-found", + message: `Entry not found: ${entryKey} in ${moduleFilePath}`, + }; + } + return { status: "success", content: baseContent }; + } + const patchOps = await this.fetchPatches({ excludePatchOps: false }); + if (patchOps.error) { + return { status: "error", message: patchOps.error.message }; + } + if (patchOps.errors && Object.keys(patchOps.errors).length > 0) { + return { + status: "error", + message: `Could not fetch patches: ${JSON.stringify(patchOps.errors)}`, + }; + } + let serializedSchema: SerializedSchema | undefined = undefined; + try { + serializedSchema = schemas[moduleFilePath]?.["executeSerialize"](); + } catch { + // Serialization errors are reported elsewhere; treat as "no schema". + } + const res = applyJsonValuesEntryPatches({ + serializedSchema, + entryKey, + baseContent, + patches: patchOps.patches + .filter((p) => p.path === moduleFilePath && !p.appliedAt) + .map((p) => ({ patchId: p.patchId, patch: p.patch })), + }); + if (res.kind === "error") { + return { status: "error", message: res.message }; + } + if (res.kind === "deleted") { + return { + status: "not-found", + message: `Entry not found: ${entryKey} in ${moduleFilePath}`, + }; + } + return { status: "success", content: res.content }; + } async getSchemas(): Promise { return this.initSources().then((result) => result.schemas); } diff --git a/packages/server/src/ValOpsFS.jsonValues.test.ts b/packages/server/src/ValOpsFS.jsonValues.test.ts index adff3682d..90d4ff919 100644 --- a/packages/server/src/ValOpsFS.jsonValues.test.ts +++ b/packages/server/src/ValOpsFS.jsonValues.test.ts @@ -30,6 +30,9 @@ const JSON_FILES: Record = { function setup() { const { s, c, config } = initVal(); + // Use the OS temp dir (NOT the repo-local ".tmp", which other suites wipe). + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "val-jsonvalues-test")); + const evalModule = (code: string) => new Script( transform(code, { transforms: ["imports"] }).code, @@ -39,15 +42,23 @@ function setup() { if (p === "val.config") { return { s, c, config }; } + // Resolve the `c.json(() => import("./x.val.json"))` thunks against the + // module's directory in the TEMP root — plain `require` would resolve + // them relative to this test file. + if (p.startsWith("./") || p.startsWith("../")) { + const abs = path.resolve( + path.join(rootDir, path.dirname(MODULE_PATH)), + p, + ); + // eslint-disable-next-line @typescript-eslint/no-require-imports + return require(abs); + } // eslint-disable-next-line @typescript-eslint/no-require-imports return require(p); }, module: { exports: {} }, }); - // Use the OS temp dir (NOT the repo-local ".tmp", which other suites wipe). - const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "val-jsonvalues-test")); - const moduleAbs = path.join(rootDir, MODULE_PATH); fs.mkdirSync(path.dirname(moduleAbs), { recursive: true }); fs.writeFileSync( @@ -115,6 +126,100 @@ async function getSourcesWith( return ops.getSources({ ...analysis, patches }); } +/** Creates a real pending patch on disk, as the Studio would. */ +async function createPatch( + ops: ValOpsFS, + patch: OrderedPatches["patches"][number]["patch"], +) { + const existing = await ops.fetchPatches({ excludePatchOps: true }); + const last = existing.patches[existing.patches.length - 1]; + const res = await ops.createPatch( + MODULE_PATH, + patch, + crypto.randomUUID() as PatchId, + last + ? { type: "patch", patchId: last.patchId } + : { type: "head", headBaseSha: await ops.getBaseSha() }, + null, + null, + ); + if ("error" in res) { + throw new Error(`Could not create patch: ${JSON.stringify(res)}`); + } + return res; +} + +describe("ValOps.getJsonEntry", () => { + test("returns the committed content when there are no patches", async () => { + const { ops } = setup(); + const res = await ops.getJsonEntry(MODULE_PATH, "/blog/hello"); + expect(res).toEqual({ + status: "success", + content: { title: "Hello", order: 1 }, + }); + }); + + test("applies a pending content patch (the draft read path)", async () => { + const { ops } = setup(); + await createPatch(ops, [ + { op: "replace", path: ["/blog/hello", "title"], value: "Draft!" }, + ]); + const res = await ops.getJsonEntry(MODULE_PATH, "/blog/hello"); + expect(res).toEqual({ + status: "success", + content: { title: "Draft!", order: 1 }, + }); + }); + + test("applyPatches:false returns the committed content (what the Studio asks for)", async () => { + const { ops } = setup(); + await createPatch(ops, [ + { op: "replace", path: ["/blog/hello", "title"], value: "Draft!" }, + ]); + const res = await ops.getJsonEntry(MODULE_PATH, "/blog/hello", { + applyPatches: false, + }); + expect(res).toEqual({ + status: "success", + content: { title: "Hello", order: 1 }, + }); + }); + + test("resolves an entry that only exists in a pending patch", async () => { + const { ops } = setup(); + await createPatch(ops, [ + { op: "add", path: ["/blog/new"], value: { title: "New", order: 3 } }, + ]); + expect(await ops.getJsonEntry(MODULE_PATH, "/blog/new")).toEqual({ + status: "success", + content: { title: "New", order: 3 }, + }); + // ...but not when the caller wants committed content only. + expect( + ( + await ops.getJsonEntry(MODULE_PATH, "/blog/new", { + applyPatches: false, + }) + ).status, + ).toBe("not-found"); + }); + + test("an entry removed by a pending patch is not-found", async () => { + const { ops } = setup(); + await createPatch(ops, [{ op: "remove", path: ["/blog/world"] }]); + expect((await ops.getJsonEntry(MODULE_PATH, "/blog/world")).status).toBe( + "not-found", + ); + }); + + test("an unknown key is not-found", async () => { + const { ops } = setup(); + expect((await ops.getJsonEntry(MODULE_PATH, "/nope")).status).toBe( + "not-found", + ); + }); +}); + describe("ValOpsFS jsonValues commit flow", () => { test("content edit writes only the *.val.json (not the .val.ts)", async () => { const { ops } = setup(); diff --git a/packages/server/src/ValServer.ts b/packages/server/src/ValServer.ts index 08641c2fb..39fd10cb1 100644 --- a/packages/server/src/ValServer.ts +++ b/packages/server/src/ValServer.ts @@ -1363,8 +1363,9 @@ export const ValServer = ( // #region json // Loads the content of a single `.jsonValues()` entry by key, so the Studio - // can lazily load just the entry being opened. Returns the committed content - // (from the base source's import thunk); the client overlays its own patches. + // can lazily load just the entry being opened, and the runtime can read draft + // edits. With apply_patches (default true) pending patches for the entry are + // replayed server-side; the Studio passes false and overlays its own. "/json": { GET: async (req) => { const auth = getAuth(req.cookies); @@ -1376,53 +1377,23 @@ export const ValServer = ( } const moduleFilePath = req.query.path as ModuleFilePath; const key = req.query.key; - const sources = await serverOps.getBaseSources(); - const moduleSource = sources[moduleFilePath]; - if (moduleSource === undefined || moduleSource === null) { - return { - status: 404, - json: { message: `Module not found: ${moduleFilePath}` }, - }; - } - if (typeof moduleSource !== "object" || Array.isArray(moduleSource)) { - return { - status: 404, - json: { message: `Module is not a record: ${moduleFilePath}` }, - }; - } - const marker = (moduleSource as Record)[key]; - if (marker === undefined) { - return { - status: 404, - json: { message: `Entry not found: ${key} in ${moduleFilePath}` }, - }; + // Defaults to true, mirroring /sources/~. The Studio opts out. + const applyPatches = req.query.apply_patches !== false; + const res = await serverOps.getJsonEntry(moduleFilePath, key, { + applyPatches, + }); + if (res.status === "unauthorized") { + return { status: 401, json: { message: res.message } }; } - if (!Internal.isJson(marker)) { - // Not a jsonValues entry — return the inlined value as-is (defensive). - return { - status: 200, - json: { path: moduleFilePath, key, content: marker }, - }; + if (res.status === "not-found") { + return { status: 404, json: { message: res.message } }; } - const thunk = Internal.getJsonImport(marker); - let content: unknown = null; - if (thunk) { - try { - content = (await thunk()).default; - } catch (e) { - return { - status: 500, - json: { - message: `Failed to load JSON entry '${key}': ${ - e instanceof Error ? e.message : JSON.stringify(e) - }`, - }, - }; - } + if (res.status === "error") { + return { status: 500, json: { message: res.message } }; } return { status: 200, - json: { path: moduleFilePath, key, content }, + json: { path: moduleFilePath, key, content: res.content }, }; }, }, diff --git a/packages/server/src/patch/jsonValuesPatch.test.ts b/packages/server/src/patch/jsonValuesPatch.test.ts index 6301c7e40..f82ef90e5 100644 --- a/packages/server/src/patch/jsonValuesPatch.test.ts +++ b/packages/server/src/patch/jsonValuesPatch.test.ts @@ -1,10 +1,14 @@ -import { initVal, type SerializedSchema } from "@valbuild/core"; +import { initVal, PatchId, type SerializedSchema } from "@valbuild/core"; import { + applyJsonValuesEntryPatches, classifyJsonValuesOp, findNestedJsonValuesRecords, getNewJsonEntryPaths, + rebaseContentOp, resolveExistingJsonPath, } from "./jsonValuesPatch"; +import { result } from "@valbuild/core/fp"; +import type { Patch } from "@valbuild/core/patch"; const { s } = initVal(); @@ -146,6 +150,153 @@ describe("findNestedJsonValuesRecords", () => { }); }); +describe("rebaseContentOp", () => { + test("drops the record + entry-key prefix from path and from", () => { + const res = rebaseContentOp( + { op: "move", from: ["/a", "items", "0"], path: ["/a", "items", "2"] }, + 1, + ); + expect(result.isOk(res) && res.value).toEqual({ + op: "move", + from: ["items", "0"], + path: ["items", "2"], + }); + }); + + test("rejects removing the entry root", () => { + const res = rebaseContentOp({ op: "remove", path: ["/a"] }, 1); + expect(result.isErr(res)).toBe(true); + }); + + test("rejects a file op", () => { + const res = rebaseContentOp( + { + op: "file", + path: ["/a", "img"], + filePath: "/public/val/x.png", + value: "data:...", + remote: false, + }, + 1, + ); + expect(result.isErr(res)).toBe(true); + }); +}); + +describe("applyJsonValuesEntryPatches", () => { + const schema: SerializedSchema = s + .record(s.object({ title: s.string(), order: s.number() })) + .jsonValues() + ["executeSerialize"](); + const patch = (patch: Patch, n = 1) => ({ + patchId: `p${n}` as PatchId, + patch, + }); + + test("replays a content sub-op onto the committed content", () => { + const res = applyJsonValuesEntryPatches({ + serializedSchema: schema, + entryKey: "/a", + baseContent: { title: "A", order: 1 }, + patches: [patch([{ op: "replace", path: ["/a", "title"], value: "A!" }])], + }); + expect(res).toEqual({ + kind: "content", + content: { title: "A!", order: 1 }, + appliedPatchIds: ["p1"], + }); + }); + + test("ignores ops for other entries and other schemas' paths", () => { + const res = applyJsonValuesEntryPatches({ + serializedSchema: schema, + entryKey: "/a", + baseContent: { title: "A", order: 1 }, + patches: [ + patch([{ op: "replace", path: ["/b", "title"], value: "B!" }], 1), + patch([{ op: "replace", path: ["/a", "order"], value: 9 }], 2), + ], + }); + expect(res).toEqual({ + kind: "content", + content: { title: "A", order: 9 }, + appliedPatchIds: ["p2"], + }); + }); + + test("a whole-entry add creates content that did not exist", () => { + const res = applyJsonValuesEntryPatches({ + serializedSchema: schema, + entryKey: "/new", + baseContent: undefined, + patches: [ + patch([{ op: "add", path: ["/new"], value: { title: "N", order: 3 } }]), + ], + }); + expect(res).toEqual({ + kind: "content", + content: { title: "N", order: 3 }, + appliedPatchIds: ["p1"], + }); + }); + + test("a whole-entry remove reports deleted", () => { + const res = applyJsonValuesEntryPatches({ + serializedSchema: schema, + entryKey: "/a", + baseContent: { title: "A", order: 1 }, + patches: [patch([{ op: "remove", path: ["/a"] }])], + }); + expect(res).toEqual({ kind: "deleted", appliedPatchIds: ["p1"] }); + }); + + test("an entry missing from the base with no patches is deleted", () => { + const res = applyJsonValuesEntryPatches({ + serializedSchema: schema, + entryKey: "/nope", + baseContent: undefined, + patches: [], + }); + expect(res).toEqual({ kind: "deleted", appliedPatchIds: [] }); + }); + + test("editing an entry that does not exist is an error", () => { + const res = applyJsonValuesEntryPatches({ + serializedSchema: schema, + entryKey: "/nope", + baseContent: undefined, + patches: [ + patch([{ op: "replace", path: ["/nope", "title"], value: "x" }]), + ], + }); + expect(res.kind).toBe("error"); + }); + + test("a whole-entry move into this key is reported as an error (needs the source entry)", () => { + const res = applyJsonValuesEntryPatches({ + serializedSchema: schema, + entryKey: "/renamed", + baseContent: undefined, + patches: [patch([{ op: "move", from: ["/a"], path: ["/renamed"] }])], + }); + expect(res.kind).toBe("error"); + }); + + test("without a schema nothing is treated as an entry op", () => { + const res = applyJsonValuesEntryPatches({ + serializedSchema: undefined, + entryKey: "/a", + baseContent: { title: "A", order: 1 }, + patches: [patch([{ op: "replace", path: ["/a", "title"], value: "A!" }])], + }); + expect(res).toEqual({ + kind: "content", + content: { title: "A", order: 1 }, + appliedPatchIds: [], + }); + }); +}); + describe("resolveExistingJsonPath", () => { test("resolves a hand-placed import path relative to the module dir", () => { expect( diff --git a/packages/server/src/patch/jsonValuesPatch.ts b/packages/server/src/patch/jsonValuesPatch.ts index 6002d9f15..f1b052cc0 100644 --- a/packages/server/src/patch/jsonValuesPatch.ts +++ b/packages/server/src/patch/jsonValuesPatch.ts @@ -1,5 +1,17 @@ import * as path from "path"; -import type { SerializedSchema } from "@valbuild/core"; +import type { PatchId, SerializedSchema } from "@valbuild/core"; +import { array, result } from "@valbuild/core/fp"; +import { + applyPatch, + deepClone, + JSONOps, + JSONValue, + Operation, + Patch, + PatchError, +} from "@valbuild/core/patch"; + +const jsonOps = new JSONOps(); /** * Classification of a single patch op against a module's serialized schema, @@ -157,6 +169,139 @@ export function getNewJsonEntryPaths( return { jsonPath, importPath }; } +/** + * Rebases a patch op that targets a `.jsonValues()` entry's content so its paths + * are relative to the entry's `*.val.json` root (drops the record + entry-key + * prefix). Used to replay the op against the backing JSON file. + */ +export function rebaseContentOp( + op: Operation, + prefixLen: number, +): result.Result { + const path = op.path.slice(prefixLen); + switch (op.op) { + case "add": + case "replace": + case "test": + return result.ok({ ...op, path }); + case "remove": { + if (!array.isNonEmpty(path)) { + return result.err( + new PatchError("Cannot remove the root of a jsonValues entry"), + ); + } + return result.ok({ ...op, path }); + } + case "move": { + const from = op.from.slice(prefixLen); + if (!array.isNonEmpty(from)) { + return result.err( + new PatchError("Cannot move from the root of a jsonValues entry"), + ); + } + return result.ok({ ...op, path, from }); + } + case "copy": + return result.ok({ ...op, path, from: op.from.slice(prefixLen) }); + case "file": + return result.err( + new PatchError("Cannot apply a file op to a jsonValues entry"), + ); + } +} + +/** The outcome of replaying pending patches onto one `.jsonValues()` entry. */ +export type JsonEntryResolution = + | { kind: "content"; content: JSONValue | null; appliedPatchIds: PatchId[] } + | { kind: "deleted"; appliedPatchIds: PatchId[] } + | { kind: "error"; message: string; patchId?: PatchId }; + +/** + * Replays the ops of `patches` that target ONE `.jsonValues()` entry onto its + * committed content, yielding the entry's draft content. + * + * This is the read-side counterpart to the commit flow in `ValOps.prepare`: + * both route ops with {@link classifyJsonValuesOp} and replay content sub-ops + * with {@link rebaseContentOp}, but this one produces a value instead of files + * and never touches the `.val.ts`. + * + * Root-only, like the rest of the `.jsonValues()` machinery: ops targeting a + * nested record are ignored (nested `.jsonValues()` is rejected at startup). + */ +export function applyJsonValuesEntryPatches(args: { + serializedSchema: SerializedSchema | undefined; + entryKey: string; + /** `undefined` when the entry does not exist in the committed source. */ + baseContent: JSONValue | undefined; + /** Ordered, already filtered to the entry's module. */ + patches: { patchId: PatchId; patch: Patch }[]; +}): JsonEntryResolution { + const { serializedSchema, entryKey, baseContent, patches } = args; + let content: JSONValue | undefined = baseContent; + let deleted = false; + const appliedPatchIds: PatchId[] = []; + for (const { patchId, patch } of patches) { + let touched = false; + for (const op of patch) { + if (op.op === "file") { + continue; + } + const cls = serializedSchema + ? classifyJsonValuesOp(serializedSchema, op.path) + : ({ kind: "normal" } as const); + if ( + cls.kind !== "entry" || + cls.recordPath.length > 0 || + cls.entryKey !== entryKey + ) { + continue; + } + touched = true; + if (cls.subPath.length === 0) { + if (op.op === "add" || op.op === "replace") { + content = op.value as JSONValue; + deleted = false; + } else if (op.op === "remove") { + content = undefined; + deleted = true; + } else { + // move/copy INTO this key: the content comes from the source entry, + // which the caller must resolve (it is a different `*.val.json`). + return { + kind: "error", + message: `Cannot resolve '${op.op}' of jsonValues entry '${entryKey}' from its own content`, + patchId, + }; + } + continue; + } + if (content === undefined) { + return { + kind: "error", + message: `Cannot edit jsonValues entry '${entryKey}': it does not exist`, + patchId, + }; + } + const rebased = rebaseContentOp(op, cls.recordPath.length + 1); + if (result.isErr(rebased)) { + return { kind: "error", message: rebased.error.message, patchId }; + } + const applied = applyPatch(deepClone(content), jsonOps, [rebased.value]); + if (result.isErr(applied)) { + return { kind: "error", message: applied.error.message, patchId }; + } + content = applied.value; + } + if (touched) { + appliedPatchIds.push(patchId); + } + } + if (deleted || content === undefined) { + return { kind: "deleted", appliedPatchIds }; + } + return { kind: "content", content, appliedPatchIds }; +} + /** * Resolves an EXISTING entry's `*.val.json` path (relative to rootDir) from the * `import(...)` path recorded in the `.val.ts` thunk (from diff --git a/packages/shared/src/internal/ApiRoutes.ts b/packages/shared/src/internal/ApiRoutes.ts index ba350a339..333230823 100644 --- a/packages/shared/src/internal/ApiRoutes.ts +++ b/packages/shared/src/internal/ApiRoutes.ts @@ -1012,13 +1012,19 @@ export const Api = { }, // Loads the content of a single `.jsonValues()` record/router entry by key, // so the Studio can lazily load just the entry being opened (instead of the - // whole record). Returns the committed entry content; client overlays patches. + // whole record), and so the runtime can read draft edits in draft mode. + // + // `apply_patches` defaults to TRUE (as on `/sources/~`): the server replays + // pending patches for the entry. The Studio passes `false` explicitly, because + // it owns in-flight client patches the server has not seen yet and applies + // them itself — letting the server apply them too would double-apply. "/json": { GET: { req: { query: { path: onlyOneStringQueryParam, key: onlyOneStringQueryParam, + apply_patches: onlyOneBooleanQueryParam.optional(), }, cookies: { [VAL_SESSION_COOKIE]: z.string().optional() }, }, diff --git a/packages/ui/spa/ValSyncEngine.ts b/packages/ui/spa/ValSyncEngine.ts index bec0ee21f..f612c06ab 100644 --- a/packages/ui/spa/ValSyncEngine.ts +++ b/packages/ui/spa/ValSyncEngine.ts @@ -967,7 +967,10 @@ export class ValSyncEngine { // about to get was produced before that invalidation. this.staleJsonEntries.delete(loadingKey); const promise = this.client("/json", "GET", { - query: { path: moduleFilePath, key }, + // apply_patches=false: we own in-flight client patches the server has not + // seen yet and apply them ourselves in `getPatchedSource`. Letting the + // server apply them too would double-apply. + query: { path: moduleFilePath, key, apply_patches: false }, }) .then((res) => { if (res.status === 200) { From 91a56a41e77c4cb4e7b4e98ffafc81b3ecdf35e7 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Tue, 28 Jul 2026 09:29:18 +0200 Subject: [PATCH 29/35] jsonValues: give entries edit tags + read drafts in the RSC path Two halves of the enabled/draft runtime path. Edit tags. `stegaEncode` seeds its recursion only from the selector branch, so calling it on a `.jsonValues()` entry's raw content -- plain JSON with no Path/GetSchema symbols -- left `recOpts` undefined and every string hit the `!recOpts` bail in the encoder. All four jsonValues call sites were therefore silently identity transforms: entries had NO click-to-edit at all, even with Val enabled. `stegaEncode` now takes an optional `root: {path, schema}` seed, and `getJsonEntryStegaRoot` builds it from the entry's path plus the serialized item schema. A `title` field ends up tagged `/app/support/[slug]/page.val.ts?p="/support/faq"."title"` -- the same shape `findUnloadedJsonEntryKey` walks, so click-to-edit and lazy load line up. Note this needed a seed parameter rather than a real sub-selector: `newSelectorProxy` is not exported from core, it would require the private `RecordSchema["item"]` class instance, and re-entering the selector branch would make `getModuleIds` report the sub-path as a module id, breaking store lookups keyed by ModuleFilePath. Draft content (RSC). `fetchValKey` and the jsonValues branch of `fetchValRoute` now read through the in-process `/json` endpoint when Val is enabled, so uncommitted Studio edits are visible; they fall back to the local committed thunk when the draft read yields nothing. Both also call SET_AUTO_TAG_JSX_ENABLED, without which a page whose only Val call is fetchValKey/fetchValRoute gets no data-val-path attributes even with correct stega strings. The client hooks get the stega seed but still render committed content in draft mode -- same limitation `useValStega` has today. Wiring them to drafts needs the overlay emitter to carry patched sources, which is a separate change. Co-Authored-By: Claude Opus 5 --- packages/next/src/client/initValClient.ts | Bin 9364 -> 9487 bytes packages/next/src/routeFromVal.ts | 30 ++++ packages/next/src/rsc/initValRsc.ts | 161 ++++++++++++++----- packages/react/src/stega/stegaEncode.test.ts | 43 +++++ packages/react/src/stega/stegaEncode.ts | 15 +- 5 files changed, 208 insertions(+), 41 deletions(-) diff --git a/packages/next/src/client/initValClient.ts b/packages/next/src/client/initValClient.ts index 1f7fa1fe072412c9004ba50c468843ae5e70131d..23fbcf93f528a713fa2272a8f2d5f007f802d8f0 100644 GIT binary patch delta 132 zcmbQ@+3&SsBBQWZaekg_UP)19a7k) | unknown, methodName: string, diff --git a/packages/next/src/rsc/initValRsc.ts b/packages/next/src/rsc/initValRsc.ts index c4c548bbc..09fc2a7b6 100644 --- a/packages/next/src/rsc/initValRsc.ts +++ b/packages/next/src/rsc/initValRsc.ts @@ -21,6 +21,7 @@ import { VAL_SESSION_COOKIE } from "@valbuild/shared/internal"; import { createValServer, ValServer } from "@valbuild/server"; import { VERSION } from "../version"; import { + getJsonEntryStegaRoot, getValRouteUrlFromVal, initValRouteFromVal, isJsonValuesRecordSchema, @@ -223,17 +224,32 @@ const initFetchValRouteStega = if (!url) { return null as FetchValRouteReturnType; } - const content = await loadJsonEntryContent(source, url); - if (content === undefined) { - return null as FetchValRouteReturnType; - } let enabled = false; try { enabled = await isEnabled(); } catch { // not in a server context where draftMode is readable — treat as disabled } - return stegaEncode(content, { disabled: !enabled }); + let content: unknown = undefined; + if (enabled && path) { + SET_AUTO_TAG_JSX_ENABLED(true); + content = await loadDraftJsonEntry( + valServerPromise, + getCookies, + path as unknown as ModuleFilePath, + url, + ); + } + if (content === undefined) { + content = await loadJsonEntryContent(source, url); + } + if (content === undefined) { + return null as FetchValRouteReturnType; + } + return stegaEncode(content, { + disabled: !enabled, + root: getJsonEntryStegaRoot(selector, url), + }); } const fetchVal = initFetchValStega( config, @@ -277,6 +293,49 @@ async function loadJsonEntryContent( return (await thunk()).default; } +/** + * Loads a single `.jsonValues()` entry's DRAFT content via the in-process + * `/json` endpoint (which replays pending patches). Returns `undefined` when the + * entry has no draft content to serve — the caller then falls back to the + * locally-bundled committed content. + */ +async function loadDraftJsonEntry( + valServerPromise: Promise, + getCookies: () => Promise<{ + get(name: string): { name: string; value: string } | undefined; + }>, + moduleFilePath: ModuleFilePath, + key: string, +): Promise { + let cookies; + try { + cookies = await getCookies(); + } catch { + // not in a server context where cookies are readable + return undefined; + } + const valServer = await valServerPromise; + const res = await valServer["/json"]["GET"]({ + query: { path: moduleFilePath, key, apply_patches: true }, + cookies: { + [VAL_SESSION_COOKIE]: cookies?.get(VAL_SESSION_COOKIE)?.value, + }, + }); + if (res.status === 200) { + return res.json.content; + } + if (res.status === 401) { + console.warn("Val: authentication error: ", res.json.message); + return undefined; + } + if (res.status === 404) { + // No such entry in the draft state (e.g. removed by a pending patch). + return undefined; + } + console.error("Val: could not load draft JSON entry: ", res.json.message); + return undefined; +} + // The (loosened) content type a single `.jsonValues()` entry resolves to. type JsonEntryContentOf>> = T extends ValModule @@ -288,43 +347,59 @@ type JsonEntryContentOf>> = : never; /** - * Resolves a SINGLE entry of a `.jsonValues()` record/router by key, loading - * only that entry's backing `*.val.json` (one dynamic import) instead of the - * whole record. This is the runtime-scaling counterpart to the eager `fetchVal`. + * Resolves ONE `.jsonValues()` entry by key, loading only that entry instead of + * the whole record — the runtime-scaling counterpart to the eager `fetchVal`. * * Production (Val disabled): resolves the entry's lazy import thunk from the - * local module and returns its content. + * locally-bundled module. One dynamic import, no server round-trip. * - * TODO(enabled/Studio): when Val is enabled, draft edits to the entry live in - * patches on the server. Until the single-entry fetch endpoint is wired, this - * resolves the locally-bundled (committed) content. Visual-editing tags for the - * resolved entry are also a follow-up (needs a sub-selector at the entry path). + * Enabled (draft mode): reads the entry through `/json`, which replays pending + * patches, so uncommitted Studio edits show up. Falls back to the local thunk if + * the draft read yields nothing. */ const initFetchValKeyStega = - // NOTE: only needs `isEnabled` for the production read path. The - // enabled/Studio draft path (TODO) will also need the server deps the sibling - // init* factories take. - (isEnabled: () => Promise) => - async >>( - selector: T, - key: string, - ): Promise | undefined> => { - let enabled = false; - try { - enabled = await isEnabled(); - } catch { - // not in a server context where draftMode is readable — treat as disabled - } - const source = selector && Internal.getSource(selector); - const content = await loadJsonEntryContent(source, key); - if (content === undefined) { - // missing key, or transport marker without a runtime thunk — see TODO above - return undefined; - } - return stegaEncode(content, { - disabled: !enabled, - }); - }; + ( + valServerPromise: Promise, + isEnabled: () => Promise, + getCookies: () => Promise<{ + get(name: string): { name: string; value: string } | undefined; + }>, + ) => + async >>( + selector: T, + key: string, + ): Promise | undefined> => { + let enabled = false; + try { + enabled = await isEnabled(); + } catch { + // not in a server context where draftMode is readable — treat as disabled + } + const source = selector && Internal.getSource(selector); + const moduleFilePath = + selector && (Internal.getValPath(selector) as unknown as ModuleFilePath); + let content: unknown = undefined; + if (enabled && moduleFilePath) { + SET_AUTO_TAG_JSX_ENABLED(true); + content = await loadDraftJsonEntry( + valServerPromise, + getCookies, + moduleFilePath, + key, + ); + } + if (content === undefined) { + content = await loadJsonEntryContent(source, key); + } + if (content === undefined) { + // missing key, or transport marker without a runtime thunk + return undefined; + } + return stegaEncode(content, { + disabled: !enabled, + root: getJsonEntryStegaRoot(selector, key), + }); + }; const initFetchValRouteUrl = ( @@ -434,9 +509,15 @@ export function initValRsc( return await rscNextConfig.cookies(); }, ), - fetchValKeyStega: initFetchValKeyStega(async () => { - return (await rscNextConfig.draftMode()).isEnabled; - }), + fetchValKeyStega: initFetchValKeyStega( + valServerPromise, + async () => { + return (await rscNextConfig.draftMode()).isEnabled; + }, + async () => { + return await rscNextConfig.cookies(); + }, + ), fetchValRouteStega: initFetchValRouteStega( config, valApiEndpoints, diff --git a/packages/react/src/stega/stegaEncode.test.ts b/packages/react/src/stega/stegaEncode.test.ts index 3263e9add..e51534b30 100644 --- a/packages/react/src/stega/stegaEncode.test.ts +++ b/packages/react/src/stega/stegaEncode.test.ts @@ -293,5 +293,48 @@ describe("stega transform", () => { }); }); +describe("stegaEncode root seed (jsonValues entries)", () => { + // A `.jsonValues()` entry's content is plain JSON — it carries no selector + // path/schema — so without a `root` seed stegaEncode cannot tag anything. + const itemSchema = s.object({ title: s.string(), body: s.string() }); + const entryPath = '/app/support/[slug]/page.val.ts?p="/support/faq"'; + const content = { title: "FAQ", body: "Body" }; + + test("without a root seed it is an identity transform (the bug)", () => { + const res = stegaEncode(content, {}); + expect(res).toEqual(content); + expect(vercelStegaDecode(res.title)).toBeUndefined(); + }); + + test("with a root seed each string is tagged at the entry sub-path", () => { + const res = stegaEncode(content, { + root: { + path: entryPath, + schema: itemSchema["executeSerialize"](), + }, + }); + expect(vercelStegaSplit(res.title).cleaned).toBe("FAQ"); + expect(vercelStegaDecode(res.title)).toEqual({ + origin: "val.build", + data: { valPath: `${entryPath}."title"` }, + }); + expect(vercelStegaDecode(res.body)).toEqual({ + origin: "val.build", + data: { valPath: `${entryPath}."body"` }, + }); + }); + + test("disabled wins over the root seed", () => { + const res = stegaEncode(content, { + disabled: true, + root: { + path: entryPath, + schema: itemSchema["executeSerialize"](), + }, + }); + expect(res).toEqual(content); + }); +}); + type SchemaOf> = T extends Schema ? S : never; diff --git a/packages/react/src/stega/stegaEncode.ts b/packages/react/src/stega/stegaEncode.ts index fb532aa59..f4f1a6fac 100644 --- a/packages/react/src/stega/stegaEncode.ts +++ b/packages/react/src/stega/stegaEncode.ts @@ -351,6 +351,19 @@ export function stegaEncode( opts: { getModule?: (modulePath: string) => any; disabled?: boolean; + /** + * Seeds the recursion for a RAW source value that is not a selector, so its + * strings still get edit tags. + * + * Needed for a `.jsonValues()` entry loaded by key: its content is plain + * JSON with no `Path`/`GetSchema` symbols, so the selector branch below + * cannot fire and — without this — every string hits the `!recOpts` bail in + * the encoder and the whole call is an identity transform. + * + * `path` is the entry's path (`Internal.createValPathOfItem(modulePath, key)`) + * and `schema` is the SERIALIZED item schema. + */ + root?: { path: any; schema: any }; }, ): any { function rec( @@ -528,7 +541,7 @@ export function stegaEncode( ); return sourceOrSelector; } - return rec(input); + return rec(input, opts.disabled ? undefined : opts.root); } function isRecordSchema( From 20905c04f381bb8f90329e15fd5994e9f5b4892e Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Tue, 28 Jul 2026 09:31:24 +0200 Subject: [PATCH 30/35] Tracker: draft runtime path + edit tags done; client hooks still committed-only Ticks the /json draft-awareness and getSources boxes in Phase 2 and the edit-tag + RSC draft boxes in Phase 4. Records what is still open with the reason: client useValKey/useValRoute render committed content in draft mode because overlayEmitter is handed the raw un-patched /sources/~ module, and draft-added routes are unreachable via fetchValRoute because params -> key resolves from the local source. The manual Studio walkthrough (V1-V9) is still unrun. Co-Authored-By: Claude Opus 5 --- docs/plans/jsonValues.md | 72 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md index 01cdcb689..90656cced 100644 --- a/docs/plans/jsonValues.md +++ b/docs/plans/jsonValues.md @@ -10,6 +10,18 @@ ## Current state / resume here +> **Draft runtime path DONE (2026-07-28):** the last Phase 4 box is closed on the server + RSC side. +> `/json` takes `apply_patches` (default **true**, mirroring `/sources/~`) and replays pending +> patches via the new pure `applyJsonValuesEntryPatches`; the Studio passes `false` because it owns +> in-flight client patches. RSC `fetchValKey`/`fetchValRoute` read drafts through it when enabled and +> fall back to the local thunk. **jsonValues entries now get click-to-edit at all** — `stegaEncode` +> gained an optional `root` seed (entry path + serialized item schema), without which every +> jsonValues stega call was a silent identity transform. Also fixed `getSources` poisoning a +> jsonValues module's whole patch chain (observed live on the example support router). +> Still open: client `useValKey`/`useValRoute` render committed content in draft mode (needs the +> overlay emitter to carry patched sources — same limitation `useValStega` has today), and the +> **manual Studio walkthrough (V1–V9) has still NOT been run.** + > **Studio hardening DONE (2026-07-27):** the Phase 3 defects are fixed. Renaming an entry now > commits (`move`/`copy` arms in `ValOps.prepare`), a published edit no longer appears to revert > (json entry cache is invalidated on publish), a failed `/json` load renders an error instead of a @@ -203,10 +215,20 @@ entries; runtime/Studio/validation work one entry at a time; zero overhead when (`ValOpsFS.jsonValues.test.ts`). ✅ (Still to confirm: shallow `/sources/~` serialization end to end against the Studio.) - [x] Core eager resolver `Internal.resolveJsonValues(source)` (for `fetchVal`/`useVal`). ✅ -- [ ] `ValServer.ts`: endpoint to fetch one entry's content (draft-aware via `patch_id`); - `/sources/~` returns shallow markers for json records. -- [ ] **Verify**: `pnpm test packages/server/...` green (ops add/replace/remove, loader fixture, - incremental validation). +- [x] `ValServer.ts` `/json`: fetch one entry's content, **draft-aware** (2026-07-28). It takes + `apply_patches` (default TRUE, mirroring `/sources/~`); the Studio passes `false` because it owns + in-flight client patches and would otherwise double-apply. The handler is a thin adapter over + `ValOps.getJsonEntry(path, key, {applyPatches})`, which reads committed content from the import + thunk (fs + http, no extra I/O) and replays pending patches with `applyJsonValuesEntryPatches`. + NOTE: no `patch_id[]` pinning — no caller has patch ids, and `fetchPatches({patchIds})` is + already there if one appears. `/sources/~` shallow markers fall out of `JSON.stringify` dropping + the thunk. ✅ +- [x] `getSources` is jsonValues-aware (2026-07-28). It used to apply entry-content ops with `jsonOps` + against the opaque marker → "Cannot replace object element which does not exist", which then + marked the module poisoned and skipped **every** later patch for it. Content sub-ops are now + skipped (the module source is genuinely unaffected); whole-entry add/replace/move/copy push the + MARKER so the key set stays right for drafts. ✅ +- [x] **Verify**: `pnpm test packages/server/...` green. ✅ ## Phase 3 — UI (`packages/ui/spa`) — NEXT MILESTONE (do with Studio running) @@ -266,9 +288,30 @@ JSONValue>>` and `private loadingJsonEntries: Set<"mfp\0key">`. Add `requestJson gained a `NonNullable[string] extends JsonSource ? C | null : …` branch. ✅ - [x] Example wires `fetchValRoute` (support pages) — `next build` green; `/support/[slug]` is a single-entry dynamic route. (Was `fetchValKey`.) ✅ -- [ ] Enabled/Studio draft path: resolve draft content via `/json` (+ sub-selector stega tags). - (fetchValKey/useValKey + fetchValRoute/useValRoute all still read the LOCAL committed thunk on - the enabled/draft path — draft edits aren't reflected until commit.) +- [x] **Edit tags for jsonValues entries** (2026-07-28). `stegaEncode` seeds its recursion only from + the selector branch, so calling it on an entry's RAW content (plain JSON, no `Path`/`GetSchema` + symbols) left `recOpts` undefined and every string hit the `!recOpts` bail — all four jsonValues + call sites were **silent identity transforms**, i.e. entries had no click-to-edit at all. + `stegaEncode` now takes `root?: {path, schema}`; `getJsonEntryStegaRoot` (in `routeFromVal.ts`) + builds it from `Internal.createValPathOfItem(modulePath, key)` + the serialized `item` schema, so + a field is tagged `…?p="/support/faq"."title"` — the shape `findUnloadedJsonEntryKey` walks. + Also needs `SET_AUTO_TAG_JSX_ENABLED(true)`, which only `initFetchValStega` used to call. + **Not** a sub-selector: `newSelectorProxy` isn't exported from core, it needs the private + `RecordSchema["item"]` instance, and re-entering the selector branch makes `getModuleIds` report + the sub-path as a module id. ✅ +- [x] RSC draft path (2026-07-28): `fetchValKey` + the jsonValues branch of `fetchValRoute` read + through the in-process `/json` (`loadDraftJsonEntry`) when enabled, falling back to the local + committed thunk. `initFetchValKeyStega` now takes `valServerPromise` + `getCookies`. ✅ +- [ ] **Client hooks draft path**: `useValKey`/`useValRoute` have the stega tags but still render + COMMITTED content in draft mode — the same limitation `useValStega` has today. The blocker is + that `overlayEmitter` is handed the raw `/sources/~` module (`apply_patches:false`, json entries + still markers), so reading `valOverlayContext.store` would be a no-op. Fixing that means + emitting `getPatchedSource(...)` (ideally from `invalidateSource`, the single choke point) and + having the overlay's engine proactively `requestJsonEntry` for entries that have drafts. That + changes `useValStega` behaviour for ALL client components, so it wants its own change. +- [ ] Draft-added routes are not reachable via `fetchValRoute` (params → key is resolved from the + LOCAL source), and draft-removed routes still route, then 404 → `null`. Revisit by resolving the + key set from `/sources/~` now that `getSources` keeps it correct for drafts. ## Phase 5 — Example + CI gate @@ -302,6 +345,21 @@ unconditionally (accepted "validation takes more time" tradeoff). ## Changelog +- **Session 5 (2026-07-28)**: enabled/draft runtime path (server + RSC) and edit tags. + - Merged `main` (which fixed an unrelated blocker: the publish gate read the RAW + `errors.validationErrors` instead of the surfaced snapshot, so every `s.images()`/`s.files()` + gallery's always-on `check-unique-folder`/`check-all-files` errors blocked publish while the + Studio showed none — `0fcfecf0`). + - `getSources` jsonValues-aware — entry-content ops no longer poison the module's patch chain. + - `applyJsonValuesEntryPatches` + `rebaseContentOp` moved into `patch/jsonValuesPatch.ts`; new + `ValOps.getJsonEntry`; `/json` gained `apply_patches` (default true; Studio opts out). + - `stegaEncode` `root` seed + `getJsonEntryStegaRoot` → jsonValues entries finally get edit tags; + `SET_AUTO_TAG_JSX_ENABLED` now also set by the single-entry readers. + - RSC `fetchValKey`/`fetchValRoute` read drafts via `/json`. + - Test-harness fix: `ValOpsFS.jsonValues.test.ts` evaluated the module in a `vm` whose `require` + resolved `import("./x.val.json")` relative to the TEST FILE, so entry thunks never loaded. + - Tests: +2 `getSources` routing, +6 `ValOps.getJsonEntry`, +11 `rebaseContentOp` / + `applyJsonValuesEntryPatches`, +3 `stegaEncode` root seed (incl. a guard that no seed ⇒ identity). - **Session 4 (2026-07-27)**: Studio hardening — closes the Phase 3 defects. - **Rename/duplicate now commit.** `ValOps.prepare` classifies `op.from` as well as `op.path` and gained `move`/`copy` arms: load the source entry's content, remove the old thunk (move only), From 80aa4120eb027667582b74f630ef96a547d924ea Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Thu, 30 Jul 2026 10:14:07 +0200 Subject: [PATCH 31/35] =?UTF-8?q?Tracker:=20add=20Phase=206=20=E2=80=94=20?= =?UTF-8?q?reference=20integrity,=20list=20view,=20search=20over=20unloade?= =?UTF-8?q?d=20entries?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Studio's global scans (route refs, keyOf refs, referenced files) and search silently skip un-loaded `{_type:"json"}` markers. That makes the delete gate and the rename fixup answer "no references" for a ref living inside an entry the user happens not to have opened — nondeterministically, since it depends on which entries were visited this session. Phase 6 records the fix: - Scoping rule (direction decides): incoming refs need only the record's key set, which the markers already carry, so the common case loads NOTHING. Content is needed only when a jsonValues entry's own item schema points outward at the thing being edited — a schema-only predicate. `route` is the one over-approximation (SerializedRouteSchema names no target module). - Batch `/json` (`keys` + `offset`/`limit`), with `initSources`/`fetchPatches` hoisted out of the per-entry loop. - Record list view: it already renders a per-entry preview for every key, unvirtualized, so a jsonValues record shows N broken previews today. Virtualize with @tanstack/react-virtual and load only the rendered window; skeletons on all three preview paths so a marker never reaches a preview component. - Search: lazy (nothing before the first query), partial results immediately, with a percentage while the index fills. - Windowed `.render()` list layouts — gated on a decision, since renders are null Studio-wide today (`/sources/~` only computes them when apply_patches is true). - Order of work (8 steps), V10–V18, and two decision items for Fredrik: browser caching of `/json`, and the renders pincer (select is server-only, patched sources are client-only). Also supersedes V1 ("zero /json on open") and adds watch-list entries for `jsonEntryContents` eviction and for skipping markers being unsafe in scans that gate mutations. --- docs/plans/jsonValues.md | 313 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 313 insertions(+) diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md index 90656cced..1c0a053c1 100644 --- a/docs/plans/jsonValues.md +++ b/docs/plans/jsonValues.md @@ -10,6 +10,19 @@ ## Current state / resume here +> **Reference-integrity defect found (2026-07-30) — Phase 6 is the next milestone, NOT started.** +> The Studio's three global scans (route refs, keyOf refs, referenced files) and search silently skip +> un-loaded `{_type:"json"}` markers, so the delete gate and the rename fixup can both answer "no +> references" for a ref that lives inside an entry the user happens not to have opened. That is a +> data-integrity hole, not a cosmetic one, and the result is nondeterministic (it depends on session +> history). Phase 6 below fixes it with a **schema-derived scoping rule** (in the common case NOTHING +> needs loading) plus a **batch `/json`** for the cases that do, and for search. Read Phase 6 before +> touching any of the `get*References` / `traverseSchemas` / search files. +> Phase 6 also covers the **record LIST view**, which turns out to be the most user-visible half: it +> renders a per-entry preview for every key, unvirtualized, so a jsonValues record currently shows N +> broken previews. Fix = virtualize with `@tanstack/react-virtual` + load only the rendered window. +> This **supersedes V1** ("zero `/json` on open"). + > **Draft runtime path DONE (2026-07-28):** the last Phase 4 box is closed on the server + RSC side. > `/json` takes `apply_patches` (default **true**, mirroring `/sources/~`) and replays pending > patches via the new pure `applyJsonValuesEntryPatches`; the Studio passes `false` because it owns @@ -263,6 +276,9 @@ JSONValue>>` and `private loadingJsonEntries: Set<"mfp\0key">`. Add `requestJson in-flight response. `retryJsonEntry` / `ensureJsonEntry` added. - [ ] **Verify** (Studio running) — **NOT YET RUN**. Walkthrough: - **V1** open the router module → both keys listed, **zero** `/api/val/json` requests. + ⚠️ **SUPERSEDED by Phase 6**: once the list view loads previews for visible rows, opening a module + legitimately requests the visible window. Verify V16 instead (bounded by visible + overscan, never + by total key count). "Zero on open" only holds while the list renders no per-entry preview. - **V2** open an entry → exactly one `/json`; revisiting it → no new request. - **V3** edit `title`, publish → only `content/faq.val.json` modified, `page.val.ts` untouched, and the new title **survives the publish without a reload**. @@ -322,6 +338,273 @@ JSONValue>>` and `private loadingJsonEntries: Set<"mfp\0key">`. Add `requestJson `pnpm test`, `pnpm run build` (root preconstruct+ui; remember `pnpm preconstruct dev` after), `cd examples/next && pnpm run build`. +## Phase 6 — Reference integrity + search over un-loaded entries — NEXT MILESTONE + +### The defect (found 2026-07-30, reviewing PR #453) + +Four client-side global scans are blind to un-loaded entry content: + +| Scan | Marker handling | Backs | +| -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | ---------------------------------- | +| [getRouteReferences.ts:31](../../packages/ui/spa/components/getRouteReferences.ts#L31) | `isJson` → `return` | route refs | +| [traverseSchemas.ts:38](../../packages/ui/spa/components/traverseSchemas.ts#L38) | `isJson` → `return` | `getKeysOf` + `getReferencedFiles` | +| [traverseSchemaSource.ts](../../packages/ui/spa/utils/traverseSchemaSource.ts) | **none** — marker hits the object/record branch, `_type` matches no sub-schema, `continue` ⇒ indexes nothing | the LIVE search worker | +| [createSearchIndex.ts:25](../../packages/ui/spa/search/createSearchIndex.ts#L25) | `isJson` → `return` | **nothing — file is DEAD** | + +Why it is not cosmetic: route refs and keyOf refs merge into `allRefs` +([ArrayAndRecordTools.tsx:120](../../packages/ui/spa/components/ArrayAndRecordTools.tsx#L120)), which feeds +two mutating decisions — `refs.length > 0` is the ONLY thing that turns `DeleteRecordPopover` into +"Cannot delete", and `existingKeys` is the list of referring fields `ChangeRecordPopover` rewrites on +rename. A missed ref ⇒ delete looks safe (dangling ref left behind) / rename silently leaves the +referrer pointing at a key that no longer exists. Worse, the answer depends on which entries the user +happened to open this session, so it is nondeterministic. + +`traverseSchemaSource`'s accidental skip has one edge: an entry schema with a field literally named +`_type` or `patch_id` would index the marker's own value. + +### Scoping rule — direction decides (LOCKED 2026-07-30) + +The serialized schema names the target module for 2 of the 3 ref kinds: + +| Ref kind | Matcher | Names target module? | +| ---------- | ----------------------------------------------------------------- | --------------------------------------------------------------------- | +| `keyOf` | `schema.path === parentSourcePath` (`SerializedKeyOfSchema.path`) | **yes** | +| image/file | `schema.referencedModule === parent` | **yes** | +| `route` | `source === routeKey` (plain string compare) | **no** — `SerializedRouteSchema` only carries include/exclude regexes | + +Therefore: + +- **Incoming refs → keys only, ZERO loading.** Renaming/deleting key `K` of a jsonValues record `M`: + the referrers are `keyOf(M)` / `route` fields _elsewhere_, and `M`'s own key set is fully present in + the base source (markers preserve keys). `M`'s entry CONTENT is irrelevant to finding them. The one + entry we do need is the one being moved — already handled by the `ensureJsonEntry` await in + `ChangeRecordPopover`. One entry, not all. +- **Outgoing refs → the only case that needs content.** Loading is required only when a jsonValues + entry's own schema can contain a referrer to the thing being edited, e.g. + `s.record(s.object({ test: s.keyOf(otherModule) })).jsonValues()`. +- **The predicate** (schemas only — no sources, no fetching): for a query "refs to `M`", the set to + load = every jsonValues record whose **item schema transitively contains** (through + object/array/record/union) a `keyOf` with `path === M`, or an image/file with + `referencedModule === M`. In the overwhelmingly common case the set is EMPTY: no requests, no + progress UI, guard complete and correct immediately. Self-reference falls out for free. +- **Route refs are the one over-approximation**: since `s.route()` does not record which router it + points into, the predicate degrades to "does this jsonValues item schema contain ANY `route` field". + Still a large cut. Optional later narrowing: test the field's `options.include` regex against the + route key being renamed and skip fields that cannot match. +- **Search cannot be scoped** — it indexes all content by definition. It is the one unconditional + full-load consumer, which is why it (and only it) needs pagination + visible progress. + +### Third consumer: the record LIST view — visible rows only (added 2026-07-30) + +The list view already renders a per-entry preview for EVERY key, unvirtualized: +[RecordFields.tsx:162](../../packages/ui/spa/components/fields/RecordFields.tsx#L162) (default grid) and +[:191](../../packages/ui/spa/components/fields/RecordFields.tsx#L191) (`ListRecordRenderComponent`), both +``. For a jsonValues record that preview reads +the entry path, i.e. an un-loaded marker — so **today a jsonValues record list shows N broken/empty +previews**, and the naive fix (load what the preview needs) would load every entry on open, defeating +the whole feature. + +**Requirement (LOCKED 2026-07-30):** the list must be virtualized with `@tanstack/react-virtual` +(already a dependency — used in `FileGalleryListView` / `MediaPicker`, not yet in record lists) and +entry content loaded **only for the rows the virtualizer actually renders** (visible + overscan). + +- The virtualizer's rendered window IS the key window → **one batch request per window**, not one per + row. This is the strongest argument for the `keys` batch param. +- Debounce on scroll and drop stale windows: flinging through 10k rows must not enqueue 10k keys. + Requests already in flight for keys that scrolled out are harmless (they fill the cache) but must + not block newer windows. +- Rows whose content is not loaded yet render a skeleton/spinner, NOT an error — an un-loaded marker in + a read/render path is normal (see the watch-list note). +- **This changes V1.** "Open the module → zero `/json` requests" is no longer the invariant, because the + visible rows now load. The new invariant: requests are bounded by (visible + overscan) and by the + batch chunk size — never by the record's total key count. +- The list shares the SAME `jsonEntryContents` cache, so scrolling warms it for later ref scans. It must + NOT be mistaken for completeness: the refs guard still runs its own `ensureJsonEntries` for the + modules the predicate names. +- **`.render()` list layouts for jsonValues records** — now its own work item + step below (was a + deferred note). `RecordSchema.executeRender` returns `{}` when `isJsonValues` + ([record.ts:795](../../packages/core/src/schema/record.ts#L795)), so BOTH the per-entry renders and the + record-level `ListRecordRender` are dropped for these records (the early return precedes the + `renderInput` block). + +### Order of work (decided 2026-07-30) + +The list view comes early — it is the most user-visible breakage (a jsonValues record shows N broken +previews today) and it exercises the batch path under real scroll load before anything subtle depends +on it. Steps 1–2 are the only hard prerequisites; 4–6 are independent of each other after that. + +1. **Batch transport** — `ValOps.getJsonEntries` (hoist `initSources`/`fetchPatches`) + `/json` batch + mode + `ApiRoutes` zod, with server tests. Everything below depends on this. +2. **Engine primitives** — `requestJsonEntries(mfp, keys)` (window, fire-and-forget) + + `ensureJsonEntries(modules)` (whole-module, awaitable) + progress store, and fold + `markAllJsonEntriesStale` into the batch path. Extend the `jsonValues` describe in + `ValSyncEngine.test.ts`. +3. **List view** — virtualize `RecordFields` (both branches) with `@tanstack/react-virtual`, load the + rendered window via `requestJsonEntries`, **skeletons for un-loaded rows on all three preview paths** + (a marker must never reach a preview component). → **V16**. First visible win; also the first real + load-test of steps 1–2. +4. **Predicate** — `jsonValuesLoadRequirements` + its unit tests (pure; could be done any time, but it + only pays off once the hooks below use it). +5. **Ref hooks + popover gating** — hooks return a status, destructive popovers gate on `success`. + → **V10–V13, V17**. This is where the data-integrity hole actually closes. +6. **Search** — first-query trigger, debounced re-index, marker skip in `traverseSchemaSource`, delete + the dead `createSearchIndex.ts`. → **V14**. +7. **`.render()` list layouts (windowed)** — per-key `render` on the batch `/json` response + partial + `items` merged into the client renders map. → **V18**. **Gated on the renders-are-null decision** in + its work item; until that is answered, step 3's skeleton + `` fallback IS the list preview. + Deliberately last: it is the only step that depends on a Studio-wide question rather than on + jsonValues. +8. **Verify + gate** — the full manual walkthrough (V1–V18, noting V1 is superseded; V1–V9 have still + never been run) then the Phase 5 CI gate. + +The caching decision (last item below) is deliberately NOT in this order — it is a question for Fredrik, +not a step. + +### Work items + +- [ ] **Batch `/json` (transport)** — `ApiRoutes.ts`: GET `/json` grows a batch mode. `keys` (JSON + array param, client-driven paging, cap the batch server-side) **and** `offset`/`limit` for + all-mode. Response returns an array of `{key, content}` (plus the resolved `offset`/`limit`/ + `total` for all-mode). Keep the existing single `key` form working — the entry-open path must not + get slower. `apply_patches` semantics unchanged (Studio passes `false`). +- [ ] **Batch `ValOps.getJsonEntries(mfp, {keys | offset+limit}, {applyPatches})`** — + `getJsonEntry` currently calls `initSources()` AND `fetchPatches()` per entry + ([ValOps.ts:272](../../packages/server/src/ValOps.ts#L272)); a batch MUST hoist both out of the + loop or "load 500 entries" becomes 500 patch fetches. Keep `getJsonEntry` as a thin wrapper over + the batch so there is one code path. +- [ ] **`jsonValuesLoadRequirements(schemas, query)`** — new pure module in `packages/ui/spa` + implementing the predicate above. `query` is one of `{kind:"keyOf", module}` / + `{kind:"file", module}` / `{kind:"route"}`; returns the `ModuleFilePath[]` whose entries must be + loaded. Own unit test file: empty result for the incoming-ref case, non-empty for nested + `keyOf`/image/file, `route` over-approximation, transitivity through object/array/record/union. +- [ ] **`ValSyncEngine.ensureJsonEntries(moduleFilePaths)`** — idempotent batch loader: for each + module, diff the marker key set against `jsonEntryContents` + `staleJsonEntries`, request the + missing keys in chunks (start at 50) through the batch endpoint, fill the SAME + `jsonEntryContents` cache, then the existing invalidate + emit. Returns a promise that resolves + when the requested modules are fully loaded. Never fires on Studio boot. +- [ ] **Progress store** — `{loaded, total, status}` (not a boolean) exposed as a sync-external-store + snapshot (`subscribe("json-entries-progress")` + `getJsonEntriesProgressSnapshot()`), so any + component can render "Checking references… 340/5000". `total` needs no server help: the client + already has every entry key from the marker record. +- [ ] **`ValSyncEngine.requestJsonEntries(mfp, keys)`** — the window-based sibling of + `ensureJsonEntries`: fire-and-forget, takes an explicit key list, skips cached/in-flight/errored + keys, batches the rest in one request. This is what the virtualized list calls per rendered + window; `ensureJsonEntries(modules)` (awaitable, whole-module) stays for the refs guard and search. + Both share the chunking + cache + emit path. +- [ ] **Virtualize the record list + load visible rows only** — `RecordFields.tsx`: wrap both list + branches (default grid at :162 and `ListRecordRenderComponent` at :191) in + `useVirtualizer` from `@tanstack/react-virtual`, and for a jsonValues record call + `requestJsonEntries(mfp, visibleKeys)` from the rendered window (debounced; drop stale windows). + Un-loaded rows render a skeleton, not an error. Non-jsonValues records get virtualization too + (same code path, no loading) — a 10k-key ordinary record renders 10k previews today. +- [ ] **Skeletons for un-loaded jsonValues rows (all three preview paths)** — a marker must never reach + a preview component. `PreviewWithRender` → `useRefPreview` miss → `` → + `ObjectPreview`/etc. reading a marker is exactly today's broken state. So: when the parent schema is + a jsonValues record and the entry's content is not in `jsonEntryContents`, render a skeleton + (a) instead of the `` fallback, (b) instead of a `ListPreviewItem` when the key is missing + from a partial `items` array, and (c) for the `errored` case a small retry affordance rather than a + dead row. Skeleton must be the same height as a loaded row or the virtualizer's measurements jump. +- [ ] **`.render()` list layouts for jsonValues records (windowed)** — drop the `isJsonValues` early + return in `RecordSchema.executeRender` and instead compute renders for the keys whose content the + caller has. Shape: - The user's `select({key, val})` lives in the schema INSTANCE, so only the SERVER can run it (the + client has serialized schemas only). Batch `/json` therefore returns `render` per key next to + `content` — one more field on the same response, no extra round trip. - The record-level `ListRecordRender.items` is `[key, {title, subtitle?, image?}][]` + ([render.ts:9](../../packages/core/src/render.ts#L9)) — an ALL-ROWS payload, so it can never be + produced eagerly for a 10k jsonValues record. It becomes **partial**: the client merges the render + values it has received into the module's renders map, and `items` holds only loaded keys. - This works with zero changes to the consumer: `resolveRefPreview` looks the row up by key + (`items.find(([itemKey]) => itemKey === key)`, + [useRefPreview.ts:82](../../packages/ui/spa/components/useRefPreview.ts#L82)) and returns + `undefined` on a miss — which the previous work item turns into a skeleton. - **BLOCKER (Studio-wide, not jsonValues-specific): renders are null in the Studio today.** + `/sources/~` only computes them when `apply_patches` is true + ([ValServer.ts:1479](../../packages/server/src/ValServer.ts#L1479)) and the Studio always sends + `false`, so `ValSyncEngine` stores `valModule.render || null` and every list already falls back to + ``. `setRenders` is called from stories only. So `.render()` list layouts are dead for + ALL schema types right now, and fixing jsonValues alone changes nothing visible. - **DECISION NEEDED (ask Fredrik)**: renders need the user's `select` (server-only) AND the PATCHED + source (client-only, because the Studio owns in-flight patches) — that pincer is why they are off. + Options: (i) a dedicated render request with `apply_patches:true`, accepting that a render lags an + unsaved edit (a row would show the committed title while the user is editing it); (ii) make the + title/subtitle/image selection serializable so the client can evaluate it; (iii) leave renders off + and treat `` as the only list preview. Same class of limitation as the `useValKey` draft + path already tracked in Phase 4. Pick one before building this item. +- [ ] **Ref hooks stop lying** — `useEagerRouteReferences` / `useKeysOf` / `useReferencedFiles` return + a status (`loading` + progress → `success` with COMPLETE refs → `error`) instead of a bare array. + Each hook calls `jsonValuesLoadRequirements` first; empty ⇒ synchronously `success` (today's + behaviour, no request). Non-empty ⇒ `ensureJsonEntries` and report progress. +- [ ] **Popovers gate on completeness** — `DeleteRecordPopover` / `ChangeRecordPopover` render progress + and refuse to act until `success`; on `error` they stay blocked with a retry. Never + "no refs found, go ahead". This is the actual fix for the defect. +- [ ] **Search** — trigger `ensureJsonEntries(all jsonValues modules)` **only on user intent**: search + is lazy, so nothing loads before it is requested. Trigger on the first non-empty query, NOT on + `SearchField` mount / dialog open (radix mounts the content when the dialog opens, so a + mount-effect trigger would load on open). **Debounce the re-index** — + [Search.tsx:121-138](../../packages/ui/spa/components/Search.tsx#L121-L138) rebuilds the whole index + on every `sources` change and would otherwise rebuild once per batch. +- [ ] **Search shows partial results + a percentage while loading** — results appear IMMEDIATELY from + whatever is already indexed (never a blocked/empty dropdown waiting for a full load), and the + dropdown carries a loading indicator with a **percentage** — `Math.round(loaded / total * 100)` off + the progress store, e.g. "Searching… 42% indexed". The list re-renders as each batch lands and the + indicator disappears at 100%. Two details that matter: the percentage must be over the whole + requested set (all jsonValues modules), not per module, or it resets visibly on every module + boundary; and results arriving late must not reorder/jump what the user is already looking at more + than the new matches require. +- [ ] **Fix the live search traversal** — add the marker skip to `traverseSchemaSource` (interim + correctness + kills the `_type`/`patch_id` edge). +- [ ] **Delete the dead `createSearchIndex.ts` + `search.test.ts`** and the unused barrel export in + `search/index.ts`. The marker skip added there was on unreachable code. +- [ ] **Fold `markAllJsonEntriesStale` into the batch path** — + [ValSyncEngine.ts:1063](../../packages/ui/spa/ValSyncEngine.ts#L1063) currently re-requests every + loaded entry one-by-one, so with hundreds cached a publish is a request storm today. The progress + store must cover this post-publish refresh too: anything transitively derived from stale entries + (entry detail view, refs guard, search index) goes back to `loading` with progress rather than + briefly rendering stale or content-free values. In particular a refs query must NOT answer from + stale content — it re-enters `loading`. +- [ ] **Verify** (Studio running): + - **V10** rename/delete a key in a jsonValues router while another ORDINARY module holds a + `keyOf`/`route` ref to it → refs found, delete blocked, rename rewrites the referrer, and **zero** + `/json` requests (incoming-ref case). + - **V11** same, but the referrer lives inside a jsonValues entry (`keyOf` in the item schema) → + progress shown, refs found after load, delete blocked, rename rewrites it. + - **V12** delete a key in an ordinary record while no jsonValues item schema references it → zero + `/json` requests, guard instant. + - **V13** a batch load that fails mid-way → guard shows an error + retry, destructive action stays + blocked (never silently "no refs"). + - **V14** open search, type nothing → zero `/json`. Type a query → matches from already-indexed + content appear IMMEDIATELY (not after the load), the dropdown shows a percentage that climbs to + 100%, entry content becomes findable as batches land, and the index is NOT rebuilt once per batch. + - **V15** publish with many entries cached → one batched refresh (not N requests), progress visible, + no stale answers from the refs guard in between. + - **V16** open a jsonValues record with many entries → only the visible window (+ overscan) is + requested, in ONE batch, and those rows show real previews; rows below the fold show skeletons and + have NOT been requested. Scroll slowly → one batch per window. Fling to the bottom → the number of + requests is bounded by windows actually rendered, never by total keys. + - **V17** scroll a jsonValues list to the bottom, then rename a key whose only referrer is an entry + that was scrolled past → the refs guard still reports COMPLETE (cache warm ⇒ instant) and the + rename rewrites the referrer. + - **V18** (step 7 only) a jsonValues record WITH `.render({layout:"list"})` → visible rows show the + user's title/subtitle/image; rows below the fold are skeletons of the same height (no measurement + jump, no marker reaching a preview component); scrolling fills them in. Then edit a visible row's + title without publishing → confirm what the row shows, and that it matches whichever option was + chosen in the renders decision (it will show COMMITTED content under option (i)). +- [ ] **DECISION NEEDED — ask Fredrik: efficient browser caching for `/json`.** Deferred from the + Phase 6 design round (2026-07-30). Fredrik's idea: in **http/prod mode there is a stable commit**, + so put it in the request and make the response immutably cacheable by the browser. Open parts: + (1) FS/dev mode has no commit — what is the key there, or do we simply not cache? + (2) `ValClient` sets its own headers, exposes no response headers, and zod-validates a JSON body, + so `ETag`/`If-None-Match`/304 is NOT reachable without extending that layer + ([ValClient.ts:88](../../packages/shared/src/internal/ValClient.ts#L88)). + (3) `apply_patches:false` (what the Studio sends) is a pure function of committed files, so it is + the cacheable case; `apply_patches:true` is mutable. + (4) Alternative if headers stay off-limits: in-payload conditional loading (client sends known + version tokens, server replies "unchanged") — needs a per-entry version token, and + `jsonValuesSha.ts` was deliberately deleted (locked decision #3), so there is none today. + Until this is decided, "caching" means only: the client never refetches what is in + `jsonEntryContents`, and the server hoists `initSources`/`fetchPatches` per batch. + +**Longer-term alternative (not now):** a server-side reference-scan endpoint ("which source paths hold +`keyOf`/`route` value X"), which the server can answer from the `*.val.json` files on disk with no +client download at all. Phase 6 keeps the scan client-side; the schema predicate is what makes that +affordable. Revisit if the outgoing-ref case (V11) turns out to be common in real projects. + --- ## sha design — DROPPED (2026-07-02) @@ -342,9 +625,39 @@ unconditionally (accepted "validation takes more time" tradeoff). draft/endpoint path later). So committed validation is correct. - `vm` loader dynamic `import()` + `.json` resolution. - resolvePath/selector must never throw on a not-yet-loaded entry. +- **Skipping an un-loaded marker is NOT automatically safe.** It is fine for a read/render path (the + content is simply not there yet) but WRONG for any scan whose answer gates a mutation — see Phase 6. + When adding a new traversal, decide explicitly which of the two it is. +- Browser caching of `/json` — deferred; the last Phase 6 item holds the open questions. +- **Does `jsonEntryContents` need eviction?** With visible-row loading, scrolling a 10k-entry list (or + running search once) eventually caches every entry's content in the client. Nothing evicts today. If + this bites, an LRU keyed by `mfp\0key` is the obvious answer — but eviction must never silently + downgrade a refs guard that reported COMPLETE, so the guard needs to re-check (or pin) the keys it + depended on. Decide only when we see real memory numbers. +- **Where do per-entry renders come from for jsonValues?** `executeRender` returns `{}` for these + records and Studio renders are null across the board today — see the Phase 6 list-view subsection. ## Changelog +- **Session 6 (2026-07-30)**: planning only — NO code. Reviewing PR #453 surfaced the + reference-integrity defect (the marker skips in `getRouteReferences`/`traverseSchemas` silently make + the delete gate and rename fixup answer "no references", nondeterministically) and that the live + search path never handled markers at all while the file that does (`createSearchIndex.ts`) is dead. + Designed Phase 6: schema-derived scoping rule (direction decides — incoming refs need keys only, + so the common case loads NOTHING), batch `/json` (`keys` + `offset`/`limit`), a progress store, + status-returning ref hooks that gate the destructive popovers, lazy search-triggered full load, and + batched publish invalidation. Browser caching deferred to a decision item at the end of Phase 6. + Then added the **visible-rows-only list requirement** (Fredrik): the list view already renders a + per-entry `` for every key with no virtualization, so jsonValues records show N broken + previews AND the naive fix would load everything on open. Virtualize with `@tanstack/react-virtual` + and request only the rendered window (`requestJsonEntries(mfp, keys)`). Supersedes V1; added V16/V17, + plus watch-list entries for cache eviction and for `executeRender` returning `{}` on jsonValues. + Then promoted the deferred render note to real work: an explicit **Order of work** (8 steps, list view + early because it is the visible breakage), **skeletons on all three preview paths** (a marker must never + reach a preview component), **windowed `.render()` list layouts** (per-key `render` on the batch + response + partial `items`; gated on a DECISION about renders being null Studio-wide — `/sources/~` only + computes them when `apply_patches` is true, and the Studio always sends false), and **search showing + partial results with a percentage** while the index fills. Added V18. - **Session 5 (2026-07-28)**: enabled/draft runtime path (server + RSC) and edit tags. - Merged `main` (which fixed an unrelated blocker: the publish gate read the RAW `errors.validationErrors` instead of the surfaced snapshot, so every `s.images()`/`s.files()` From acd40aa11cdb79d7310b671c245393b485ee1880 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Fri, 31 Jul 2026 12:17:12 +0200 Subject: [PATCH 32/35] jsonValues: batch /json (Phase 6 step 1) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Loading many `.jsonValues()` entries was one request per entry, and each request re-ran initSources() AND fetchPatches() — so "load 500 entries" meant 500 patch fetches. This adds a batch shape and makes the batch the single implementation. - `/json` GET now takes exactly one of: `key` (unchanged single-entry shape), `keys` (repeated query param), or `offset`+`limit`. Batch responses are `{path, entries, missing, errors, total, offset?, limit?}`. - `keys` is a repeated param rather than a JSON array: ValClient already serializes array query values and the router already hands zod a `string[]`, so no plumbing was needed and there is nothing to fail to parse. - `ValOps.getJsonEntries` hoists initSources/fetchPatches out of the per-entry loop and resolves thunks with Promise.all. `getJsonEntry` is now a one-key wrapper over it, so there is one code path. - Per-entry problems stay per-entry: unknown key -> `missing`, corrupt *.val.json -> `errors`. Only a missing/non-record MODULE fails the request, so one bad entry cannot fail a whole batch. - `offset`/`limit` REQUIRES apply_patches=false and errors otherwise: enumerating from the base source would silently omit draft-added keys, which is the bug class this phase exists to remove. - JSON_ENTRIES_BATCH_MAX (100) is exported so client and server agree; over-cap requests fail zod validation. ValidQueryParamTypes gained `number`. Tests: +9 ValOpsFS.jsonValues (order, missing, per-key errors, patched entries, paging, the apply_patches guard, one-fetchPatches-per-batch), +5 ValRouter shape validation. The test client mock serves all three shapes. Tracker: Phase 6 step 1 marked done; Phase 7 (client-side schema instances) added with the custom-validate design (worker keeps structural validation and reports flagged paths; the main thread executes them time-sliced). Co-Authored-By: Claude Opus 5 --- docs/plans/jsonValues.md | 236 ++++++++++++++---- packages/next/src/rsc/initValRsc.ts | 16 +- packages/server/src/ValOps.ts | 218 ++++++++++++---- .../server/src/ValOpsFS.jsonValues.test.ts | 148 +++++++++++ packages/server/src/ValRouter.test.ts | 49 ++++ packages/server/src/ValServer.ts | 59 ++++- packages/shared/src/internal/ApiRoutes.ts | 82 +++++- packages/ui/spa/ValSyncEngine.test.ts | 63 ++++- packages/ui/spa/ValSyncEngine.ts | 11 +- 9 files changed, 761 insertions(+), 121 deletions(-) diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md index 1c0a053c1..195a68683 100644 --- a/docs/plans/jsonValues.md +++ b/docs/plans/jsonValues.md @@ -10,7 +10,15 @@ ## Current state / resume here -> **Reference-integrity defect found (2026-07-30) — Phase 6 is the next milestone, NOT started.** +> **Phase 6 step 1 DONE (2026-07-31): batch `/json`.** `ValOps.getJsonEntries` is the single +> implementation (`getJsonEntry` is a one-key wrapper); `initSources`/`fetchPatches` are hoisted out of +> the per-entry loop; per-entry failures are per-entry (`missing`/`errors`) so one corrupt `*.val.json` +> cannot fail a batch; `offset`/`limit` refuses `apply_patches` rather than silently omitting +> draft-added keys. `keys` is a repeated query param (no plumbing needed — the client already serializes +> array query values). +14 tests, `pnpm test` 1118 green, `-r typecheck` clean, lint clean. +> **Next: step 2 — engine primitives (`requestJsonEntries` / `ensureJsonEntries` + progress store).** + +> **Reference-integrity defect found (2026-07-30) — Phase 6 is the next milestone, step 1 done.** > The Studio's three global scans (route refs, keyOf refs, referenced files) and search silently skip > un-loaded `{_type:"json"}` markers, so the delete gate and the rename fixup can both answer "no > references" for a ref that lives inside an entry the user happens not to have opened. That is a @@ -22,6 +30,11 @@ > renders a per-entry preview for every key, unvirtualized, so a jsonValues record currently shows N > broken previews. Fix = virtualize with `@tanstack/react-virtual` + load only the rendered window. > This **supersedes V1** ("zero `/json` on open"). +> **Phase 7 (2026-07-31)** is the follow-on: the SPA already holds the user's REAL schema instances +> (`window.__VAL_MODULES__` → `extractValModules().schemas`) and throws them away in `setValModules`, +> re-deriving a lobotomised schema with `deserializeSchema` that has no render `select`, no custom +> validators and no router. Using the instances fixes renders (all schema types, draft-accurate) and +> finally lets custom validators run client-side. Phase 6 step 7 moved there. > **Draft runtime path DONE (2026-07-28):** the last Phase 4 box is closed on the server + RSC side. > `/json` takes `apply_patches` (default **true**, mirroring `/sources/~`) and replays pending @@ -449,11 +462,9 @@ on it. Steps 1–2 are the only hard prerequisites; 4–6 are independent of eac → **V10–V13, V17**. This is where the data-integrity hole actually closes. 6. **Search** — first-query trigger, debounced re-index, marker skip in `traverseSchemaSource`, delete the dead `createSearchIndex.ts`. → **V14**. -7. **`.render()` list layouts (windowed)** — per-key `render` on the batch `/json` response + partial - `items` merged into the client renders map. → **V18**. **Gated on the renders-are-null decision** in - its work item; until that is answered, step 3's skeleton + `` fallback IS the list preview. - Deliberately last: it is the only step that depends on a Studio-wide question rather than on - jsonValues. +7. **`.render()` list layouts (windowed)** → **Phase 7 stage 1** (client-side schema instances), verified + by **V18**. Not gated on anything any more; it is simply a different phase's work. Until it lands, + step 3's skeleton + `` fallback IS the list preview. 8. **Verify + gate** — the full manual walkthrough (V1–V18, noting V1 is superseded; V1–V9 have still never been run) then the Phase 5 CI gate. @@ -462,16 +473,23 @@ not a step. ### Work items -- [ ] **Batch `/json` (transport)** — `ApiRoutes.ts`: GET `/json` grows a batch mode. `keys` (JSON - array param, client-driven paging, cap the batch server-side) **and** `offset`/`limit` for - all-mode. Response returns an array of `{key, content}` (plus the resolved `offset`/`limit`/ - `total` for all-mode). Keep the existing single `key` form working — the entry-open path must not - get slower. `apply_patches` semantics unchanged (Studio passes `false`). -- [ ] **Batch `ValOps.getJsonEntries(mfp, {keys | offset+limit}, {applyPatches})`** — - `getJsonEntry` currently calls `initSources()` AND `fetchPatches()` per entry - ([ValOps.ts:272](../../packages/server/src/ValOps.ts#L272)); a batch MUST hoist both out of the - loop or "load 500 entries" becomes 500 patch fetches. Keep `getJsonEntry` as a thin wrapper over - the batch so there is one code path. +- [x] **Batch `/json` (transport)** — DONE (2026-07-31). GET `/json` now takes exactly one of `key` + (unchanged single-entry shape), `keys` (**repeated** query param, not a JSON array — `ValClient` + already serialises array query values, and the router's `groupQueryParams` already hands zod a + `string[]`, so no plumbing was needed), or `offset`+`limit`. Batch responses are + `{path, entries, missing, errors, total, offset?, limit?}`. `JSON_ENTRIES_BATCH_MAX = 100` is + exported from `ApiRoutes` so client and server agree; over-cap requests fail zod validation. + Shape violations are 400s, checked in `ValRouter.test.ts` (+5). `ValidQueryParamTypes` gained + `number` (the client stringifies query values anyway). +- [x] **Batch `ValOps.getJsonEntries(mfp, {keys | offset+limit}, {applyPatches})`** — DONE (2026-07-31). + `initSources()` and `fetchPatches()` are hoisted out of the per-entry loop (pinned by a test that + asserts `fetchPatches` is called exactly once for a 2-key batch); thunks resolve via `Promise.all`. + `getJsonEntry` is now a one-key wrapper over it, so there is a single code path. Per-entry problems + stay per-entry: an unknown key lands in `missing`, a corrupt `*.val.json` in `errors`, and only a + missing/non-record MODULE is a whole-request `not-found`. `offset`/`limit` REQUIRES + `applyPatches:false` and errors otherwise — enumerating from the base source would silently omit + draft-added keys, which is the exact bug class this phase exists to kill. Tests: +9 in + `ValOpsFS.jsonValues.test.ts`. - [ ] **`jsonValuesLoadRequirements(schemas, query)`** — new pure module in `packages/ui/spa` implementing the predicate above. `query` is one of `{kind:"keyOf", module}` / `{kind:"file", module}` / `{kind:"route"}`; returns the `ModuleFilePath[]` whose entries must be @@ -504,28 +522,17 @@ not a step. (a) instead of the `` fallback, (b) instead of a `ListPreviewItem` when the key is missing from a partial `items` array, and (c) for the `errored` case a small retry affordance rather than a dead row. Skeleton must be the same height as a loaded row or the virtualizer's measurements jump. -- [ ] **`.render()` list layouts for jsonValues records (windowed)** — drop the `isJsonValues` early - return in `RecordSchema.executeRender` and instead compute renders for the keys whose content the - caller has. Shape: - The user's `select({key, val})` lives in the schema INSTANCE, so only the SERVER can run it (the - client has serialized schemas only). Batch `/json` therefore returns `render` per key next to - `content` — one more field on the same response, no extra round trip. - The record-level `ListRecordRender.items` is `[key, {title, subtitle?, image?}][]` - ([render.ts:9](../../packages/core/src/render.ts#L9)) — an ALL-ROWS payload, so it can never be - produced eagerly for a 10k jsonValues record. It becomes **partial**: the client merges the render - values it has received into the module's renders map, and `items` holds only loaded keys. - This works with zero changes to the consumer: `resolveRefPreview` looks the row up by key - (`items.find(([itemKey]) => itemKey === key)`, - [useRefPreview.ts:82](../../packages/ui/spa/components/useRefPreview.ts#L82)) and returns - `undefined` on a miss — which the previous work item turns into a skeleton. - **BLOCKER (Studio-wide, not jsonValues-specific): renders are null in the Studio today.** - `/sources/~` only computes them when `apply_patches` is true - ([ValServer.ts:1479](../../packages/server/src/ValServer.ts#L1479)) and the Studio always sends - `false`, so `ValSyncEngine` stores `valModule.render || null` and every list already falls back to - ``. `setRenders` is called from stories only. So `.render()` list layouts are dead for - ALL schema types right now, and fixing jsonValues alone changes nothing visible. - **DECISION NEEDED (ask Fredrik)**: renders need the user's `select` (server-only) AND the PATCHED - source (client-only, because the Studio owns in-flight patches) — that pincer is why they are off. - Options: (i) a dedicated render request with `apply_patches:true`, accepting that a render lags an - unsaved edit (a row would show the committed title while the user is editing it); (ii) make the - title/subtitle/image selection serializable so the client can evaluate it; (iii) leave renders off - and treat `` as the only list preview. Same class of limitation as the `useValKey` draft - path already tracked in Phase 4. Pick one before building this item. +- [ ] **`.render()` list layouts for jsonValues records (windowed)** — **moved to Phase 7 stage 1**; it is + a client-side-instance change, not a transport one. Summary: replace the `isJsonValues` early return + in `RecordSchema.executeRender` with a per-key `isJson(itemSrc) → continue`, then call + `executeRender(mfp, patchedSource)` on the CLIENT instance. Because `executeRender` iterates + `Object.entries(src)` and un-loaded entries are still markers, the resulting + `ListRecordRender.items` ([render.ts:9](../../packages/core/src/render.ts#L9)) is naturally + **partial** — exactly the loaded keys — and `resolveRefPreview`'s keyed lookup + ([useRefPreview.ts:82](../../packages/ui/spa/components/useRefPreview.ts#L82)) misses on the rest, + which the skeleton item above turns into a skeleton. No `render` field on `/json`, no pagination in + the render path. (Earlier draft of this item assumed only the server could run `select`; wrong — see + Phase 7.) - [ ] **Ref hooks stop lying** — `useEagerRouteReferences` / `useKeysOf` / `useReferencedFiles` return a status (`loading` + progress → `success` with COMPLETE refs → `error`) instead of a bare array. Each hook calls `jsonValuesLoadRequirements` first; empty ⇒ synchronously `success` (today's @@ -580,11 +587,11 @@ not a step. - **V17** scroll a jsonValues list to the bottom, then rename a key whose only referrer is an entry that was scrolled past → the refs guard still reports COMPLETE (cache warm ⇒ instant) and the rename rewrites the referrer. - - **V18** (step 7 only) a jsonValues record WITH `.render({layout:"list"})` → visible rows show the + - **V18** (Phase 7 stage 1) a jsonValues record WITH `.render({layout:"list"})` → visible rows show the user's title/subtitle/image; rows below the fold are skeletons of the same height (no measurement jump, no marker reaching a preview component); scrolling fills them in. Then edit a visible row's - title without publishing → confirm what the row shows, and that it matches whichever option was - chosen in the renders decision (it will show COMMITTED content under option (i)). + title WITHOUT publishing → the row updates as you type (the render is computed from the patched + source on the client). - [ ] **DECISION NEEDED — ask Fredrik: efficient browser caching for `/json`.** Deferred from the Phase 6 design round (2026-07-30). Fredrik's idea: in **http/prod mode there is a stable commit**, so put it in the request and make the response immutably cacheable by the browser. Open parts: @@ -605,6 +612,133 @@ not a step. client download at all. Phase 6 keeps the scan client-side; the schema predicate is what makes that affordable. Revisit if the outgoing-ref case (V11) turns out to be common in real projects. +## Phase 7 — Use the CLIENT-SIDE schema instances (added 2026-07-31) + +### The realisation + +The SPA already has the user's REAL schema instances and has been throwing them away. +`` registers the registry on `window.__VAL_MODULES__` +([ValModulesClient.tsx:27](../../packages/next/src/ValModulesClient.tsx#L27)), `useValModules` reads it +([hooks/useValModules.ts](../../packages/ui/spa/hooks/useValModules.ts)), `ValProvider` pushes it into +`syncEngine.setValModules`, and `extractValModules` returns BOTH +`schemas: Record>` (instances) and `serializedSchemas` +([extractValModules.ts:113](../../packages/core/src/extractValModules.ts#L113)) — but `setValModules` +keeps only the serialized ones. Everything downstream then re-derives a lobotomised schema with +`deserializeSchema`, which drops (a) the render `select`, (b) `customValidateFunctions` (every case passes +`[]`), and (c) the router. Cross-bundle identity is already handled: the identity symbols use +`Symbol.for` so the SPA's copy of core and the host bundle's copy agree +([selector/index.ts:57](../../packages/core/src/selector/index.ts#L57)), and protected methods are called +by bracket access (`schema["executeSerialize"]()`) precisely because `instanceof` is unreliable across +copies. So: keep the instances and use them. + +Availability caveat: instances exist only when the host app renders ``. Without it +`window.__VAL_MODULES__` is undefined, `setValModules(null)` runs, and everything must fall back to +today's serialized behaviour. `localModulesStatus` (`absent`/`loading`/`error`/`loaded`) is the flag. + +### Stage 1 — renders from instances (absorbs Phase 6 step 7) → **V18** + +- [ ] Retain `extracted.schemas` on the engine as `localSchemaInstances`. +- [ ] Core: replace the `isJsonValues` early return in `RecordSchema.executeRender` + ([record.ts:795](../../packages/core/src/schema/record.ts#L795)) with a per-key + `isJson(itemSrc) → continue`, so a partially-substituted record renders its loaded keys. +- [ ] Engine `computeRender(mfp)` = `instance["executeRender"](mfp, patchedSource)` → the existing + `renders` map + `invalidateRenders`; memoized per (module, sourceSha); recomputed on source + invalidation. Wrap each key's `select` in try/catch so one throwing user function is one error row, + not a dead Studio. +- [ ] Windowing is free: un-loaded entries are markers, markers are skipped, `items` comes out partial, + `resolveRefPreview` misses → skeleton (Phase 6). +- What this buys beyond jsonValues: renders work again for ALL schema types (they are dead Studio-wide + today) and they are computed from the PATCHED source, so a row's title updates as the user types — + something the server path could never do. + +### Stage 2 — custom validators, worker kept (LOCKED 2026-07-31) + +The worker stays. `executeCustomValidateFunctions` only ever APPENDS errors +([schema/index.ts:91](../../packages/core/src/schema/index.ts#L91)) and `CustomValidateFunction` is +`(src, ctx:{path}) => false | string` — pure, per-node — so custom errors can be merged AFTER the +worker's structural result. No synchronous worker→main round trip is needed. + +- [ ] **Gate**: one boolean per module, memoized by `schemaSha` — does the serialized tree contain any + `customValidate: true`? If not (the common case) nothing changes: no extra message, no main-thread + work. Only modules that actually declare a custom validator pay anything. +- [ ] **Worker reports WHERE**: while it has the module, the worker walks (serialized schema, source) and + collects the paths of flagged nodes it actually visited (so unions/optional branches only report + paths that exist). Response grows `customValidatePaths`. This walk must be separate from + `executeValidate` — the deserialized schema has no validators, so it cannot report that it skipped + any. Walking stays off the main thread; the main thread only EXECUTES. +- [ ] **Main thread executes, time-sliced**: per path, `Internal.resolvePath(modulePath, source, instance)` + (already generic over `Schema | SerializedSchema`, + [module.ts:279](../../packages/core/src/module.ts#L279)) → node instance + src, then run that node's + validators. Slice on a ~5ms budget yielding via `scheduler.postTask({priority:"background"})` → + `requestIdleCallback` → `MessageChannel`. Abort when the module's `latestRequestId` changes + (`ValidationWorkerClient` already tracks it) or the sourceSha moves. + A slow user validator still blocks — accepted: devs who write slow validators see their own slowness. +- [ ] **Error store must MERGE, not replace** — structural errors publish immediately (fast feedback), + custom errors merge per chunk. A wholesale replace would erase the first publish. +- [ ] **API gap**: running one node's validators needs `customValidateFunctions`, which is + `private readonly` on EACH SUBCLASS, so no base-class helper can reach it. Chosen fix: add a one-line + `protected executeCustomValidateAt(path, src)` to each schema class (~15 files, mechanical, matches + the `schema["executeX"]()` convention). Rejected: bracket-accessing a private field from the SPA; + hoisting the field to the base class (cleanest end state, biggest diff — revisit in stage 3). + +**Triggers (LOCKED 2026-07-31)** — custom validation is NOT part of the load path: + +- [ ] **On update only**: `addPatch` → `requestModuleValidation(mfp)` runs structural (as today) plus + custom for THAT module. Note `requestAllModuleValidation()` currently fires from `setValModules` + ([ValSyncEngine.ts:2631](../../packages/ui/spa/ValSyncEngine.ts#L2631)), i.e. on boot and every HMR — + structural may keep doing that, custom must not. +- [ ] **Pre-publish**: validate every module touched by patches (the engine already knows the patch set + per module), custom included, with every needs-keys round resolved before publish is allowed. +- [ ] **"Validate absolutely everything" switch**: one call — + `validateAll({ custom: true, loadAllJsonEntries: true })` — walks every module, resolves every + needs-keys round, reports progress through the Phase 6 progress store. For dev/CI/debugging. + +**needs-keys protocol for `.jsonValues()` (LOCKED 2026-07-31)**: a custom validator cannot run against +opaque markers, so the call returns what it needs instead of guessing: + +``` +runCustomValidation(path) → + | { status: "done", errors } + | { status: "needs-keys", moduleFilePath, keys: string[] } +``` + +- [ ] The keys come from the record's MARKER key set in the module source — always present, no loading + required to compute them. A validator on the RECORD needs every key in its subtree; a validator on + the ITEM schema needs only that entry's key. +- [ ] The caller resolves them with Phase 6's `ensureJsonEntries` / `requestJsonEntries(mfp, keys)` and + retries. Guard the loop: if a retry returns the same `needs-keys` set, report an error instead of + spinning. Validation is thereby the FOURTH consumer of the batch loader (list view, refs guard, + search, validation). +- [ ] **Cost to document for users**: a custom validator on a `.jsonValues()` RECORD forces a full load of + that record whenever it runs (on update of that module, or pre-publish). Prefer putting custom + validators on the ITEM schema, which needs one key. This is inherent, not a bug: a record-level + validator is by definition a statement about all entries. +- [ ] **The server stays the authority for publish.** Client-side custom validation of a jsonValues module + is only ever as complete as what is loaded; `validateJsonValuesEntries` on the server validates every + entry. The client pass is feedback, not proof. +- [ ] **Expect newly-surfaced errors.** Client validation cannot run custom validators AT ALL today, so + existing projects may light up with errors that were previously invisible — and the publish gate + reads validation errors ([DraftChanges.tsx:143](../../packages/ui/spa/components/DraftChanges.tsx#L143)), + so some will block publish. Correct behaviour that will read as a regression; worth a release note. + +### Stage 3 — instances become the primary schema source + +- [ ] Serialized schemas stay for what genuinely must be JSON: `schemaSha` (the change-detection key + against the server), patch-op classification, `/sources/~` comparison. Anything needing BEHAVIOUR + (render, validate, `emptyOf`) reads instances. Honest framing: serialize never fully goes away; + `deserialize` **on the client** does. +- [ ] Ends with `deserializeSchema` removed from the SPA (the worker keeps it only if stage 2's structural + pass still runs there — which it does, so this is "removed from the main thread"). +- [ ] Consider hoisting `customValidateFunctions` (and the router) to the `Schema` base here, which + retires the stage-2 per-class helper. + +### Stage 4 — delete the server render path for the Studio + +- [ ] `getRenders` has exactly ONE caller — `/sources/~` + ([ValServer.ts:1480](../../packages/server/src/ValServer.ts#L1480)) — and only when + `apply_patches` is true, which the Studio never sends. Once stage 1 lands that branch is dead for the + Studio: remove it, and the `render` field on the `/sources/~` response if no other consumer wants it. + --- ## sha design — DROPPED (2026-07-02) @@ -634,8 +768,10 @@ unconditionally (accepted "validation takes more time" tradeoff). this bites, an LRU keyed by `mfp\0key` is the obvious answer — but eviction must never silently downgrade a refs guard that reported COMPLETE, so the guard needs to re-check (or pin) the keys it depended on. Decide only when we see real memory numbers. -- **Where do per-entry renders come from for jsonValues?** `executeRender` returns `{}` for these - records and Studio renders are null across the board today — see the Phase 6 list-view subsection. +- ~~**Where do per-entry renders come from for jsonValues?**~~ **Resolved (2026-07-31)**: from the CLIENT + schema instances — the SPA already has them via `window.__VAL_MODULES__` and discards them in + `setValModules`. See Phase 7. (`executeRender` returning `{}` for jsonValues and Studio renders being + null across the board are both fixed there.) ## Changelog @@ -658,6 +794,20 @@ unconditionally (accepted "validation takes more time" tradeoff). response + partial `items`; gated on a DECISION about renders being null Studio-wide — `/sources/~` only computes them when `apply_patches` is true, and the Studio always sends false), and **search showing partial results with a percentage** while the index fills. Added V18. +- **Session 7 (2026-07-31)**: planning — **Phase 7 (client-side schema instances)** added, and Phase 6 + step 7 moved into it. The SPA already has the user's real `Schema` instances via + `window.__VAL_MODULES__` → `extractValModules().schemas`; `setValModules` discards them and everything + downstream re-derives a `deserializeSchema` copy with no render `select`, no `customValidateFunctions` + (every case passes `[]`) and no router. Stage 1 renders from instances (all schema types, computed from + the PATCHED source, so draft-accurate). Stage 2 keeps the validation WORKER and adds custom validators: + gate per module on the serialized `customValidate` flag, worker reports the flagged PATHS, main thread + executes them time-sliced, error store merges instead of replacing. Triggers locked: on update only, + all patch-touched modules pre-publish, plus a `validateAll({custom, loadAllJsonEntries})` switch. New + `needs-keys` protocol so a custom validator on a `.jsonValues()` record asks for the keys it needs and + the caller loads them via the Phase 6 batch loader (validation = its fourth consumer). Stages 3–4: + instances become primary (`deserialize` leaves the main thread), then delete the now-dead server render + path. The "renders pincer" decision item is gone — it was premised on a wrong claim that only the server + could run `select`. - **Session 5 (2026-07-28)**: enabled/draft runtime path (server + RSC) and edit tags. - Merged `main` (which fixed an unrelated blocker: the publish gate read the RAW `errors.validationErrors` instead of the surfaced snapshot, so every `s.images()`/`s.files()` diff --git a/packages/next/src/rsc/initValRsc.ts b/packages/next/src/rsc/initValRsc.ts index 09fc2a7b6..ec58bb6ed 100644 --- a/packages/next/src/rsc/initValRsc.ts +++ b/packages/next/src/rsc/initValRsc.ts @@ -316,12 +316,19 @@ async function loadDraftJsonEntry( } const valServer = await valServerPromise; const res = await valServer["/json"]["GET"]({ - query: { path: moduleFilePath, key, apply_patches: true }, + query: { + path: moduleFilePath, + key, + keys: undefined, // single-entry shape + offset: undefined, + limit: undefined, + apply_patches: true, + }, cookies: { [VAL_SESSION_COOKIE]: cookies?.get(VAL_SESSION_COOKIE)?.value, }, }); - if (res.status === 200) { + if (res.status === 200 && "content" in res.json) { return res.json.content; } if (res.status === 401) { @@ -332,7 +339,10 @@ async function loadDraftJsonEntry( // No such entry in the draft state (e.g. removed by a pending patch). return undefined; } - console.error("Val: could not load draft JSON entry: ", res.json.message); + console.error( + "Val: could not load draft JSON entry: ", + "message" in res.json ? res.json.message : `status ${res.status}`, + ); return undefined; } diff --git a/packages/server/src/ValOps.ts b/packages/server/src/ValOps.ts index 8aff6f82d..d4233e344 100644 --- a/packages/server/src/ValOps.ts +++ b/packages/server/src/ValOps.ts @@ -279,6 +279,73 @@ export abstract class ValOps { | { status: "error"; message: string } | { status: "unauthorized"; message: string } > { + const res = await this.getJsonEntries( + moduleFilePath, + { keys: [entryKey] }, + opts, + ); + if (res.status !== "success") { + return res; + } + const entry = res.entries[0]; + if (entry !== undefined) { + return { status: "success", content: entry.content }; + } + const error = res.errors[0]; + if (error !== undefined) { + return { status: "error", message: error.message }; + } + return { + status: "not-found", + message: `Entry not found: ${entryKey} in ${moduleFilePath}`, + }; + } + + /** + * Resolves the content of MANY `.jsonValues()` entries in one pass. + * + * This is the single implementation; {@link getJsonEntry} is a one-key wrapper. + * Batching matters because the expensive parts — `initSources()` and + * `fetchPatches()` — are hoisted OUT of the per-entry loop: resolving 500 + * entries one-by-one would otherwise mean 500 patch fetches. + * + * Per-entry problems stay per-entry (`missing` / `errors`) so one corrupt + * `*.val.json` cannot fail a whole batch. Only a missing or non-record MODULE + * is a whole-request `not-found`. + * + * `selector` is either explicit `keys` or an `offset`/`limit` window over every + * key of the record, in module key order. The window form requires + * `applyPatches: false`: enumerating from the base source would silently omit + * draft-added keys, and a silently-short key list is exactly the class of bug + * this endpoint exists to avoid. + */ + async getJsonEntries( + moduleFilePath: ModuleFilePath, + selector: { keys: string[] } | { offset: number; limit: number }, + opts?: { applyPatches?: boolean }, + ): Promise< + | { + status: "success"; + entries: { key: string; content: JSONValue | null }[]; + missing: string[]; + errors: { key: string; message: string }[]; + total: number; + offset?: number; + limit?: number; + } + | { status: "not-found"; message: string } + | { status: "error"; message: string } + | { status: "unauthorized"; message: string } + > { + const applyPatches = opts?.applyPatches !== false; + const isWindow = !("keys" in selector); + if (isWindow && applyPatches) { + return { + status: "error", + message: + "Cannot enumerate json entries by offset/limit with apply_patches: the base key set would omit draft-added entries. Pass apply_patches=false, or request explicit keys.", + }; + } const { sources, schemas } = await this.initSources(); const moduleSource = sources[moduleFilePath]; if (moduleSource === undefined || moduleSource === null) { @@ -293,72 +360,115 @@ export abstract class ValOps { message: `Module is not a record: ${moduleFilePath}`, }; } - const marker = (moduleSource as Record)[entryKey]; - let baseContent: JSONValue | undefined = undefined; - if (marker !== undefined) { - if (!Internal.isJson(marker)) { - // Not a jsonValues entry — return the inlined value as-is (defensive). - return { status: "success", content: marker as JSONValue }; + const record = moduleSource as Record; + const allKeys = Object.keys(record); + const requestedKeys = isWindow + ? allKeys.slice(selector.offset, selector.offset + selector.limit) + : selector.keys; + + // Fetched once for the whole batch, not per entry. + let modulePatches: { patchId: PatchId; patch: Patch }[] = []; + let serializedSchema: SerializedSchema | undefined = undefined; + if (applyPatches) { + const patchOps = await this.fetchPatches({ excludePatchOps: false }); + if (patchOps.error) { + return { status: "error", message: patchOps.error.message }; } - const thunk = Internal.getJsonImport(marker); - if (thunk) { + if (patchOps.errors && Object.keys(patchOps.errors).length > 0) { + return { + status: "error", + message: `Could not fetch patches: ${JSON.stringify(patchOps.errors)}`, + }; + } + modulePatches = patchOps.patches + .filter((p) => p.path === moduleFilePath && !p.appliedAt) + .map((p) => ({ patchId: p.patchId, patch: p.patch })); + try { + serializedSchema = schemas[moduleFilePath]?.["executeSerialize"](); + } catch { + // Serialization errors are reported elsewhere; treat as "no schema". + } + } + + const entries: { key: string; content: JSONValue | null }[] = []; + const missing: string[] = []; + const errors: { key: string; message: string }[] = []; + type ResolvedEntry = + | { + entryKey: string; + baseContent: JSONValue | undefined; + /** Set when the value lives in the module source, not a `*.val.json`. */ + inline?: true; + } + | { entryKey: string; message: string }; + const resolved = await Promise.all( + requestedKeys.map(async (entryKey): Promise => { + const marker = record[entryKey]; + if (marker !== undefined && !Internal.isJson(marker)) { + // Not a jsonValues entry — return the inlined value as-is (defensive). + // `inline` skips patch replay: entry patches are expressed against a + // jsonValues entry, and this value is part of the module source proper. + return { entryKey, baseContent: marker as JSONValue, inline: true }; + } + if (marker === undefined) { + return { entryKey, baseContent: undefined }; + } + const thunk = Internal.getJsonImport(marker); + if (!thunk) { + return { entryKey, baseContent: null }; + } try { - baseContent = ((await thunk()).default ?? null) as JSONValue; + return { + entryKey, + baseContent: ((await thunk()).default ?? null) as JSONValue, + }; } catch (e) { return { - status: "error", + entryKey, message: `Failed to load JSON entry '${entryKey}': ${ e instanceof Error ? e.message : String(e) }`, }; } - } else { - baseContent = null; + }), + ); + for (const result of resolved) { + const { entryKey } = result; + if ("message" in result) { + errors.push({ key: entryKey, message: result.message }); + continue; } - } - if (opts?.applyPatches === false) { - if (baseContent === undefined) { - return { - status: "not-found", - message: `Entry not found: ${entryKey} in ${moduleFilePath}`, - }; + const { baseContent } = result; + if (!applyPatches || "inline" in result) { + if (baseContent === undefined) { + missing.push(entryKey); + } else { + entries.push({ key: entryKey, content: baseContent }); + } + continue; + } + const res = applyJsonValuesEntryPatches({ + serializedSchema, + entryKey, + baseContent, + patches: modulePatches, + }); + if (res.kind === "error") { + errors.push({ key: entryKey, message: res.message }); + } else if (res.kind === "deleted") { + missing.push(entryKey); + } else { + entries.push({ key: entryKey, content: res.content }); } - return { status: "success", content: baseContent }; - } - const patchOps = await this.fetchPatches({ excludePatchOps: false }); - if (patchOps.error) { - return { status: "error", message: patchOps.error.message }; - } - if (patchOps.errors && Object.keys(patchOps.errors).length > 0) { - return { - status: "error", - message: `Could not fetch patches: ${JSON.stringify(patchOps.errors)}`, - }; - } - let serializedSchema: SerializedSchema | undefined = undefined; - try { - serializedSchema = schemas[moduleFilePath]?.["executeSerialize"](); - } catch { - // Serialization errors are reported elsewhere; treat as "no schema". - } - const res = applyJsonValuesEntryPatches({ - serializedSchema, - entryKey, - baseContent, - patches: patchOps.patches - .filter((p) => p.path === moduleFilePath && !p.appliedAt) - .map((p) => ({ patchId: p.patchId, patch: p.patch })), - }); - if (res.kind === "error") { - return { status: "error", message: res.message }; - } - if (res.kind === "deleted") { - return { - status: "not-found", - message: `Entry not found: ${entryKey} in ${moduleFilePath}`, - }; } - return { status: "success", content: res.content }; + return { + status: "success", + entries, + missing, + errors, + total: allKeys.length, + ...(isWindow ? { offset: selector.offset, limit: selector.limit } : {}), + }; } async getSchemas(): Promise { return this.initSources().then((result) => result.schemas); diff --git a/packages/server/src/ValOpsFS.jsonValues.test.ts b/packages/server/src/ValOpsFS.jsonValues.test.ts index 90d4ff919..c70d6a0cc 100644 --- a/packages/server/src/ValOpsFS.jsonValues.test.ts +++ b/packages/server/src/ValOpsFS.jsonValues.test.ts @@ -220,6 +220,154 @@ describe("ValOps.getJsonEntry", () => { }); }); +describe("ValOps.getJsonEntries (batch)", () => { + test("resolves many keys in one call", async () => { + const { ops } = setup(); + const res = await ops.getJsonEntries(MODULE_PATH, { + keys: ["/blog/hello", "/blog/world"], + }); + expect(res).toEqual({ + status: "success", + entries: [ + { key: "/blog/hello", content: { title: "Hello", order: 1 } }, + { key: "/blog/world", content: { title: "World", order: 2 } }, + ], + missing: [], + errors: [], + total: 2, + }); + }); + + test("keeps the requested key order, and echoes it back", async () => { + const { ops } = setup(); + const res = await ops.getJsonEntries(MODULE_PATH, { + keys: ["/blog/world", "/blog/hello"], + }); + expect(res.status === "success" && res.entries.map((e) => e.key)).toEqual([ + "/blog/world", + "/blog/hello", + ]); + }); + + test("an unknown key is `missing`, NOT a failed batch", async () => { + const { ops } = setup(); + const res = await ops.getJsonEntries(MODULE_PATH, { + keys: ["/blog/hello", "/nope"], + }); + expect(res).toMatchObject({ + status: "success", + entries: [{ key: "/blog/hello", content: { title: "Hello", order: 1 } }], + missing: ["/nope"], + errors: [], + }); + }); + + test("a corrupt entry is a per-key error, NOT a failed batch", async () => { + const { ops, rootDir } = setup(); + fs.writeFileSync( + path.join(rootDir, "test/content/world.val.json"), + "{ not json", + "utf-8", + ); + const res = await ops.getJsonEntries(MODULE_PATH, { + keys: ["/blog/hello", "/blog/world"], + }); + expect(res.status).toBe("success"); + if (res.status !== "success") throw new Error("unreachable"); + expect(res.entries).toEqual([ + { key: "/blog/hello", content: { title: "Hello", order: 1 } }, + ]); + expect(res.errors).toHaveLength(1); + expect(res.errors[0].key).toBe("/blog/world"); + }); + + test("applies pending patches per entry", async () => { + const { ops } = setup(); + await createPatch(ops, [ + { op: "replace", path: ["/blog/hello", "title"], value: "Draft!" }, + { op: "remove", path: ["/blog/world"] }, + ]); + const res = await ops.getJsonEntries(MODULE_PATH, { + keys: ["/blog/hello", "/blog/world"], + }); + expect(res).toMatchObject({ + status: "success", + entries: [{ key: "/blog/hello", content: { title: "Draft!", order: 1 } }], + missing: ["/blog/world"], + }); + }); + + test("offset/limit pages over the record in key order", async () => { + const { ops } = setup(); + const first = await ops.getJsonEntries( + MODULE_PATH, + { offset: 0, limit: 1 }, + { applyPatches: false }, + ); + expect(first).toEqual({ + status: "success", + entries: [{ key: "/blog/hello", content: { title: "Hello", order: 1 } }], + missing: [], + errors: [], + offset: 0, + limit: 1, + total: 2, + }); + const second = await ops.getJsonEntries( + MODULE_PATH, + { offset: 1, limit: 10 }, + { applyPatches: false }, + ); + expect(second).toMatchObject({ + status: "success", + entries: [{ key: "/blog/world", content: { title: "World", order: 2 } }], + offset: 1, + limit: 10, + total: 2, + }); + // Past the end: empty, not an error. + expect( + await ops.getJsonEntries( + MODULE_PATH, + { offset: 5, limit: 10 }, + { applyPatches: false }, + ), + ).toMatchObject({ status: "success", entries: [], total: 2 }); + }); + + test("offset/limit is rejected when patches would be applied", async () => { + const { ops } = setup(); + // The base key set cannot represent draft-added keys, so enumerating with + // apply_patches would silently return a short list. + const res = await ops.getJsonEntries(MODULE_PATH, { offset: 0, limit: 10 }); + expect(res.status).toBe("error"); + }); + + test("an unknown module is not-found for the whole request", async () => { + const { ops } = setup(); + expect( + ( + await ops.getJsonEntries("/test/nope.val.ts" as ModuleFilePath, { + keys: ["/blog/hello"], + }) + ).status, + ).toBe("not-found"); + }); + + test("fetches patches ONCE for the whole batch", async () => { + const { ops } = setup(); + await createPatch(ops, [ + { op: "replace", path: ["/blog/hello", "title"], value: "Draft!" }, + ]); + const fetchPatches = jest.spyOn(ops, "fetchPatches"); + await ops.getJsonEntries(MODULE_PATH, { + keys: ["/blog/hello", "/blog/world"], + }); + expect(fetchPatches).toHaveBeenCalledTimes(1); + fetchPatches.mockRestore(); + }); +}); + describe("ValOpsFS jsonValues commit flow", () => { test("content edit writes only the *.val.json (not the .val.ts)", async () => { const { ops } = setup(); diff --git a/packages/server/src/ValRouter.test.ts b/packages/server/src/ValRouter.test.ts index 7c2f8ae9a..c7a2a29d9 100644 --- a/packages/server/src/ValRouter.test.ts +++ b/packages/server/src/ValRouter.test.ts @@ -95,6 +95,55 @@ describe("ValRouter", () => { expect("json" in serverRes && serverRes.json).toBeTruthy(); }); + // `/json` takes exactly one of `key`, `keys` or `offset`+`limit`. These are + // rejected before the ops layer is consulted, so the fixture module (an + // ordinary record) is enough to pin the contract. + describe("/json request shapes", () => { + const jsonRequest = (query: string) => + onRoute( + fakeRequest({ + method: "GET", + url: new URL(`http://localhost:3000/api/val/json?${query}`), + headers: new Headers({ + Cookie: `val_session=${encodeJwt({}, "")}`, + }), + }), + ); + + test("no shape is a 400", async () => { + expect((await jsonRequest("path=/content/authors.val.ts")).status).toBe( + 400, + ); + }); + + test("key AND keys together is a 400", async () => { + expect( + (await jsonRequest("path=/content/authors.val.ts&key=a&keys=b")).status, + ).toBe(400); + }); + + test("offset without limit is a 400", async () => { + expect( + (await jsonRequest("path=/content/authors.val.ts&offset=0")).status, + ).toBe(400); + }); + + test("more keys than the batch cap is rejected", async () => { + const keys = Array.from({ length: 101 }, (_, i) => `keys=k${i}`).join( + "&", + ); + expect( + (await jsonRequest(`path=/content/authors.val.ts&${keys}`)).status, + ).toBe(400); + }); + + test("a valid shape gets past validation (404: no such module)", async () => { + expect( + (await jsonRequest("path=/content/nope.val.ts&key=a")).status, + ).toBe(404); + }); + }); + test("smoke test invalid route", async () => { const serverRes = await onRoute( fakeRequest({ diff --git a/packages/server/src/ValServer.ts b/packages/server/src/ValServer.ts index 39fd10cb1..b7ef0f2ac 100644 --- a/packages/server/src/ValServer.ts +++ b/packages/server/src/ValServer.ts @@ -1376,12 +1376,53 @@ export const ValServer = ( return { status: 401, json: { message: "Unauthorized" } }; } const moduleFilePath = req.query.path as ModuleFilePath; - const key = req.query.key; + const { key, keys, offset, limit } = req.query; // Defaults to true, mirroring /sources/~. The Studio opts out. const applyPatches = req.query.apply_patches !== false; - const res = await serverOps.getJsonEntry(moduleFilePath, key, { - applyPatches, - }); + const isWindow = offset !== undefined || limit !== undefined; + const shapes = [key !== undefined, keys !== undefined, isWindow].filter( + Boolean, + ).length; + if (shapes !== 1) { + return { + status: 400, + json: { + message: + "Exactly one of 'key', 'keys' or 'offset'+'limit' must be given", + }, + }; + } + if (isWindow && (offset === undefined || limit === undefined)) { + return { + status: 400, + json: { message: "'offset' and 'limit' must be given together" }, + }; + } + if (key !== undefined) { + const res = await serverOps.getJsonEntry(moduleFilePath, key, { + applyPatches, + }); + if (res.status === "unauthorized") { + return { status: 401, json: { message: res.message } }; + } + if (res.status === "not-found") { + return { status: 404, json: { message: res.message } }; + } + if (res.status === "error") { + return { status: 500, json: { message: res.message } }; + } + return { + status: 200, + json: { path: moduleFilePath, key, content: res.content }, + }; + } + const res = await serverOps.getJsonEntries( + moduleFilePath, + keys !== undefined + ? { keys } + : { offset: offset as number, limit: limit as number }, + { applyPatches }, + ); if (res.status === "unauthorized") { return { status: 401, json: { message: res.message } }; } @@ -1393,7 +1434,15 @@ export const ValServer = ( } return { status: 200, - json: { path: moduleFilePath, key, content: res.content }, + json: { + path: moduleFilePath, + entries: res.entries, + missing: res.missing, + errors: res.errors, + ...(res.offset !== undefined ? { offset: res.offset } : {}), + ...(res.limit !== undefined ? { limit: res.limit } : {}), + total: res.total, + }, }; }, }, diff --git a/packages/shared/src/internal/ApiRoutes.ts b/packages/shared/src/internal/ApiRoutes.ts index 333230823..eb13a668b 100644 --- a/packages/shared/src/internal/ApiRoutes.ts +++ b/packages/shared/src/internal/ApiRoutes.ts @@ -128,6 +128,19 @@ const onlyOneBooleanQueryParam = onlyOneStringQueryParam "Value must be true or false", ) .transform((arg) => arg === "true"); +const onlyOneIntQueryParam = onlyOneStringQueryParam + .refine( + (arg) => /^\d+$/.test(arg), + "Value must be a non-negative whole number", + ) + .transform((arg) => parseInt(arg, 10)); + +/** + * Upper bound on how many `.jsonValues()` entries one `/json` request may ask + * for. Callers page through bigger sets; the Studio chunks its key windows to + * this. Shared so client and server agree on the limit. + */ +export const JSON_ENTRIES_BATCH_MAX = 100; export const Api = { "/draft/enable": { @@ -1010,20 +1023,45 @@ export const Api = { ]), }, }, - // Loads the content of a single `.jsonValues()` record/router entry by key, - // so the Studio can lazily load just the entry being opened (instead of the - // whole record), and so the runtime can read draft edits in draft mode. + // Loads the content of `.jsonValues()` record/router entries, so the Studio can + // lazily load just the entries it needs (instead of the whole record), and so + // the runtime can read draft edits in draft mode. + // + // Three request shapes, exactly one of which must be used: + // 1. `key=` — ONE entry; responds `{path, key, content}`. + // 2. `keys=&keys=&…` — a BATCH; responds `{path, entries, missing, errors}`. + // 3. `offset=&limit=` — a PAGE of every entry, in module key order; + // responds as (2) plus `{offset, limit, total}`. + // (2) and (3) are what make loading many entries one round trip instead of N: + // the server resolves sources and pending patches ONCE per request, not per entry. + // + // Batch failures are per entry, never whole-request: a key with no entry lands in + // `missing`, a key whose `*.val.json` fails to load lands in `errors`. Only a + // missing/non-record MODULE is a 404. // // `apply_patches` defaults to TRUE (as on `/sources/~`): the server replays // pending patches for the entry. The Studio passes `false` explicitly, because // it owns in-flight client patches the server has not seen yet and applies // them itself — letting the server apply them too would double-apply. + // + // Shape (3) REQUIRES `apply_patches=false`. Enumerating "every entry" from the + // base source would silently omit draft-added keys, and the only all-mode caller + // (the Studio) derives its key set from its own patched source anyway. Callers + // that need draft-aware enumeration pass explicit `keys`. "/json": { GET: { req: { query: { path: onlyOneStringQueryParam, - key: onlyOneStringQueryParam, + key: onlyOneStringQueryParam.optional(), + keys: z.array(z.string()).max(JSON_ENTRIES_BATCH_MAX).optional(), + offset: onlyOneIntQueryParam.optional(), + limit: onlyOneIntQueryParam + .refine( + (arg) => arg > 0 && arg <= JSON_ENTRIES_BATCH_MAX, + `limit must be between 1 and ${JSON_ENTRIES_BATCH_MAX}`, + ) + .optional(), apply_patches: onlyOneBooleanQueryParam.optional(), }, cookies: { [VAL_SESSION_COOKIE]: z.string().optional() }, @@ -1031,6 +1069,7 @@ export const Api = { res: z.union([ unauthorizedResponse, notFoundResponse, + // Shape (1): single `key`. z.object({ status: z.literal(200), json: z.object({ @@ -1040,6 +1079,32 @@ export const Api = { content: z.any(), }), }), + // Shapes (2) and (3): `keys` or `offset`+`limit`. + z.object({ + status: z.literal(200), + json: z.object({ + path: ModuleFilePath, + entries: z.array( + z.object({ + key: z.string(), + // The entry's JSON content (or null if the entry has no value). + content: z.any(), + }), + ), + // Requested keys that have no entry (neither committed nor drafted). + missing: z.array(z.string()), + // Requested keys whose content could not be loaded. + errors: z.array(z.object({ key: z.string(), message: z.string() })), + // All-mode only: the resolved window and the record's total key count. + offset: z.number().optional(), + limit: z.number().optional(), + total: z.number().optional(), + }), + }), + z.object({ + status: z.literal(400), + json: GenericError, + }), z.object({ status: z.literal(500), json: GenericError, @@ -1289,7 +1354,14 @@ type DefinedObject = Pick>; * * Do not change this without updating the ValRouter query parsing logic * */ -export type ValidQueryParamTypes = boolean | string | string[] | undefined; +// `number` is allowed because the client stringifies every query value before it +// goes on the URL (see createValClient), so a numeric param round-trips fine. +export type ValidQueryParamTypes = + | boolean + | number + | string + | string[] + | undefined; export type ApiEndpoint = { req: { path?: z.ZodString | z.ZodOptional; diff --git a/packages/ui/spa/ValSyncEngine.test.ts b/packages/ui/spa/ValSyncEngine.test.ts index e8152194d..a8423a6cf 100644 --- a/packages/ui/spa/ValSyncEngine.test.ts +++ b/packages/ui/spa/ValSyncEngine.test.ts @@ -771,6 +771,8 @@ class SyncEngineTester { fakeJsonEntries: Record> = {}; /** How many times GET /json was called, keyed `${path}\0${key}`. */ jsonRequestCounts: Record = {}; + /** HTTP requests (not keys) served by the batch shapes of `/json`, per module. */ + jsonBatchRequestCounts: Record = {}; constructor( private mode: "fs" | "http", @@ -858,22 +860,65 @@ class SyncEngineTester { req: InferReq, ): z.infer { const path = req.query.path; - const key = req.query.key; - this.jsonRequestCounts[`${path}\0${key}`] = - (this.jsonRequestCounts[`${path}\0${key}`] ?? 0) + 1; - const entries = this.fakeJsonEntries[path]; - if (entries === undefined || !(key in entries)) { + const { key, keys, offset, limit } = req.query; + const countKey = (k: string) => { + this.jsonRequestCounts[`${path}\0${k}`] = + (this.jsonRequestCounts[`${path}\0${k}`] ?? 0) + 1; + }; + const available = this.fakeJsonEntries[path]; + if (key !== undefined) { + countKey(key); + if (available === undefined || !(key in available)) { + return { + status: 404, + json: { message: `Entry not found: ${key} in ${path}` }, + }; + } return { - status: 404, - json: { message: `Entry not found: ${key} in ${path}` }, + status: 200, + json: { + path: path as ModuleFilePath, + key, + content: available[key], + }, + }; + } + // Batch shapes: one HTTP request, per-key outcomes. + this.jsonBatchRequestCounts[path] = + (this.jsonBatchRequestCounts[path] ?? 0) + 1; + const allKeys = Object.keys(available ?? {}); + const isWindow = offset !== undefined && limit !== undefined; + const requestedKeys = isWindow + ? allKeys.slice(offset, offset + limit) + : keys; + if (requestedKeys === undefined) { + return { + status: 400, + json: { + message: + "Exactly one of 'key', 'keys' or 'offset'+'limit' must be given", + }, }; } + const entries: { key: string; content: unknown }[] = []; + const missing: string[] = []; + for (const requestedKey of requestedKeys) { + countKey(requestedKey); + if (available !== undefined && requestedKey in available) { + entries.push({ key: requestedKey, content: available[requestedKey] }); + } else { + missing.push(requestedKey); + } + } return { status: 200, json: { path: path as ModuleFilePath, - key, - content: entries[key], + entries, + missing, + errors: [], + ...(isWindow ? { offset, limit } : {}), + total: allKeys.length, }, }; } diff --git a/packages/ui/spa/ValSyncEngine.ts b/packages/ui/spa/ValSyncEngine.ts index f612c06ab..8d2e11cc5 100644 --- a/packages/ui/spa/ValSyncEngine.ts +++ b/packages/ui/spa/ValSyncEngine.ts @@ -970,10 +970,17 @@ export class ValSyncEngine { // apply_patches=false: we own in-flight client patches the server has not // seen yet and apply them ourselves in `getPatchedSource`. Letting the // server apply them too would double-apply. - query: { path: moduleFilePath, key, apply_patches: false }, + query: { + path: moduleFilePath, + key, + keys: undefined, // single-entry shape + offset: undefined, + limit: undefined, + apply_patches: false, + }, }) .then((res) => { - if (res.status === 200) { + if (res.status === 200 && "content" in res.json) { if (this.jsonEntryContents[moduleFilePath] === undefined) { this.jsonEntryContents[moduleFilePath] = {}; } From dbdaa1c68371e7d89676978fa7de8e1b375c70af Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Fri, 31 Jul 2026 12:25:29 +0200 Subject: [PATCH 33/35] jsonValues: batch entry loading in the sync engine (Phase 6 step 2) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the two engine primitives that sit on top of the batch /json endpoint, so the Studio can load many `.jsonValues()` entries without one request per entry. - `requestJsonEntries(mfp, keys)`: fire-and-forget window loader — what a virtualized list will call for the rows it just rendered. - `ensureJsonEntries(mfps)`: awaitable, whole-module, and returns `{complete, errors}` rather than void. A guard that gates a delete has to be able to tell "loaded everything" from "an entry failed"; collapsing those is the bug this phase exists to fix. Runs up to 3 passes because an invalidation landing mid-flight re-marks entries stale, and reporting `complete` while holding pre-invalidation content would be the same lie in a new place. Both delegate to one private `loadJsonEntries` (filter -> chunk at 50 -> `loadJsonEntryChunk`), so there is a single cache / in-flight / emit path. Notable behaviours: - Keys that exist ONLY in a pending patch are never requested: they have no committed content, so `/json` would 404 and wrongly mark the row errored, when in fact their value comes from the patch via getPatchedSource. - Cached, in-flight and previously-failed keys are skipped, so re-rendering the same window costs nothing. - ONE `invalidateSource` per batch instead of one per entry. Progress store: `{status, loaded, total, percentage}` on `subscribe("json-entries-progress")` / `getJsonEntriesProgressSnapshot()`. Counted per RUN across modules and batches, so a UI percentage never resets at a module boundary; resets once nothing is in flight. `loaded` counts resolved keys (loaded, missing AND failed) so a failing entry cannot stall a bar at 99%. The single-entry path counts into the same run. `markJsonEntriesStale` now batches: a publish with hundreds of cached entries is one request per module instead of one per entry. Tests: +8 (one batch per window, no refetch of cached/in-flight keys, patch-only keys never requested, ensureJsonEntries complete vs failed, progress counting and reset, subscriber notification, batched publish refresh). Co-Authored-By: Claude Opus 5 --- docs/plans/jsonValues.md | 57 +++-- packages/ui/spa/ValSyncEngine.test.ts | 183 +++++++++++++ packages/ui/spa/ValSyncEngine.ts | 355 +++++++++++++++++++++++++- 3 files changed, 572 insertions(+), 23 deletions(-) diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md index 195a68683..18ff95448 100644 --- a/docs/plans/jsonValues.md +++ b/docs/plans/jsonValues.md @@ -10,6 +10,16 @@ ## Current state / resume here +> **Phase 6 step 2 DONE (2026-07-31): engine primitives.** `requestJsonEntries(mfp, keys)` +> (fire-and-forget window) and `ensureJsonEntries(mfps)` (awaitable, whole-module, returns +> `{complete, errors}` so a guard can tell "loaded" from "failed") both delegate to one private +> `loadJsonEntries` — filter, chunk at 50, one `/json` per chunk, one `invalidateSource` per batch. +> Keys that exist only in a pending patch are never requested (no committed content to fetch; a request +> would 404 and wrongly mark the row errored). Progress is `{status, loaded, total, percentage}` on +> `subscribe("json-entries-progress")`, counted per RUN across modules so a percentage never resets at a +> module boundary. Publish invalidation is batched. +8 tests (28 in the jsonValues describe). +> **Next: step 3 — virtualize the record list + load the rendered window (V16).** + > **Phase 6 step 1 DONE (2026-07-31): batch `/json`.** `ValOps.getJsonEntries` is the single > implementation (`getJsonEntry` is a one-key wrapper); `initSources`/`fetchPatches` are hoisted out of > the per-entry loop; per-entry failures are per-entry (`missing`/`errors`) so one corrupt `*.val.json` @@ -495,20 +505,25 @@ not a step. `{kind:"file", module}` / `{kind:"route"}`; returns the `ModuleFilePath[]` whose entries must be loaded. Own unit test file: empty result for the incoming-ref case, non-empty for nested `keyOf`/image/file, `route` over-approximation, transitivity through object/array/record/union. -- [ ] **`ValSyncEngine.ensureJsonEntries(moduleFilePaths)`** — idempotent batch loader: for each - module, diff the marker key set against `jsonEntryContents` + `staleJsonEntries`, request the - missing keys in chunks (start at 50) through the batch endpoint, fill the SAME - `jsonEntryContents` cache, then the existing invalidate + emit. Returns a promise that resolves - when the requested modules are fully loaded. Never fires on Studio boot. -- [ ] **Progress store** — `{loaded, total, status}` (not a boolean) exposed as a sync-external-store - snapshot (`subscribe("json-entries-progress")` + `getJsonEntriesProgressSnapshot()`), so any - component can render "Checking references… 340/5000". `total` needs no server help: the client - already has every entry key from the marker record. -- [ ] **`ValSyncEngine.requestJsonEntries(mfp, keys)`** — the window-based sibling of - `ensureJsonEntries`: fire-and-forget, takes an explicit key list, skips cached/in-flight/errored - keys, batches the rest in one request. This is what the virtualized list calls per rendered - window; `ensureJsonEntries(modules)` (awaitable, whole-module) stays for the refs guard and search. - Both share the chunking + cache + emit path. +- [x] **`ValSyncEngine.ensureJsonEntries(moduleFilePaths)`** — DONE (2026-07-31). Awaitable, + whole-module, and it returns `{complete, errors}` rather than plain `void`: a guard that gates a + delete must be able to tell "loaded everything" from "some entry failed", which is the entire + point. Runs up to 3 passes, because an invalidation landing mid-flight re-marks entries stale and + reporting `complete` while holding pre-invalidation content would be the same lie in a new place. + Never called on boot. +- [x] **Progress store** — DONE (2026-07-31). `{status, loaded, total, percentage}` via + `subscribe("json-entries-progress")` + `getJsonEntriesProgressSnapshot()`. Counts the whole RUN + across modules and batches (not per module) so a percentage never resets at a module boundary; + resets to zero once nothing is in flight. `loaded` counts resolved keys — loaded, missing AND + failed — so a failing entry cannot stall the bar at 99%. The single-entry `ensureJsonEntry` counts + into the same run, so one indicator covers an opened entry and a loading list. +- [x] **`ValSyncEngine.requestJsonEntries(mfp, keys)`** — DONE (2026-07-31). Fire-and-forget window + loader; both public methods delegate to one private `loadJsonEntries` (filter → chunk at 50 → + `loadJsonEntryChunk`), so there is a single cache/in-flight/emit path. Skips + cached/in-flight/errored keys, so re-rendering the same window costs nothing, and **skips keys that + exist only in a pending patch** — they have no committed content, so requesting them would 404 and + wrongly mark the row errored (their value comes from the patch, via `getPatchedSource`). One + `invalidateSource` per batch rather than per entry. - [ ] **Virtualize the record list + load visible rows only** — `RecordFields.tsx`: wrap both list branches (default grid at :162 and `ListRecordRenderComponent` at :191) in `useVirtualizer` from `@tanstack/react-virtual`, and for a jsonValues record call @@ -558,13 +573,13 @@ not a step. correctness + kills the `_type`/`patch_id` edge). - [ ] **Delete the dead `createSearchIndex.ts` + `search.test.ts`** and the unused barrel export in `search/index.ts`. The marker skip added there was on unreachable code. -- [ ] **Fold `markAllJsonEntriesStale` into the batch path** — - [ValSyncEngine.ts:1063](../../packages/ui/spa/ValSyncEngine.ts#L1063) currently re-requests every - loaded entry one-by-one, so with hundreds cached a publish is a request storm today. The progress - store must cover this post-publish refresh too: anything transitively derived from stale entries - (entry detail view, refs guard, search index) goes back to `loading` with progress rather than - briefly rendering stale or content-free values. In particular a refs query must NOT answer from - stale content — it re-enters `loading`. +- [x] **Fold `markAllJsonEntriesStale` into the batch path** — DONE (2026-07-31). + `markJsonEntriesStale` now marks the module's loaded keys stale and hands them to + `requestJsonEntries` in one call, so a publish with hundreds of cached entries is one batch per + module instead of one request per entry (pinned by a test that counts batches AND per-key + requests). It feeds the same progress store, so the post-publish refresh is visible. + **Still open**: making the consumers re-enter `loading` on that refresh (refs guard, search index, + entry detail) — that lands with those consumers in steps 5–6, since they do not exist yet. - [ ] **Verify** (Studio running): - **V10** rename/delete a key in a jsonValues router while another ORDINARY module holds a `keyOf`/`route` ref to it → refs found, delete blocked, rename rewrites the referrer, and **zero** diff --git a/packages/ui/spa/ValSyncEngine.test.ts b/packages/ui/spa/ValSyncEngine.test.ts index a8423a6cf..c3db24aa1 100644 --- a/packages/ui/spa/ValSyncEngine.test.ts +++ b/packages/ui/spa/ValSyncEngine.test.ts @@ -695,6 +695,189 @@ describe("ValSyncEngine", () => { // real content, NOT an opaque marker expect(source["/renamed"]).toEqual({ title: "A", order: 1 }); }); + + describe("batch loading", () => { + test("requestJsonEntries loads a window in ONE request", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + engine.requestJsonEntries(toModuleFilePath(PAGES), ["/a", "/b"]); + await flush(); + + const source = engine.getSourceSnapshot(toModuleFilePath(PAGES)) + .data as any; + expect(source["/a"]).toEqual({ title: "A", order: 1 }); + expect(source["/b"]).toEqual({ title: "B", order: 2 }); + // Two keys, ONE HTTP request — the point of the batch. + expect(tester.jsonBatchRequestCounts[PAGES]).toBe(1); + }); + + test("already-loaded and in-flight keys are not refetched", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/a"); + // /a is cached; only /b should go out. + engine.requestJsonEntries(toModuleFilePath(PAGES), ["/a", "/b"]); + // Re-rendering the same window while it is in flight must add nothing. + engine.requestJsonEntries(toModuleFilePath(PAGES), ["/a", "/b"]); + await flush(); + + expect(tester.jsonRequestCounts[`${PAGES}\0/a`]).toBe(1); + expect(tester.jsonRequestCounts[`${PAGES}\0/b`]).toBe(1); + expect(tester.jsonBatchRequestCounts[PAGES]).toBe(1); + }); + + test("keys that exist only in a pending patch are never requested", async () => { + // They have no committed content: their value comes from the patch, so + // asking /json would 404 and wrongly mark the row errored. + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + engine.addPatch( + toSourcePath(PAGES), + "record", + [{ op: "add", path: ["/drafted"], value: { title: "D", order: 3 } }], + tester.getNextNow(), + ); + engine.requestJsonEntries(toModuleFilePath(PAGES), ["/drafted"]); + await flush(); + + expect(tester.jsonRequestCounts[`${PAGES}\0/drafted`]).toBeUndefined(); + expect( + engine.getJsonEntryError(toModuleFilePath(PAGES), "/drafted"), + ).toBe(null); + expect( + (engine.getSourceSnapshot(toModuleFilePath(PAGES)).data as any)[ + "/drafted" + ], + ).toEqual({ title: "D", order: 3 }); + }); + + test("ensureJsonEntries loads every committed entry and reports complete", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + const res = await engine.ensureJsonEntries([toModuleFilePath(PAGES)]); + expect(res).toEqual({ complete: true, errors: [] }); + const source = engine.getSourceSnapshot(toModuleFilePath(PAGES)) + .data as any; + expect(source["/a"]).toEqual({ title: "A", order: 1 }); + expect(source["/b"]).toEqual({ title: "B", order: 2 }); + expect(tester.jsonBatchRequestCounts[PAGES]).toBe(1); + + // Fully cached: resolves without another request. + expect( + await engine.ensureJsonEntries([toModuleFilePath(PAGES)]), + ).toEqual({ complete: true, errors: [] }); + expect(tester.jsonBatchRequestCounts[PAGES]).toBe(1); + }); + + test("ensureJsonEntries reports NOT complete when an entry fails", async () => { + // The whole point: a guard that gates a delete must never read a failed + // load as "no references found". + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + delete tester.fakeJsonEntries[PAGES]["/b"]; + + const res = await engine.ensureJsonEntries([toModuleFilePath(PAGES)]); + expect(res.complete).toBe(false); + expect(res.errors).toHaveLength(1); + expect(res.errors[0]).toMatchObject({ + moduleFilePath: PAGES, + key: "/b", + }); + // The entry that DID load is still usable. + expect( + (engine.getSourceSnapshot(toModuleFilePath(PAGES)).data as any)["/a"], + ).toEqual({ title: "A", order: 1 }); + }); + + test("progress counts up across the run and resets when it settles", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + expect(engine.getJsonEntriesProgressSnapshot()).toEqual({ + status: "idle", + loaded: 0, + total: 0, + percentage: 100, + }); + + const done = engine.ensureJsonEntries([toModuleFilePath(PAGES)]); + const during = engine.getJsonEntriesProgressSnapshot(); + expect(during.status).toBe("loading"); + expect(during.total).toBe(2); + expect(during.loaded).toBe(0); + expect(during.percentage).toBe(0); + + await done; + // Nothing in flight: the run is over and the next one starts from zero. + expect(engine.getJsonEntriesProgressSnapshot()).toEqual({ + status: "idle", + loaded: 0, + total: 0, + percentage: 100, + }); + }); + + test("progress notifies subscribers", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + let notified = 0; + const unsubscribe = engine.subscribe("json-entries-progress")(() => { + notified++; + }); + + await engine.ensureJsonEntries([toModuleFilePath(PAGES)]); + expect(notified).toBeGreaterThan(0); + unsubscribe(); + }); + + test("publish refetches loaded entries in ONE batch, not one request each", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + await engine.ensureJsonEntries([toModuleFilePath(PAGES)]); + + engine.addPatch( + toSourcePath(PAGES), + "record", + [{ op: "replace", path: ["/a", "title"], value: "A edited" }], + tester.getNextNow(), + ); + tester.simulatePassingOfSeconds(5); + await engine.sync(tester.getNextNow()); + await flush(); + const patchIds = tester.fakePatches.map((p) => p.patchId); + tester.fakeJsonEntries[PAGES]["/a"] = { title: "A edited", order: 1 }; + // Measured from just before publish: the sync above also refreshes the + // module's entries (its own single batch). + const batchesBefore = tester.jsonBatchRequestCounts[PAGES] ?? 0; + const keyRequestsBefore = { + a: tester.jsonRequestCounts[`${PAGES}\0/a`] ?? 0, + b: tester.jsonRequestCounts[`${PAGES}\0/b`] ?? 0, + }; + + expect( + await engine.publish(patchIds, undefined, tester.getNextNow()), + ).toMatchObject({ status: "done" }); + await flush(); + + // BOTH cached entries were invalidated and refetched... + expect(tester.jsonRequestCounts[`${PAGES}\0/a`]).toBe( + keyRequestsBefore.a + 1, + ); + expect(tester.jsonRequestCounts[`${PAGES}\0/b`]).toBe( + keyRequestsBefore.b + 1, + ); + // ...in ONE request, not one per entry. + expect(tester.jsonBatchRequestCounts[PAGES]).toBe(batchesBefore + 1); + expect( + (engine.getSourceSnapshot(toModuleFilePath(PAGES)).data as any)["/a"], + ).toEqual({ title: "A edited", order: 1 }); + }); + }); }); }); diff --git a/packages/ui/spa/ValSyncEngine.ts b/packages/ui/spa/ValSyncEngine.ts index 8d2e11cc5..efc22df2b 100644 --- a/packages/ui/spa/ValSyncEngine.ts +++ b/packages/ui/spa/ValSyncEngine.ts @@ -25,6 +25,7 @@ import { JSONValue, } from "@valbuild/core/patch"; import { + JSON_ENTRIES_BATCH_MAX, ParentRef, ValClient, Patch, @@ -165,6 +166,17 @@ export class ValSyncEngine { * a published edit appears to revert to the pre-edit content. */ private staleJsonEntries: Set = new Set(); + /** + * Progress of the CURRENT json-entry load run, spanning every module and every + * batch in flight — deliberately not per module, so a UI percentage does not + * visibly reset at each module boundary. Counts up as keys resolve (loaded, + * missing or failed all count as resolved) and resets to zero once nothing is + * in flight. + */ + private jsonEntriesProgress: { requested: number; resolved: number } = { + requested: 0, + resolved: 0, + }; private renders: Record | null; private schemas: Record | null; private serverSideSchemaSha: string | null; @@ -426,6 +438,8 @@ export class ValSyncEngine { this.loadingJsonEntries = new Map(); this.jsonEntryErrors = {}; this.staleJsonEntries = new Set(); + this.jsonEntriesProgress = { requested: 0, resolved: 0 }; + this.cachedJsonEntriesProgressSnapshot = null; this.renders = null; this.globalServerSidePatchIds = []; this.syncedServerSidePatchIds = []; @@ -506,6 +520,9 @@ export class ValSyncEngine { type: "all-validation-errors", ): (listener: () => void) => () => void; subscribe(type: "initialized-at"): (listener: () => void) => () => void; + subscribe( + type: "json-entries-progress", + ): (listener: () => void) => () => void; subscribe( type: "sync-status", path: SourcePath, @@ -594,6 +611,11 @@ export class ValSyncEngine { this.emit(this.listeners["initialized-at"]?.[globalNamespace]); } + private invalidateJsonEntriesProgress() { + this.cachedJsonEntriesProgressSnapshot = null; + this.emit(this.listeners["json-entries-progress"]?.[globalNamespace]); + } + private invalidateSource(moduleFilePath: ModuleFilePath) { if (this.cachedSourceSnapshots !== null) { const keysToRemove: string[] = []; @@ -966,6 +988,9 @@ export class ValSyncEngine { // stale while this request is in flight must win, since the response we are // about to get was produced before that invalidation. this.staleJsonEntries.delete(loadingKey); + // Counted in the same run as batch loads, so one progress indicator covers + // both an opened entry and a loading list. + this.noteJsonEntriesRequested(1); const promise = this.client("/json", "GET", { // apply_patches=false: we own in-flight client patches the server has not // seen yet and apply them ourselves in `getPatchedSource`. Letting the @@ -1013,11 +1038,316 @@ export class ValSyncEngine { }) .finally(() => { this.loadingJsonEntries.delete(loadingKey); + this.noteJsonEntriesResolved(1); }); this.loadingJsonEntries.set(loadingKey, promise); return promise; } + /** + * The committed entry keys of a `.jsonValues()` module — i.e. the keys whose + * base-source value is still a lazy `{_type:"json"}` marker. `null` when the + * module's source is not (yet) a record. + * + * Keys that exist ONLY in a pending patch are deliberately excluded: they have + * no committed content to fetch (their value is the patch's own), so asking + * `/json` for them would 404 and poison them with an error state. + */ + private committedJsonEntryKeys( + moduleFilePath: ModuleFilePath, + ): string[] | null { + const baseSource = this.serverSources?.[moduleFilePath]; + if ( + baseSource === undefined || + baseSource === null || + typeof baseSource !== "object" || + Array.isArray(baseSource) + ) { + return null; + } + return Object.keys(baseSource).filter((key) => + Internal.isJson(baseSource[key]), + ); + } + + /** + * Loads the content of an explicit set of `.jsonValues()` entries, batched. + * Fire-and-forget: this is what a virtualized list calls for the window it just + * rendered. Already-loaded, in-flight and previously-failed keys are skipped, + * so re-rendering the same window costs nothing. + */ + requestJsonEntries(moduleFilePath: ModuleFilePath, keys: string[]): void { + void this.loadJsonEntries([{ moduleFilePath, keys }]); + } + + /** + * Resolves once EVERY committed entry of the given modules is loaded (or has + * failed). This is the completeness-critical variant: the reference guards and + * search must not answer from a partially-loaded record, so they await this and + * read `complete`. + * + * Never called on Studio boot — only when something is about to need the + * content (a destructive action's reference check, a search query, or an + * explicit "validate everything"). + */ + async ensureJsonEntries(moduleFilePaths: ModuleFilePath[]): Promise<{ + /** True when every requested key resolved to content. */ + complete: boolean; + errors: { moduleFilePath: ModuleFilePath; key: string; message: string }[]; + }> { + let errors: { + moduleFilePath: ModuleFilePath; + key: string; + message: string; + }[] = []; + // Bounded passes: an invalidation that lands mid-flight marks entries stale + // again, and reporting "complete" while holding pre-invalidation content is + // exactly the lie this method exists to prevent. + for (let pass = 0; pass < 3; pass++) { + const requests = moduleFilePaths.map((moduleFilePath) => ({ + moduleFilePath, + keys: this.committedJsonEntryKeys(moduleFilePath) ?? [], + })); + errors = (await this.loadJsonEntries(requests)).errors; + const outstanding = requests.some(({ moduleFilePath, keys }) => + keys.some((key) => { + if (this.staleJsonEntries.has(`${moduleFilePath}\0${key}`)) { + return true; + } + return ( + this.jsonEntryContents[moduleFilePath]?.[key] === undefined && + this.jsonEntryErrors[moduleFilePath]?.[key] === undefined + ); + }), + ); + if (!outstanding) { + break; + } + } + return { complete: errors.length === 0, errors }; + } + + /** + * The shared core of {@link requestJsonEntries} / {@link ensureJsonEntries}: + * filters each module's requested keys down to the ones actually worth + * fetching, chunks them, and resolves when every chunk has settled. + */ + private async loadJsonEntries( + requests: { moduleFilePath: ModuleFilePath; keys: string[] }[], + ): Promise<{ + errors: { moduleFilePath: ModuleFilePath; key: string; message: string }[]; + }> { + const waits: Promise[] = []; + const requestedBaseKeys: { + moduleFilePath: ModuleFilePath; + keys: string[]; + }[] = []; + for (const { moduleFilePath, keys } of requests) { + const committed = this.committedJsonEntryKeys(moduleFilePath); + if (committed === null) { + continue; + } + const committedKeys = new Set(committed); + const baseKeys: string[] = []; + const wanted: string[] = []; + const seen = new Set(); + for (const requestedKey of keys) { + const key = this.resolveBaseJsonEntryKey(moduleFilePath, requestedKey); + if (seen.has(key) || !committedKeys.has(key)) { + continue; + } + seen.add(key); + baseKeys.push(key); + const loadingKey = `${moduleFilePath}\0${key}`; + const inFlight = this.loadingJsonEntries.get(loadingKey); + if (inFlight !== undefined) { + waits.push(inFlight); + continue; + } + const isStale = this.staleJsonEntries.has(loadingKey); + if (isStale) { + wanted.push(key); + continue; + } + if (this.jsonEntryContents[moduleFilePath]?.[key] !== undefined) { + continue; + } + // A memoized failure stops the refetch loop (see `jsonEntryErrors`); + // `retryJsonEntry` clears it. + if (this.jsonEntryErrors[moduleFilePath]?.[key] !== undefined) { + continue; + } + wanted.push(key); + } + requestedBaseKeys.push({ moduleFilePath, keys: baseKeys }); + for (let i = 0; i < wanted.length; i += JSON_ENTRIES_CHUNK_SIZE) { + waits.push( + this.loadJsonEntryChunk( + moduleFilePath, + wanted.slice(i, i + JSON_ENTRIES_CHUNK_SIZE), + ), + ); + } + } + await Promise.all(waits); + const errors: { + moduleFilePath: ModuleFilePath; + key: string; + message: string; + }[] = []; + for (const { moduleFilePath, keys } of requestedBaseKeys) { + for (const key of keys) { + const message = this.jsonEntryErrors[moduleFilePath]?.[key]; + if (message !== undefined) { + errors.push({ moduleFilePath, key, message }); + } + } + } + return { errors }; + } + + /** Loads ONE batch of entries: a single `/json` request for many keys. */ + private loadJsonEntryChunk( + moduleFilePath: ModuleFilePath, + keys: string[], + ): Promise { + const setJsonEntryError = (key: string, message: string) => { + if (this.jsonEntryErrors[moduleFilePath] === undefined) { + this.jsonEntryErrors[moduleFilePath] = {}; + } + this.jsonEntryErrors[moduleFilePath][key] = message; + }; + // Cleared at request START, not on success: anything that marks an entry + // stale while this request is in flight must win, since the response we are + // about to get was produced before that invalidation. + for (const key of keys) { + this.staleJsonEntries.delete(`${moduleFilePath}\0${key}`); + } + this.noteJsonEntriesRequested(keys.length); + const promise = this.client("/json", "GET", { + // apply_patches=false: we own in-flight client patches the server has not + // seen yet and apply them ourselves in `getPatchedSource`. Letting the + // server apply them too would double-apply. + query: { + path: moduleFilePath, + key: undefined, + keys, + offset: undefined, + limit: undefined, + apply_patches: false, + }, + }) + .then((res) => { + if (res.status === 200 && "entries" in res.json) { + if (this.jsonEntryContents[moduleFilePath] === undefined) { + this.jsonEntryContents[moduleFilePath] = {}; + } + for (const entry of res.json.entries) { + this.jsonEntryContents[moduleFilePath][entry.key] = + entry.content ?? null; + if (this.jsonEntryErrors[moduleFilePath] !== undefined) { + delete this.jsonEntryErrors[moduleFilePath][entry.key]; + } + } + // A committed key that the server cannot resolve (deleted on disk + // between our source sync and this request) is an error, not silence. + for (const key of res.json.missing) { + setJsonEntryError( + key, + `Entry not found: ${key} in ${moduleFilePath}`, + ); + } + for (const { key, message } of res.json.errors) { + setJsonEntryError(key, message); + } + } else { + const message = + "message" in res.json + ? res.json.message + : `Request failed with status ${res.status}`; + console.error("Val: SyncEngine: failed to load json entries", { + moduleFilePath, + keys, + res, + }); + for (const key of keys) { + setJsonEntryError(key, message); + } + } + }) + .catch((err) => { + console.error("Val: SyncEngine: error loading json entries", { + moduleFilePath, + keys, + err, + }); + const message = err instanceof Error ? err.message : String(err); + for (const key of keys) { + setJsonEntryError(key, message); + } + }) + .finally(() => { + for (const key of keys) { + this.loadingJsonEntries.delete(`${moduleFilePath}\0${key}`); + } + // ONE invalidation for the whole batch, not one per entry. + this.invalidateSource(moduleFilePath); + // After the in-flight deletes above, so "nothing left in flight" is + // accurate and the run can reset. + this.noteJsonEntriesResolved(keys.length); + }); + for (const key of keys) { + this.loadingJsonEntries.set(`${moduleFilePath}\0${key}`, promise); + } + return promise; + } + + private noteJsonEntriesRequested(count: number): void { + if (count <= 0) { + return; + } + this.jsonEntriesProgress = { + requested: this.jsonEntriesProgress.requested + count, + resolved: this.jsonEntriesProgress.resolved, + }; + this.invalidateJsonEntriesProgress(); + } + + private noteJsonEntriesResolved(count: number): void { + if (count <= 0) { + return; + } + const { requested, resolved } = this.jsonEntriesProgress; + this.jsonEntriesProgress = + this.loadingJsonEntries.size === 0 + ? // The run is over — reset so the next one starts from 0%. + { requested: 0, resolved: 0 } + : { requested, resolved: Math.min(resolved + count, requested) }; + this.invalidateJsonEntriesProgress(); + } + + private cachedJsonEntriesProgressSnapshot: JsonEntriesProgress | null = null; + /** + * Progress of the current json-entry load run, for UI that must show it rather + * than pretend it has a complete answer. Spans every module in flight, so the + * percentage does not reset at module boundaries. + */ + getJsonEntriesProgressSnapshot(): JsonEntriesProgress { + if (this.cachedJsonEntriesProgressSnapshot === null) { + const { requested, resolved } = this.jsonEntriesProgress; + this.cachedJsonEntriesProgressSnapshot = { + status: requested === 0 ? "idle" : "loading", + loaded: resolved, + total: requested, + // 100 while idle: check `status` first if "nothing requested yet" and + // "everything done" must look different. + percentage: + requested === 0 ? 100 : Math.round((resolved / requested) * 100), + }; + } + return this.cachedJsonEntriesProgressSnapshot; + } + /** * The error message of the last failed load of a `.jsonValues()` entry, or * `null`. Returns a primitive so it is safe to read from a @@ -1052,10 +1382,13 @@ export class ValSyncEngine { if (contents === undefined) { return; } - for (const key of Object.keys(contents)) { + const keys = Object.keys(contents); + for (const key of keys) { this.staleJsonEntries.add(`${moduleFilePath}\0${key}`); - this.requestJsonEntry(moduleFilePath, key); } + // Batched: with hundreds of entries cached, refetching one-by-one made a + // publish a request storm. + this.requestJsonEntries(moduleFilePath, keys); } /** @@ -3567,6 +3900,23 @@ export class ValSyncEngine { // #region Supporting code const ops = new JSONOps(); const globalNamespace = "global"; +/** + * How many `.jsonValues()` entries one `/json` request asks for. Smaller than + * the server's hard cap so a chunk stays a modest URL and lands incrementally + * (each landed chunk re-renders the rows it covers). + */ +const JSON_ENTRIES_CHUNK_SIZE = Math.min(50, JSON_ENTRIES_BATCH_MAX); + +/** Progress of the current `.jsonValues()` entry load run. */ +export type JsonEntriesProgress = { + status: "idle" | "loading"; + /** Keys resolved so far — loaded, missing or failed all count as resolved. */ + loaded: number; + /** Keys requested in this run; 0 when idle. */ + total: number; + /** 0-100, and 100 when idle. */ + percentage: number; +}; /** * These are types where we can be 100% certain that a change in this type, will not result in validations failing in some other module. * We use this to determine if syncing 1 module is enough or if we need to sync all modules. @@ -3644,6 +3994,7 @@ type SyncEngineListenerType = | "schema-out-of-date" | "local-modules-status" | "pending-ops-count" + | "json-entries-progress" | "all-sources" | "all-renders" | "render" From 53482523ad3a0c9559101fdc24ee37420c5a53b4 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Fri, 31 Jul 2026 12:53:02 +0200 Subject: [PATCH 34/35] jsonValues: virtualize record lists, and stop the N-request storm (Phase 6 step 3) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Opening a `.jsonValues()` record fired one /json request PER ENTRY. Each row renders a of its entry, `useSchemaAtPathInternal` fires requestJsonEntry for any un-loaded marker it walks into, and nothing coalesced them — so a 5k-entry router meant 5k requests, and the visible symptom (a wall of spinners) hid the actual cause. Engine: - requestJsonEntry/requestJsonEntries now queue into pendingJsonEntryRequests and flush on a microtask, so one render pass costs ONE request per module. - The duplicated single-entry fetch path is deleted: ensureJsonEntry delegates to the same loadJsonEntriesSettled as everything else, so there is one fetch path. (The Studio no longer uses /json?key= at all; only the RSC runtime.) - loadJsonEntries now reports the keys it took responsibility for, so the settle loop can distinguish "resolved" from "deliberately skipped". - Keys absent from the committed source are only skipped when the PATCHED source has them (draft-added, content comes from the patch). A key in neither is still requested, so a genuine miss surfaces as an error rather than silence. UI: - New VirtualizedRecordList wraps both branches of RecordFields (default cards and the .render() list). It requests only the rendered window, and depends on the window's CONTENT rather than the array identity — Object.keys returns a fresh array every render, which would otherwise re-fire the effect on every render of a long list. - Records at or below VIRTUALIZE_THRESHOLD (50) render plainly as before: a nested scroll container is a real UX change and is not worth imposing on the ordinary small record. The threshold also bounds the un-virtualized load to one or two batches. - RecordRowSkeleton replaces the preview while an entry is un-loaded, at a fixed height so the virtualizer's measurements do not jump as content lands. useUnloadedJsonEntryKeys computes that set once per list, not per row. Tracker: steps 1-3 marked done, plus a "Review findings" task list from a self-review — two issues were found and fixed in the pass (a missing React key in the un-virtualized branch; the re-firing effect above) and seven remain, the main one being that there are no component tests because the jest preset has no jsdom. Co-Authored-By: Claude Opus 5 --- docs/plans/jsonValues.md | 99 +++++-- packages/ui/spa/ValSyncEngine.test.ts | 22 ++ packages/ui/spa/ValSyncEngine.ts | 242 +++++++++--------- .../ui/spa/components/fields/RecordFields.tsx | 188 +++++++++++--- .../fields/VirtualizedRecordList.tsx | 166 ++++++++++++ 5 files changed, 533 insertions(+), 184 deletions(-) create mode 100644 packages/ui/spa/components/fields/VirtualizedRecordList.tsx diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md index 18ff95448..201e8b348 100644 --- a/docs/plans/jsonValues.md +++ b/docs/plans/jsonValues.md @@ -10,6 +10,18 @@ ## Current state / resume here +> **Phase 6 step 3 DONE (2026-07-31): virtualized list + the real bug it exposed.** Reviewing the list +> view turned up something worse than "N broken previews": every row's `` resolves its own path, +> and `useSchemaAtPathInternal` fires `requestJsonEntry` for any un-loaded marker it walks into — so +> opening a jsonValues record with N entries fired **N `/json` requests**. Fixed at the engine level by +> coalescing requests onto a microtask (one request per module per render pass) and deleting the +> single-entry fetch path entirely, so `ensureJsonEntry` shares `loadJsonEntriesSettled`. On top of that, +> `VirtualizedRecordList` virtualizes both list branches above 50 keys, requests only the rendered +> window, and swaps un-loaded rows for a fixed-height skeleton. +1 test (29 in the jsonValues describe). +> Self-review findings are recorded as tasks at the end of Phase 6 — two were fixed in the pass, seven +> remain (chiefly: no component tests, because the jest preset has no jsdom). +> **Next: step 4 — `jsonValuesLoadRequirements` predicate + tests.** + > **Phase 6 step 2 DONE (2026-07-31): engine primitives.** `requestJsonEntries(mfp, keys)` > (fire-and-forget window) and `ensureJsonEntries(mfps)` (awaitable, whole-module, returns > `{complete, errors}` so a guard can tell "loaded" from "failed") both delegate to one private @@ -456,16 +468,13 @@ The list view comes early — it is the most user-visible breakage (a jsonValues previews today) and it exercises the batch path under real scroll load before anything subtle depends on it. Steps 1–2 are the only hard prerequisites; 4–6 are independent of each other after that. -1. **Batch transport** — `ValOps.getJsonEntries` (hoist `initSources`/`fetchPatches`) + `/json` batch - mode + `ApiRoutes` zod, with server tests. Everything below depends on this. -2. **Engine primitives** — `requestJsonEntries(mfp, keys)` (window, fire-and-forget) + - `ensureJsonEntries(modules)` (whole-module, awaitable) + progress store, and fold - `markAllJsonEntriesStale` into the batch path. Extend the `jsonValues` describe in - `ValSyncEngine.test.ts`. -3. **List view** — virtualize `RecordFields` (both branches) with `@tanstack/react-virtual`, load the - rendered window via `requestJsonEntries`, **skeletons for un-loaded rows on all three preview paths** - (a marker must never reach a preview component). → **V16**. First visible win; also the first real - load-test of steps 1–2. +1. ✅ **Batch transport** (2026-07-31) — `ValOps.getJsonEntries` + `/json` batch mode + `ApiRoutes` zod. +2. ✅ **Engine primitives** (2026-07-31) — `requestJsonEntries` + `ensureJsonEntries` + progress store, + `markAllJsonEntriesStale` batched. +3. ✅ **List view** (2026-07-31) — virtualized `RecordFields` (both branches) with + `@tanstack/react-virtual`, loads the rendered window via `requestJsonEntries`, skeletons for un-loaded + rows. Also fixed the N-requests-on-open storm it exposed, by coalescing at the engine level. → **V16** + still to run manually. 4. **Predicate** — `jsonValuesLoadRequirements` + its unit tests (pure; could be done any time, but it only pays off once the hooks below use it). 5. **Ref hooks + popover gating** — hooks return a status, destructive popovers gate on `success`. @@ -524,19 +533,29 @@ not a step. exist only in a pending patch** — they have no committed content, so requesting them would 404 and wrongly mark the row errored (their value comes from the patch, via `getPatchedSource`). One `invalidateSource` per batch rather than per entry. -- [ ] **Virtualize the record list + load visible rows only** — `RecordFields.tsx`: wrap both list - branches (default grid at :162 and `ListRecordRenderComponent` at :191) in - `useVirtualizer` from `@tanstack/react-virtual`, and for a jsonValues record call - `requestJsonEntries(mfp, visibleKeys)` from the rendered window (debounced; drop stale windows). - Un-loaded rows render a skeleton, not an error. Non-jsonValues records get virtualization too - (same code path, no loading) — a 10k-key ordinary record renders 10k previews today. -- [ ] **Skeletons for un-loaded jsonValues rows (all three preview paths)** — a marker must never reach - a preview component. `PreviewWithRender` → `useRefPreview` miss → `` → - `ObjectPreview`/etc. reading a marker is exactly today's broken state. So: when the parent schema is - a jsonValues record and the entry's content is not in `jsonEntryContents`, render a skeleton - (a) instead of the `` fallback, (b) instead of a `ListPreviewItem` when the key is missing - from a partial `items` array, and (c) for the `errored` case a small retry affordance rather than a - dead row. Skeleton must be the same height as a loaded row or the virtualizer's measurements jump. +- [x] **Coalesce single-entry requests** — DONE (2026-07-31), and it turned out to be the actual bug. + Each row's `` resolves its own path, and `useSchemaAtPathInternal` fires + `requestJsonEntry` for any un-loaded marker it walks into — so opening a jsonValues record with N + entries fired **N `/json` requests** (the symptom is a wall of spinners, but the cause is a request + storm, not a rendering bug). `requestJsonEntry`/`requestJsonEntries` now queue into + `pendingJsonEntryRequests` and flush on a microtask, so one render pass costs ONE request per + module. The single-entry fetch path was deleted: `ensureJsonEntry` now delegates to the same + `loadJsonEntriesSettled`, so there is one fetch path (and the Studio no longer uses `/json?key=` + at all — only the RSC runtime does). +- [x] **Virtualize the record list + load visible rows only** — DONE (2026-07-31). New + `VirtualizedRecordList` wraps both branches of `RecordFields` (default cards + the + `.render()` list), requests the rendered window via `requestJsonEntries`, and depends on the + window's CONTENT (not the array identity — `Object.keys` returns a fresh array each render, which + would re-fire the effect on every render of a long list). Records at or below + `VIRTUALIZE_THRESHOLD = 50` keys render plainly as before — a nested scroll container is a real UX + change and is not worth imposing on the ordinary small record — and that threshold also bounds the + un-virtualized load to one or two batches. Non-jsonValues records virtualize through the same path. +- [x] **Skeletons for un-loaded jsonValues rows** — DONE (2026-07-31). `RecordRowSkeleton` (fixed + height, so the virtualizer's measurements do not jump as content lands) replaces the preview when + the key's value in the patched source is still a marker; `useUnloadedJsonEntryKeys` computes that + set ONCE per list rather than subscribing per row. + **Still open**: (c) from the original item — a per-row retry affordance for the `errored` case. + A failed row currently falls through to ``, which renders the existing error state. - [ ] **`.render()` list layouts for jsonValues records (windowed)** — **moved to Phase 7 stage 1**; it is a client-side-instance change, not a transport one. Summary: replace the `isJsonValues` early return in `RecordSchema.executeRender` with a per-key `isJson(itemSrc) → continue`, then call @@ -607,6 +626,40 @@ not a step. jump, no marker reaching a preview component); scrolling fills them in. Then edit a visible row's title WITHOUT publishing → the row updates as you type (the render is computed from the patched source on the client). + +### Review findings (self-review of steps 1–3, 2026-07-31) + +Two issues were found and fixed during the pass (a missing React `key` in the un-virtualized branch, +and an effect that re-fired on every render because `Object.keys` returns a fresh array). What is left: + +- [ ] **No component tests for `VirtualizedRecordList`** — the jest preset is `testEnvironment: "node"` + and `jest-environment-jsdom` is not installed, so nothing in the repo renders React. The + virtualization, the window→`requestJsonEntries` wiring and the skeleton swap are therefore covered + only by V16 (manual). Either add a jsdom project to the preset (`@testing-library/react` IS already + a devDependency of `packages/ui`) or accept manual-only and say so. +- [ ] **Nested scroll container needs a real look (V16)** — the virtualized branch introduces an + `overflow-auto` viewport capped at `VIEWPORT_MAX_HEIGHT = 800`, inside the Studio's own scroll + container. Two things to check on a real screen: that scroll chaining feels right (the inner + scroller should not trap the page), and that 800px is not taller than a small viewport. A + `max-h-[70vh]`-style cap may be better than a fixed pixel height. +- [ ] **Row-height estimates are guesses** — `CARD_ROW_HEIGHT = 186` / `RENDER_ROW_HEIGHT = 104` were + derived from the card's `max-h-[170px]` + padding, not measured. `measureElement` corrects the real + heights, so the estimate only affects the initial scrollbar and the first window's size; still + worth checking against V16 so the first window is not visibly wrong. +- [ ] **Skeleton has no retry affordance** — item (c) of the original skeleton task. A row whose entry + FAILED to load falls through to `` and renders the existing error state; there is no + per-row retry button calling `retryJsonEntry`. +- [ ] **`loadJsonEntriesSettled`'s 3-pass bound is arbitrary** — it is a backstop against an + invalidation loop, not a computed limit. If a pathological case ever exhausts it we return + `complete: false` with no errors, which reads as "incomplete for an unknown reason". Consider + logging when the bound is hit so it is diagnosable rather than mysterious. +- [ ] **The un-virtualized path requests up to 50 keys on mount** — for a jsonValues record of 50 + entries, opening it loads all 50 (one batch). That is a deliberate trade (see + `VIRTUALIZE_THRESHOLD`) but it does mean "zero requests on open" is never true for small + jsonValues records. Revisit if a 50-entry record's previews prove expensive to load. +- [ ] **Progress `percentage` is 100 when idle** — so a consumer that renders it without checking + `status` first shows "100%" before anything starts. Documented on the type; a `null` percentage + while idle would be harder to misuse. - [ ] **DECISION NEEDED — ask Fredrik: efficient browser caching for `/json`.** Deferred from the Phase 6 design round (2026-07-30). Fredrik's idea: in **http/prod mode there is a stable commit**, so put it in the request and make the response immutably cacheable by the browser. Open parts: diff --git a/packages/ui/spa/ValSyncEngine.test.ts b/packages/ui/spa/ValSyncEngine.test.ts index c3db24aa1..1cb5ae3e7 100644 --- a/packages/ui/spa/ValSyncEngine.test.ts +++ b/packages/ui/spa/ValSyncEngine.test.ts @@ -717,6 +717,7 @@ describe("ValSyncEngine", () => { const engine = await tester.createInitializedSyncEngine(); await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/a"); + const batchesAfterFirstLoad = tester.jsonBatchRequestCounts[PAGES] ?? 0; // /a is cached; only /b should go out. engine.requestJsonEntries(toModuleFilePath(PAGES), ["/a", "/b"]); // Re-rendering the same window while it is in flight must add nothing. @@ -725,7 +726,28 @@ describe("ValSyncEngine", () => { expect(tester.jsonRequestCounts[`${PAGES}\0/a`]).toBe(1); expect(tester.jsonRequestCounts[`${PAGES}\0/b`]).toBe(1); + // Two window calls, ONE additional request. + expect(tester.jsonBatchRequestCounts[PAGES]).toBe( + batchesAfterFirstLoad + 1, + ); + }); + + test("single-entry requests in the same tick collapse into ONE request", async () => { + // Regression: a record list renders a per key and each one + // triggers requestJsonEntry for its own entry, so opening a record with N + // entries fired N requests. They must coalesce. + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + engine.requestJsonEntry(toModuleFilePath(PAGES), "/a"); + engine.requestJsonEntry(toModuleFilePath(PAGES), "/b"); + await flush(); + expect(tester.jsonBatchRequestCounts[PAGES]).toBe(1); + const source = engine.getSourceSnapshot(toModuleFilePath(PAGES)) + .data as any; + expect(source["/a"]).toEqual({ title: "A", order: 1 }); + expect(source["/b"]).toEqual({ title: "B", order: 2 }); }); test("keys that exist only in a pending patch are never requested", async () => { diff --git a/packages/ui/spa/ValSyncEngine.ts b/packages/ui/spa/ValSyncEngine.ts index efc22df2b..8d40a4d71 100644 --- a/packages/ui/spa/ValSyncEngine.ts +++ b/packages/ui/spa/ValSyncEngine.ts @@ -177,6 +177,15 @@ export class ValSyncEngine { requested: 0, resolved: 0, }; + /** + * Keys queued by {@link requestJsonEntry} / {@link requestJsonEntries} for the + * next coalesced flush. A record list renders one `` per key and each + * asks for its own entry, so without this a record with N entries fires N + * requests; with it, one render pass costs one request per module. + */ + private pendingJsonEntryRequests: Map> = + new Map(); + private jsonEntryFlushScheduled = false; private renders: Record | null; private schemas: Record | null; private serverSideSchemaSha: string | null; @@ -440,6 +449,10 @@ export class ValSyncEngine { this.staleJsonEntries = new Set(); this.jsonEntriesProgress = { requested: 0, resolved: 0 }; this.cachedJsonEntriesProgressSnapshot = null; + this.pendingJsonEntryRequests = new Map(); + // Deliberately NOT clearing jsonEntryFlushScheduled: a flush may already be + // queued, and it must find an empty pending map rather than a second flush + // being scheduled behind it. this.renders = null; this.globalServerSidePatchIds = []; this.syncedServerSidePatchIds = []; @@ -879,13 +892,52 @@ export class ValSyncEngine { * server-side reorder, and any other non-append change. */ /** - * Lazily loads the content of a single `.jsonValues()` entry (GET /json) and - * folds it into the source view (re-rendering subscribers). No-op if the entry - * is already loaded or a load is in flight. Called by the field hooks when the - * Studio opens a path that descends into an unloaded json marker. + * Lazily loads the content of a single `.jsonValues()` entry and folds it into + * the source view (re-rendering subscribers). No-op if the entry is already + * loaded or a load is in flight. Called by the field hooks when the Studio + * renders a path that descends into an un-loaded json marker. + * + * COALESCED: every call in the same tick becomes ONE `/json` request per + * module. That matters because a record list renders a `` per key and + * each one asks for its own entry — before coalescing, opening a record with + * N entries fired N requests. */ requestJsonEntry(moduleFilePath: ModuleFilePath, key: string): void { - void this.ensureJsonEntry(moduleFilePath, key); + this.coalesceJsonEntryRequests(moduleFilePath, [key]); + } + + /** + * Queues keys for the next coalesced flush. The flush is a microtask, so all + * the field effects of one render pass land in a single request per module. + */ + private coalesceJsonEntryRequests( + moduleFilePath: ModuleFilePath, + keys: string[], + ): void { + let pending = this.pendingJsonEntryRequests.get(moduleFilePath); + if (pending === undefined) { + pending = new Set(); + this.pendingJsonEntryRequests.set(moduleFilePath, pending); + } + for (const key of keys) { + pending.add(key); + } + if (this.jsonEntryFlushScheduled) { + return; + } + this.jsonEntryFlushScheduled = true; + queueMicrotask(() => { + this.jsonEntryFlushScheduled = false; + const requests = Array.from( + this.pendingJsonEntryRequests, + ([path, requestedKeys]) => ({ + moduleFilePath: path, + keys: Array.from(requestedKeys), + }), + ); + this.pendingJsonEntryRequests.clear(); + void this.loadJsonEntriesSettled(requests); + }); } /** @@ -948,100 +1000,13 @@ export class ValSyncEngine { * than an opaque marker. {@link requestJsonEntry} is the fire-and-forget * variant used by the field hooks. */ - ensureJsonEntry( + async ensureJsonEntry( moduleFilePath: ModuleFilePath, requestedKey: string, ): Promise { - const key = this.resolveBaseJsonEntryKey(moduleFilePath, requestedKey); - const loadingKey = `${moduleFilePath}\0${key}`; - const inFlight = this.loadingJsonEntries.get(loadingKey); - if (inFlight !== undefined) { - // The stale flag is cleared when a request STARTS, so if it is set again - // now, this in-flight response predates whatever invalidated the entry — - // wait for it, then load again. - return inFlight.then(() => - this.staleJsonEntries.has(loadingKey) - ? this.ensureJsonEntry(moduleFilePath, key) - : undefined, - ); - } - const isStale = this.staleJsonEntries.has(loadingKey); - if ( - !isStale && - this.jsonEntryContents[moduleFilePath]?.[key] !== undefined - ) { - return Promise.resolve(); - } - // A memoized failure stops the refetch-on-every-remount loop. Cleared by - // `retryJsonEntry` or by the module's server source being replaced. - if (this.jsonEntryErrors[moduleFilePath]?.[key] !== undefined) { - return Promise.resolve(); - } - const setJsonEntryError = (message: string) => { - if (this.jsonEntryErrors[moduleFilePath] === undefined) { - this.jsonEntryErrors[moduleFilePath] = {}; - } - this.jsonEntryErrors[moduleFilePath][key] = message; - this.invalidateSource(moduleFilePath); - }; - // Cleared at request START, not on success: anything that marks the entry - // stale while this request is in flight must win, since the response we are - // about to get was produced before that invalidation. - this.staleJsonEntries.delete(loadingKey); - // Counted in the same run as batch loads, so one progress indicator covers - // both an opened entry and a loading list. - this.noteJsonEntriesRequested(1); - const promise = this.client("/json", "GET", { - // apply_patches=false: we own in-flight client patches the server has not - // seen yet and apply them ourselves in `getPatchedSource`. Letting the - // server apply them too would double-apply. - query: { - path: moduleFilePath, - key, - keys: undefined, // single-entry shape - offset: undefined, - limit: undefined, - apply_patches: false, - }, - }) - .then((res) => { - if (res.status === 200 && "content" in res.json) { - if (this.jsonEntryContents[moduleFilePath] === undefined) { - this.jsonEntryContents[moduleFilePath] = {}; - } - this.jsonEntryContents[moduleFilePath][key] = - res.json.content ?? null; - if (this.jsonEntryErrors[moduleFilePath] !== undefined) { - delete this.jsonEntryErrors[moduleFilePath][key]; - } - this.invalidateSource(moduleFilePath); - } else { - console.error("Val: SyncEngine: failed to load json entry", { - moduleFilePath, - key, - res, - }); - setJsonEntryError( - "message" in res.json - ? res.json.message - : `Request failed with status ${res.status}`, - ); - } - }) - .catch((err) => { - console.error("Val: SyncEngine: error loading json entry", { - moduleFilePath, - key, - err, - }); - setJsonEntryError(err instanceof Error ? err.message : String(err)); - }) - .finally(() => { - this.loadingJsonEntries.delete(loadingKey); - this.noteJsonEntriesResolved(1); - }); - this.loadingJsonEntries.set(loadingKey, promise); - return promise; + await this.loadJsonEntriesSettled([ + { moduleFilePath, keys: [requestedKey] }, + ]); } /** @@ -1077,7 +1042,7 @@ export class ValSyncEngine { * so re-rendering the same window costs nothing. */ requestJsonEntries(moduleFilePath: ModuleFilePath, keys: string[]): void { - void this.loadJsonEntries([{ moduleFilePath, keys }]); + this.coalesceJsonEntryRequests(moduleFilePath, keys); } /** @@ -1094,31 +1059,50 @@ export class ValSyncEngine { /** True when every requested key resolved to content. */ complete: boolean; errors: { moduleFilePath: ModuleFilePath; key: string; message: string }[]; + }> { + return this.loadJsonEntriesSettled( + moduleFilePaths.map((moduleFilePath) => ({ + moduleFilePath, + keys: this.committedJsonEntryKeys(moduleFilePath) ?? [], + })), + ); + } + + /** + * Loads `requests` and re-passes while anything it asked for is still + * outstanding, then reports whether everything resolved to content. + * + * The re-pass exists because an invalidation that lands mid-flight marks + * entries stale again: the response we were waiting for predates it, so + * reporting `complete` on the strength of it would be exactly the lie this + * method exists to prevent. Bounded, so a pathological invalidation loop + * cannot spin here forever. + */ + private async loadJsonEntriesSettled( + requests: { moduleFilePath: ModuleFilePath; keys: string[] }[], + ): Promise<{ + complete: boolean; + errors: { moduleFilePath: ModuleFilePath; key: string; message: string }[]; }> { let errors: { moduleFilePath: ModuleFilePath; key: string; message: string; }[] = []; - // Bounded passes: an invalidation that lands mid-flight marks entries stale - // again, and reporting "complete" while holding pre-invalidation content is - // exactly the lie this method exists to prevent. for (let pass = 0; pass < 3; pass++) { - const requests = moduleFilePaths.map((moduleFilePath) => ({ - moduleFilePath, - keys: this.committedJsonEntryKeys(moduleFilePath) ?? [], - })); - errors = (await this.loadJsonEntries(requests)).errors; - const outstanding = requests.some(({ moduleFilePath, keys }) => - keys.some((key) => { - if (this.staleJsonEntries.has(`${moduleFilePath}\0${key}`)) { - return true; - } - return ( - this.jsonEntryContents[moduleFilePath]?.[key] === undefined && - this.jsonEntryErrors[moduleFilePath]?.[key] === undefined - ); - }), + const res = await this.loadJsonEntries(requests); + errors = res.errors; + const outstanding = res.requestedBaseKeys.some( + ({ moduleFilePath, keys }) => + keys.some((key) => { + if (this.staleJsonEntries.has(`${moduleFilePath}\0${key}`)) { + return true; + } + return ( + this.jsonEntryContents[moduleFilePath]?.[key] === undefined && + this.jsonEntryErrors[moduleFilePath]?.[key] === undefined + ); + }), ); if (!outstanding) { break; @@ -1128,14 +1112,16 @@ export class ValSyncEngine { } /** - * The shared core of {@link requestJsonEntries} / {@link ensureJsonEntries}: - * filters each module's requested keys down to the ones actually worth - * fetching, chunks them, and resolves when every chunk has settled. + * ONE load pass: filters each module's requested keys down to the ones worth + * fetching, chunks them, and resolves when every chunk has settled. Reports the + * keys it actually took responsibility for, so the caller can tell "resolved" + * from "deliberately skipped". */ private async loadJsonEntries( requests: { moduleFilePath: ModuleFilePath; keys: string[] }[], ): Promise<{ errors: { moduleFilePath: ModuleFilePath; key: string; message: string }[]; + requestedBaseKeys: { moduleFilePath: ModuleFilePath; keys: string[] }[]; }> { const waits: Promise[] = []; const requestedBaseKeys: { @@ -1148,12 +1134,28 @@ export class ValSyncEngine { continue; } const committedKeys = new Set(committed); + const patchedSource = this.getPatchedSource(moduleFilePath); + const draftedKeys = + patchedSource !== undefined && + patchedSource !== null && + typeof patchedSource === "object" && + !Array.isArray(patchedSource) + ? new Set(Object.keys(patchedSource)) + : new Set(); const baseKeys: string[] = []; const wanted: string[] = []; const seen = new Set(); for (const requestedKey of keys) { const key = this.resolveBaseJsonEntryKey(moduleFilePath, requestedKey); - if (seen.has(key) || !committedKeys.has(key)) { + if (seen.has(key)) { + continue; + } + // A key the committed source does not have, but the PATCHED source does, + // exists only in a pending patch: its value comes from that patch, so + // there is nothing to fetch and asking would 404 and wrongly mark it + // errored. A key in neither is a genuine miss and IS requested, so the + // caller gets a real error instead of silence. + if (!committedKeys.has(key) && draftedKeys.has(key)) { continue; } seen.add(key); @@ -1203,7 +1205,7 @@ export class ValSyncEngine { } } } - return { errors }; + return { errors, requestedBaseKeys }; } /** Loads ONE batch of entries: a single `/json` request for many keys. */ diff --git a/packages/ui/spa/components/fields/RecordFields.tsx b/packages/ui/spa/components/fields/RecordFields.tsx index 00392d9b1..e7d27ecf5 100644 --- a/packages/ui/spa/components/fields/RecordFields.tsx +++ b/packages/ui/spa/components/fields/RecordFields.tsx @@ -1,12 +1,20 @@ import { + Internal, ListRecordRender as ListRecordRender, + ModuleFilePath, SourcePath, } from "@valbuild/core"; +import { useMemo } from "react"; import { useRenderOverrideAtPath, useSchemaAtPath, useShallowSourceAtPath, + useSourceAtPath, } from "../ValFieldProvider"; +import { + RecordRowSkeleton, + VirtualizedRecordList, +} from "./VirtualizedRecordList"; import { ModuleGallery } from "./ModuleGallery"; import { useAllValidationErrors } from "../ValErrorProvider"; import { sourcePathOfItem } from "../../utils/sourcePathOfItem"; @@ -19,6 +27,7 @@ import { useNavigation } from "../../components/ValRouter"; import { PreviewLoading, PreviewNull } from "../../components/Preview"; import { PreviewWithRender } from "../../components/PreviewWithRender"; import { ValidationErrors } from "../../components/ValidationError"; +import type { ValidationError } from "@valbuild/core"; import { isParentError } from "../../utils/isParentError"; import { ErrorIndicator } from "../ErrorIndicator"; import classNames from "classnames"; @@ -41,7 +50,6 @@ export function RecordFields({ }) { const type = "record"; const validationErrors = useAllValidationErrors() || {}; - const { navigate } = useNavigation(); const schemaAtPath = useSchemaAtPath(path); const renderAtPath = useRenderOverrideAtPath(path); const sourceAtPath = useShallowSourceAtPath(path, type); @@ -137,61 +145,159 @@ export function RecordFields({ )} {renderListAtPathData && ( - + )} - {!renderListAtPathData && ( -
- {source && - Object.entries(source).map(([key]) => ( -
navigate(sourcePathOfItem(path, key))} - className={classNames( - "bg-primary-foreground cursor-pointer min-w-[320px] max-h-[170px] overflow-hidden rounded-md border border-border-primary p-4", - "hover:bg-bg-secondary-hover", - )} - > -
-
{key}
- {isParentError( - sourcePathOfItem(path, key), - validationErrors, - ) && } -
-
- -
-
- ))} -
+ {!renderListAtPathData && source && ( + )} ); } +/** Row height estimate for the default card layout (`max-h-[170px]` + gap). */ +const CARD_ROW_HEIGHT = 186; +/** Row height estimate for a `.render({layout:"list"})` row. */ +const RENDER_ROW_HEIGHT = 104; + +function RecordCardList({ + path, + keys, + jsonValues, + validationErrors, +}: { + path: SourcePath; + keys: string[]; + jsonValues: boolean; + validationErrors: Record; +}) { + const { navigate } = useNavigation(); + const [moduleFilePath] = Internal.splitModuleFilePathAndModulePath(path); + const unloadedKeys = useUnloadedJsonEntryKeys(moduleFilePath, jsonValues); + return ( + ( +
+
navigate(sourcePathOfItem(path, key))} + className={classNames( + "bg-primary-foreground cursor-pointer min-w-[320px] max-h-[170px] overflow-hidden rounded-md border border-border-primary p-4", + "hover:bg-bg-secondary-hover", + )} + > +
+
{key}
+ {isParentError(sourcePathOfItem(path, key), validationErrors) && ( + + )} +
+
+ {unloadedKeys.has(key) ? ( + // An un-loaded `.jsonValues()` entry: a preview here would read + // the opaque marker, which is what made these lists a wall of + // spinners. + + ) : ( + + )} +
+
+
+ )} + /> + ); +} + +/** + * The record keys of a `.jsonValues()` module whose entry content has not been + * loaded yet — i.e. whose value in the patched source is still a lazy marker. + * + * Computed once for the whole list rather than per row: one source subscription + * instead of one per visible row. + */ +function useUnloadedJsonEntryKeys( + moduleFilePath: ModuleFilePath, + jsonValues: boolean, +): ReadonlySet { + const moduleSource = useSourceAtPath(moduleFilePath); + const data = "data" in moduleSource ? moduleSource.data : undefined; + return useMemo(() => { + const unloaded = new Set(); + if ( + !jsonValues || + data === undefined || + data === null || + typeof data !== "object" || + Array.isArray(data) + ) { + return unloaded; + } + for (const [key, value] of Object.entries(data)) { + if (Internal.isJson(value)) { + unloaded.add(key); + } + } + return unloaded; + }, [jsonValues, data]); +} + function ListRecordRenderComponent({ path, items, + jsonValues, }: { path: SourcePath; items: ListRecordRender["items"]; + jsonValues: boolean; }) { const { navigate } = useNavigation(); + const [moduleFilePath] = Internal.splitModuleFilePathAndModulePath(path); + const unloadedKeys = useUnloadedJsonEntryKeys(moduleFilePath, jsonValues); + const keys = useMemo(() => items.map(([key]) => key), [items]); return ( -
- {items.map(([key]) => ( - - ))} -
+ ( +
+ +
+ )} + /> ); } diff --git a/packages/ui/spa/components/fields/VirtualizedRecordList.tsx b/packages/ui/spa/components/fields/VirtualizedRecordList.tsx new file mode 100644 index 000000000..0bb9e3c30 --- /dev/null +++ b/packages/ui/spa/components/fields/VirtualizedRecordList.tsx @@ -0,0 +1,166 @@ +import { ModuleFilePath, SourcePath } from "@valbuild/core"; +import { useVirtualizer } from "@tanstack/react-virtual"; +import { Fragment, ReactNode, useEffect, useMemo, useRef } from "react"; +import { useSyncEngine } from "../ValFieldProvider"; + +/** + * Records below this many keys render plainly, exactly as before: a nested + * scroll container is a real UX change, and it is not worth imposing on the + * ordinary small record. Above it the list virtualizes. + * + * It also bounds the un-virtualized `.jsonValues()` load: at most this many keys + * are requested at once, which is one or two batches. + */ +const VIRTUALIZE_THRESHOLD = 50; +/** Height of the virtualized scroll viewport. */ +const VIEWPORT_MAX_HEIGHT = 800; + +/** + * Renders a record's rows, virtualizing once there are enough of them, and — for + * a `.jsonValues()` record — loading the content of ONLY the rows currently + * rendered. + * + * Why this exists: every row renders a preview of its entry, and for a + * jsonValues record that preview needs the entry's content. Rendering all rows + * therefore loaded every entry, which is exactly what `.jsonValues()` is meant + * to avoid. Virtualizing means un-rendered rows cost nothing, and the visible + * window is requested in one batch. + */ +export function VirtualizedRecordList({ + moduleFilePath, + keys, + estimatedRowHeight, + jsonValues, + className, + renderRow, +}: { + moduleFilePath: ModuleFilePath; + /** Record keys, in source order. */ + keys: string[]; + estimatedRowHeight: number; + /** True when this is a `.jsonValues()` record, whose rows must be loaded. */ + jsonValues: boolean; + className?: string; + renderRow: (key: string) => ReactNode; +}) { + const syncEngine = useSyncEngine(); + const parentRef = useRef(null); + const shouldVirtualize = keys.length > VIRTUALIZE_THRESHOLD; + + const virtualizer = useVirtualizer({ + count: shouldVirtualize ? keys.length : 0, + getScrollElement: () => parentRef.current, + estimateSize: () => estimatedRowHeight, + overscan: 8, + }); + const virtualItems = virtualizer.getVirtualItems(); + // The rendered window, as plain indices: depending on these rather than on the + // virtual-item objects keeps the effect below from re-running on every scroll + // frame that happens to render the same rows. + const firstIndex = virtualItems.length > 0 ? virtualItems[0].index : -1; + const lastIndex = + virtualItems.length > 0 ? virtualItems[virtualItems.length - 1].index : -1; + + /** The keys actually rendered — the window whose content we need. */ + const windowKeys = useMemo(() => { + if (!shouldVirtualize) { + return keys; + } + if (firstIndex < 0) { + return []; + } + return keys.slice(firstIndex, lastIndex + 1); + }, [shouldVirtualize, keys, firstIndex, lastIndex]); + + // `keys` is a fresh array on every render (Object.keys), so the effect below + // depends on the window's CONTENT, not its identity — otherwise it re-runs on + // every render of a long list. The keys themselves are read through a ref so + // they never have to be a dependency (joining and re-splitting would corrupt + // any key containing the separator). + const windowKeysId = windowKeys.join("\u0000"); + const windowKeysRef = useRef(windowKeys); + windowKeysRef.current = windowKeys; + useEffect(() => { + if (!jsonValues || windowKeysRef.current.length === 0) { + return; + } + // Coalesced by the engine into one request per module per tick, so a fast + // scroll that crosses several windows does not fan out. + syncEngine.requestJsonEntries(moduleFilePath, windowKeysRef.current); + }, [syncEngine, moduleFilePath, jsonValues, windowKeysId]); + + if (!shouldVirtualize) { + return ( +
+ {keys.map((key) => ( + {renderRow(key)} + ))} +
+ ); + } + + return ( +
+
+ {virtualItems.map((virtualRow) => { + const key = keys[virtualRow.index]; + if (key === undefined) { + return null; + } + return ( +
+ {renderRow(key)} +
+ ); + })} +
+
+ ); +} + +/** + * Placeholder for a `.jsonValues()` row whose content has not loaded yet. + * + * Deliberately NOT the row's real preview: a preview reading an un-loaded marker + * is what made jsonValues record lists render a wall of spinners. The height is + * fixed to the row estimate so the virtualizer's measurements do not jump as + * content lands. + */ +export function RecordRowSkeleton({ + path, + height, +}: { + path: SourcePath; + height: number; +}) { + return ( +
+
+
+
+ ); +} From 229a05b5f885b2cf181f2fe6db4bd95339bd2343 Mon Sep 17 00:00:00 2001 From: Fredrik Ekholdt Date: Fri, 31 Jul 2026 14:31:40 +0200 Subject: [PATCH 35/35] jsonValues: schema-only load predicate (Phase 6 step 4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds jsonValuesLoadRequirements(schemas, query): which `.jsonValues()` modules must have their entry content loaded before a reference scan can be trusted, decided from the SCHEMAS alone — no sources, no requests. Direction is what makes this cheap. A scan for references TO a module looks for referrers (keyOf / route / file fields), which live elsewhere, and the scanned record's own key set is always present (an un-loaded entry is still a marker under its key). So the only content a scan can be blind to is content that itself points OUTWARD — a jsonValues record whose item schema contains a matching referrer. In the common case the result is empty: nothing to load, and the scan is complete and correct immediately. - Candidates are only modules whose ROOT is a jsonValues record (root-only, locked decision #7); their `item` schema is walked through object/array/record/union. - keyOf and image/file name their target module, so they match exactly. `route` does not (SerializedRouteSchema carries only include/exclude patterns), so it degrades to "any jsonValues record containing a route field" — still a large cut, since most do not have one. - An unknown schema type returns TRUE, deliberately: wrongly reporting "nothing to load" is exactly how a guard starts lying. - keyOf.path is a branded SourcePath and the query carries a ModuleFilePath; they are compared as plain strings through a helper rather than asserting one brand into the other. Tests (+9): empty for the incoming-ref case, empty for a keyOf at a different module, empty for the same shape without .jsonValues(), non-empty through array / nested record / tagged union / deep nesting, self-reference, file refs matched by referencedModule (and not satisfied by a keyOf query), the route over-approximation in both directions, and several modules reported at once. Co-Authored-By: Claude Opus 5 --- docs/plans/jsonValues.md | 28 +- .../jsonValuesLoadRequirements.test.ts | 297 ++++++++++++++++++ .../components/jsonValuesLoadRequirements.ts | 133 ++++++++ 3 files changed, 451 insertions(+), 7 deletions(-) create mode 100644 packages/ui/spa/components/jsonValuesLoadRequirements.test.ts create mode 100644 packages/ui/spa/components/jsonValuesLoadRequirements.ts diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md index 201e8b348..bb44964e4 100644 --- a/docs/plans/jsonValues.md +++ b/docs/plans/jsonValues.md @@ -10,6 +10,14 @@ ## Current state / resume here +> **Phase 6 step 4 DONE (2026-07-31): the load predicate.** `jsonValuesLoadRequirements(schemas, query)` +> answers "which jsonValues modules must be loaded before a reference scan can be trusted" from the +> SCHEMAS alone — no sources, no requests. Only root `.jsonValues()` records are candidates; their item +> schema is walked through object/array/record/union; an unknown schema type answers TRUE, because +> wrongly reporting "nothing to load" is how a guard starts lying. +9 tests. +> **Next: step 5 — ref hooks return a status and the destructive popovers gate on it (V10–V13, V17). +> This is where the data-integrity hole actually closes.** + > **Phase 6 step 3 DONE (2026-07-31): virtualized list + the real bug it exposed.** Reviewing the list > view turned up something worse than "N broken previews": every row's `` resolves its own path, > and `useSchemaAtPathInternal` fires `requestJsonEntry` for any un-loaded marker it walks into — so @@ -475,8 +483,7 @@ on it. Steps 1–2 are the only hard prerequisites; 4–6 are independent of eac `@tanstack/react-virtual`, loads the rendered window via `requestJsonEntries`, skeletons for un-loaded rows. Also fixed the N-requests-on-open storm it exposed, by coalescing at the engine level. → **V16** still to run manually. -4. **Predicate** — `jsonValuesLoadRequirements` + its unit tests (pure; could be done any time, but it - only pays off once the hooks below use it). +4. ✅ **Predicate** (2026-07-31) — `jsonValuesLoadRequirements` + 9 unit tests. Pays off in step 5. 5. **Ref hooks + popover gating** — hooks return a status, destructive popovers gate on `success`. → **V10–V13, V17**. This is where the data-integrity hole actually closes. 6. **Search** — first-query trigger, debounced re-index, marker skip in `traverseSchemaSource`, delete @@ -509,11 +516,18 @@ not a step. `applyPatches:false` and errors otherwise — enumerating from the base source would silently omit draft-added keys, which is the exact bug class this phase exists to kill. Tests: +9 in `ValOpsFS.jsonValues.test.ts`. -- [ ] **`jsonValuesLoadRequirements(schemas, query)`** — new pure module in `packages/ui/spa` - implementing the predicate above. `query` is one of `{kind:"keyOf", module}` / - `{kind:"file", module}` / `{kind:"route"}`; returns the `ModuleFilePath[]` whose entries must be - loaded. Own unit test file: empty result for the incoming-ref case, non-empty for nested - `keyOf`/image/file, `route` over-approximation, transitivity through object/array/record/union. +- [x] **`jsonValuesLoadRequirements(schemas, query)`** — DONE (2026-07-31). + `components/jsonValuesLoadRequirements.ts`: pure, schemas-only, no sources and no fetching. `query` + is `{kind:"keyOf"|"file", module}` or `{kind:"route"}`; returns the `ModuleFilePath[]` whose entries + must be loaded. Only modules whose ROOT is a `.jsonValues()` record are candidates (root-only, + locked decision #7), and their `item` schema is walked through object/array/record/union. + An unknown schema type returns TRUE (conservative): wrongly reporting "nothing to load" is exactly + how a guard starts lying. `keyOf.path` (branded `SourcePath`) and the query's `ModuleFilePath` are + compared as plain strings through a helper, so no type assertion is needed. + Tests (+9): empty for the incoming-ref case, empty for a `keyOf` at a different module, empty for + the same shape WITHOUT `.jsonValues()`, non-empty through array/nested-record/tagged-union/deep + nesting, self-reference, file refs matched by `referencedModule` (and not satisfied by a `keyOf` + query), the `route` over-approximation both ways, and multiple modules reported. - [x] **`ValSyncEngine.ensureJsonEntries(moduleFilePaths)`** — DONE (2026-07-31). Awaitable, whole-module, and it returns `{complete, errors}` rather than plain `void`: a guard that gates a delete must be able to tell "loaded everything" from "some entry failed", which is the entire diff --git a/packages/ui/spa/components/jsonValuesLoadRequirements.test.ts b/packages/ui/spa/components/jsonValuesLoadRequirements.test.ts new file mode 100644 index 000000000..b5c5b9853 --- /dev/null +++ b/packages/ui/spa/components/jsonValuesLoadRequirements.test.ts @@ -0,0 +1,297 @@ +import { + Internal, + ModuleFilePath, + SerializedSchema, + Source, + SourcePath, + ValModule, + initVal, +} from "@valbuild/core"; +import { jsonValuesLoadRequirements } from "./jsonValuesLoadRequirements"; + +const { s, c } = initVal(); + +const AUTHORS = "/authors.val.ts" as ModuleFilePath; +const PAGES = "/pages.val.ts" as ModuleFilePath; +const GALLERY = "/gallery.val.ts" as ModuleFilePath; +// The same string, branded as a SourcePath: that is what a `keyOf` schema stores +// when it points at a module-level record. +const PAGES_AS_SOURCE_PATH = "/pages.val.ts" as SourcePath; + +describe("jsonValuesLoadRequirements", () => { + test("nothing to load when the referrers live in ORDINARY modules", () => { + // The incoming-ref case: renaming a key of `authors` needs the referrers, + // which are keyOf fields in a normal module whose source is fully loaded. + // `pages` is jsonValues but points at nothing, so none of its entries matter. + const authors = c.define( + AUTHORS, + s.record(s.object({ name: s.string() })), + { ["ada"]: { name: "Ada" } }, + ); + const schemas = getSchemas([ + authors, + c.define("/featured.val.ts", s.keyOf(authors), "ada"), + c.define( + PAGES, + s.record(s.object({ title: s.string() })).jsonValues(), + {}, + ), + ]); + + expect( + jsonValuesLoadRequirements(schemas, { kind: "keyOf", module: AUTHORS }), + ).toEqual([]); + }); + + test("a jsonValues record whose ITEM holds a keyOf must be loaded", () => { + const authors = c.define( + AUTHORS, + s.record(s.object({ name: s.string() })), + { ["ada"]: { name: "Ada" } }, + ); + const schemas = getSchemas([ + authors, + c.define( + PAGES, + s + .record(s.object({ title: s.string(), author: s.keyOf(authors) })) + .jsonValues(), + {}, + ), + ]); + + expect( + jsonValuesLoadRequirements(schemas, { kind: "keyOf", module: AUTHORS }), + ).toEqual([PAGES]); + }); + + test("a keyOf pointing at a DIFFERENT module does not require loading", () => { + const authors = c.define( + AUTHORS, + s.record(s.object({ name: s.string() })), + { ["ada"]: { name: "Ada" } }, + ); + const other = c.define( + "/tags.val.ts", + s.record(s.object({ label: s.string() })), + { ["x"]: { label: "X" } }, + ); + const schemas = getSchemas([ + authors, + other, + c.define( + PAGES, + s.record(s.object({ tag: s.keyOf(other) })).jsonValues(), + {}, + ), + ]); + + expect( + jsonValuesLoadRequirements(schemas, { kind: "keyOf", module: AUTHORS }), + ).toEqual([]); + }); + + test("the match is transitive through object, array, record and union", () => { + const authors = c.define( + AUTHORS, + s.record(s.object({ name: s.string() })), + { ["ada"]: { name: "Ada" } }, + ); + const query = { kind: "keyOf", module: AUTHORS } as const; + + const inArray = getSchemas([ + authors, + c.define( + PAGES, + s.record(s.object({ list: s.array(s.keyOf(authors)) })).jsonValues(), + {}, + ), + ]); + expect(jsonValuesLoadRequirements(inArray, query)).toEqual([PAGES]); + + const inNestedRecord = getSchemas([ + authors, + c.define( + PAGES, + s.record(s.object({ byId: s.record(s.keyOf(authors)) })).jsonValues(), + {}, + ), + ]); + expect(jsonValuesLoadRequirements(inNestedRecord, query)).toEqual([PAGES]); + + const inUnion = getSchemas([ + authors, + c.define( + PAGES, + s + .record( + s.union( + "type", + s.object({ type: s.literal("plain"), text: s.string() }), + s.object({ type: s.literal("ref"), author: s.keyOf(authors) }), + ), + ) + .jsonValues(), + {}, + ), + ]); + expect(jsonValuesLoadRequirements(inUnion, query)).toEqual([PAGES]); + + const deeplyNested = getSchemas([ + authors, + c.define( + PAGES, + s + .record( + s.object({ + blocks: s.array(s.object({ author: s.keyOf(authors) })), + }), + ) + .jsonValues(), + {}, + ), + ]); + expect(jsonValuesLoadRequirements(deeplyNested, query)).toEqual([PAGES]); + }); + + test("only `.jsonValues()` records are candidates", () => { + const authors = c.define( + AUTHORS, + s.record(s.object({ name: s.string() })), + { ["ada"]: { name: "Ada" } }, + ); + // Same shape, WITHOUT .jsonValues(): its content is already in the source, so + // there is nothing to load. + const schemas = getSchemas([ + authors, + c.define(PAGES, s.record(s.object({ author: s.keyOf(authors) })), { + ["p"]: { author: "ada" }, + }), + ]); + + expect( + jsonValuesLoadRequirements(schemas, { kind: "keyOf", module: AUTHORS }), + ).toEqual([]); + }); + + test("a self-referencing jsonValues record requires loading itself", () => { + const pages: ValModule = c.define( + PAGES, + s.record(s.object({ title: s.string() })).jsonValues(), + {}, + ); + // keyOf pointing back at the same module. + const schemas = getSchemas([pages]); + const schema = schemas[PAGES]; + if (schema.type !== "record") { + throw new Error("expected a record"); + } + // Hand-built, because `s.keyOf(pages)` inside `pages` is not expressible. + schemas[PAGES] = { + ...schema, + item: { + type: "object", + items: { + related: { + type: "keyOf", + path: PAGES_AS_SOURCE_PATH, + opt: false, + values: "string", + }, + }, + opt: false, + }, + }; + + expect( + jsonValuesLoadRequirements(schemas, { kind: "keyOf", module: PAGES }), + ).toEqual([PAGES]); + }); + + test("file refs match the referenced gallery module", () => { + const gallery = c.define(GALLERY, s.images(), {}); + const schemas = getSchemas([ + gallery, + c.define( + PAGES, + s.record(s.object({ hero: s.image(gallery) })).jsonValues(), + {}, + ), + ]); + + expect( + jsonValuesLoadRequirements(schemas, { kind: "file", module: GALLERY }), + ).toEqual([PAGES]); + // A file query must not be satisfied by a keyOf, or vice versa. + expect( + jsonValuesLoadRequirements(schemas, { kind: "keyOf", module: GALLERY }), + ).toEqual([]); + }); + + test("route is an over-approximation: ANY route field counts", () => { + // `s.route()` records no target module, so we cannot tell which router a + // field points into and must load every jsonValues record that has one. + const withRoute = getSchemas([ + c.define(PAGES, s.record(s.object({ link: s.route() })).jsonValues(), {}), + ]); + expect(jsonValuesLoadRequirements(withRoute, { kind: "route" })).toEqual([ + PAGES, + ]); + + const withoutRoute = getSchemas([ + c.define( + PAGES, + s.record(s.object({ title: s.string() })).jsonValues(), + {}, + ), + ]); + expect(jsonValuesLoadRequirements(withoutRoute, { kind: "route" })).toEqual( + [], + ); + }); + + test("reports every module that needs loading", () => { + const authors = c.define( + AUTHORS, + s.record(s.object({ name: s.string() })), + { ["ada"]: { name: "Ada" } }, + ); + const schemas = getSchemas([ + authors, + c.define( + PAGES, + s.record(s.object({ author: s.keyOf(authors) })).jsonValues(), + {}, + ), + c.define( + "/posts.val.ts", + s.record(s.object({ author: s.keyOf(authors) })).jsonValues(), + {}, + ), + ]); + + expect( + jsonValuesLoadRequirements(schemas, { + kind: "keyOf", + module: AUTHORS, + }).sort(), + ).toEqual([PAGES, "/posts.val.ts"]); + }); +}); + +function getSchemas( + valModules: ValModule[], +): Record { + const schemas: Record = {}; + for (const valModule of valModules) { + const moduleFilePath = Internal.getValPath( + valModule, + ) as unknown as ModuleFilePath; + const schema = Internal.getSchema(valModule)?.["executeSerialize"](); + if (!schema) { + throw new Error(`Schema not found for ${moduleFilePath}`); + } + schemas[moduleFilePath] = schema; + } + return schemas; +} diff --git a/packages/ui/spa/components/jsonValuesLoadRequirements.ts b/packages/ui/spa/components/jsonValuesLoadRequirements.ts new file mode 100644 index 000000000..b28107a44 --- /dev/null +++ b/packages/ui/spa/components/jsonValuesLoadRequirements.ts @@ -0,0 +1,133 @@ +import { + ModuleFilePath, + SerializedObjectSchema, + SerializedSchema, +} from "@valbuild/core"; + +/** + * What a reference scan is looking for. Note the asymmetry: `keyOf` and + * `file` name the module they point AT, so they can be matched exactly, while a + * `route` field records no target module at all (`SerializedRouteSchema` only + * carries include/exclude patterns) and `getRouteReferences` matches by comparing + * the field's string VALUE to the route key. + */ +export type JsonValuesLoadQuery = + | { kind: "keyOf"; module: ModuleFilePath } + | { kind: "file"; module: ModuleFilePath } + | { kind: "route" }; + +/** + * Which `.jsonValues()` modules must have their entry CONTENT loaded before a + * reference scan for `query` can be trusted — decided from the schemas alone, so + * it costs nothing and needs no sources. + * + * Direction is what makes this cheap. A scan for references TO a module finds + * referrers, which are `keyOf`/`route`/file fields living somewhere else. The + * scanned record's own key set is always available (an un-loaded entry is still a + * marker under its key), so the only content a scan can be blind to is content + * that itself POINTS OUTWARD — i.e. a jsonValues record whose item schema + * contains a matching referrer, as in: + * + * ```ts + * s.record(s.object({ test: s.keyOf(otherModule) })).jsonValues() + * ``` + * + * In the overwhelmingly common case the result is empty: nothing to load, and the + * scan is complete and correct immediately. + * + * `route` is the one over-approximation — since the schema does not say which + * router a `s.route()` field points into, ANY jsonValues record containing a + * route field has to be loaded. + */ +export function jsonValuesLoadRequirements( + schemas: Record, + query: JsonValuesLoadQuery, +): ModuleFilePath[] { + const required: ModuleFilePath[] = []; + for (const moduleFilePathS in schemas) { + const moduleFilePath = moduleFilePathS as ModuleFilePath; + const schema = schemas[moduleFilePath]; + // `.jsonValues()` is root-only (locked decision #7), so only a module whose + // ROOT is a jsonValues record can hold un-loaded content. + if ( + schema === undefined || + schema.type !== "record" || + schema.jsonValues !== true + ) { + continue; + } + if (containsReferrer(schema.item, query, new Set())) { + required.push(moduleFilePath); + } + } + return required; +} + +/** + * True when `schema` can hold a field that {@link JsonValuesLoadQuery} would + * match, looking through objects, arrays, records and unions. + * + * `seen` guards against a schema that (however unusually) refers to itself + * structurally, so this cannot recurse forever. + */ +function containsReferrer( + schema: SerializedSchema | undefined, + query: JsonValuesLoadQuery, + seen: Set, +): boolean { + if (schema === undefined || seen.has(schema)) { + return false; + } + seen.add(schema); + switch (schema.type) { + case "keyOf": + // `keyOf.path` is the referenced record's path; for a module-level record + // that is the module file path itself (the same comparison `getKeysOf` + // makes). + // Compared as plain strings: `keyOf.path` is branded `SourcePath` and the + // query carries a `ModuleFilePath`, but for a module-level record they are + // the same string (which is the comparison `getKeysOf` makes too). + return query.kind === "keyOf" && sameString(schema.path, query.module); + case "image": + case "file": + return query.kind === "file" && schema.referencedModule === query.module; + case "route": + return query.kind === "route"; + case "object": + return Object.values(schema.items).some((item) => + containsReferrer(item, query, seen), + ); + case "array": + case "record": + return containsReferrer(schema.item, query, seen); + case "union": + // Covers both the tagged form (object variants) and the literal form, + // whose items are literals and match nothing. + return ( + schema.items as (SerializedObjectSchema | SerializedSchema)[] + ).some((item) => containsReferrer(item, query, seen)); + case "string": + case "number": + case "boolean": + case "literal": + case "date": + case "dateTime": + case "richtext": + return false; + default: { + const exhaustiveCheck: never = schema; + console.error( + "Could not compute jsonValues load requirements. Unhandled schema type", + exhaustiveCheck, + ); + // Conservative: an unknown schema type might hold a referrer, and + // wrongly reporting "nothing to load" is what makes a guard lie. + return true; + } + } +} + +/** Compares two branded strings without asserting one into the other's brand. */ +function sameString(a: string, b: string): boolean { + return a === b; +}