diff --git a/docs/plans/jsonValues.md b/docs/plans/jsonValues.md new file mode 100644 index 000000000..bb44964e4 --- /dev/null +++ b/docs/plans/jsonValues.md @@ -0,0 +1,945 @@ +# Implementation tracker: `.jsonValues()` — lazily-loaded JSON entries + +> Living implementation plan for `s.record(...).jsonValues()` / `s.router(...).jsonValues()`. +> The **design rationale + locked decisions** live in the approved design doc +> (`~/.claude/plans/i-want-a-new-humming-zebra.md`). This file tracks **what's done / +> what's next** so we can resume across sessions. Keep the "Current state" block at the +> top up to date after every work chunk. + +--- + +## Current state / resume here + +> **Phase 6 step 4 DONE (2026-07-31): the load predicate.** `jsonValuesLoadRequirements(schemas, query)` +> answers "which jsonValues modules must be loaded before a reference scan can be trusted" from the +> SCHEMAS alone — no sources, no requests. Only root `.jsonValues()` records are candidates; their item +> schema is walked through object/array/record/union; an unknown schema type answers TRUE, because +> wrongly reporting "nothing to load" is how a guard starts lying. +9 tests. +> **Next: step 5 — ref hooks return a status and the destructive popovers gate on it (V10–V13, V17). +> This is where the data-integrity hole actually closes.** + +> **Phase 6 step 3 DONE (2026-07-31): virtualized list + the real bug it exposed.** Reviewing the list +> view turned up something worse than "N broken previews": every row's `` resolves its own path, +> and `useSchemaAtPathInternal` fires `requestJsonEntry` for any un-loaded marker it walks into — so +> opening a jsonValues record with N entries fired **N `/json` requests**. Fixed at the engine level by +> coalescing requests onto a microtask (one request per module per render pass) and deleting the +> single-entry fetch path entirely, so `ensureJsonEntry` shares `loadJsonEntriesSettled`. On top of that, +> `VirtualizedRecordList` virtualizes both list branches above 50 keys, requests only the rendered +> window, and swaps un-loaded rows for a fixed-height skeleton. +1 test (29 in the jsonValues describe). +> Self-review findings are recorded as tasks at the end of Phase 6 — two were fixed in the pass, seven +> remain (chiefly: no component tests, because the jest preset has no jsdom). +> **Next: step 4 — `jsonValuesLoadRequirements` predicate + tests.** + +> **Phase 6 step 2 DONE (2026-07-31): engine primitives.** `requestJsonEntries(mfp, keys)` +> (fire-and-forget window) and `ensureJsonEntries(mfps)` (awaitable, whole-module, returns +> `{complete, errors}` so a guard can tell "loaded" from "failed") both delegate to one private +> `loadJsonEntries` — filter, chunk at 50, one `/json` per chunk, one `invalidateSource` per batch. +> Keys that exist only in a pending patch are never requested (no committed content to fetch; a request +> would 404 and wrongly mark the row errored). Progress is `{status, loaded, total, percentage}` on +> `subscribe("json-entries-progress")`, counted per RUN across modules so a percentage never resets at a +> module boundary. Publish invalidation is batched. +8 tests (28 in the jsonValues describe). +> **Next: step 3 — virtualize the record list + load the rendered window (V16).** + +> **Phase 6 step 1 DONE (2026-07-31): batch `/json`.** `ValOps.getJsonEntries` is the single +> implementation (`getJsonEntry` is a one-key wrapper); `initSources`/`fetchPatches` are hoisted out of +> the per-entry loop; per-entry failures are per-entry (`missing`/`errors`) so one corrupt `*.val.json` +> cannot fail a batch; `offset`/`limit` refuses `apply_patches` rather than silently omitting +> draft-added keys. `keys` is a repeated query param (no plumbing needed — the client already serializes +> array query values). +14 tests, `pnpm test` 1118 green, `-r typecheck` clean, lint clean. +> **Next: step 2 — engine primitives (`requestJsonEntries` / `ensureJsonEntries` + progress store).** + +> **Reference-integrity defect found (2026-07-30) — Phase 6 is the next milestone, step 1 done.** +> The Studio's three global scans (route refs, keyOf refs, referenced files) and search silently skip +> un-loaded `{_type:"json"}` markers, so the delete gate and the rename fixup can both answer "no +> references" for a ref that lives inside an entry the user happens not to have opened. That is a +> data-integrity hole, not a cosmetic one, and the result is nondeterministic (it depends on session +> history). Phase 6 below fixes it with a **schema-derived scoping rule** (in the common case NOTHING +> needs loading) plus a **batch `/json`** for the cases that do, and for search. Read Phase 6 before +> touching any of the `get*References` / `traverseSchemas` / search files. +> Phase 6 also covers the **record LIST view**, which turns out to be the most user-visible half: it +> renders a per-entry preview for every key, unvirtualized, so a jsonValues record currently shows N +> broken previews. Fix = virtualize with `@tanstack/react-virtual` + load only the rendered window. +> This **supersedes V1** ("zero `/json` on open"). +> **Phase 7 (2026-07-31)** is the follow-on: the SPA already holds the user's REAL schema instances +> (`window.__VAL_MODULES__` → `extractValModules().schemas`) and throws them away in `setValModules`, +> re-deriving a lobotomised schema with `deserializeSchema` that has no render `select`, no custom +> validators and no router. Using the instances fixes renders (all schema types, draft-accurate) and +> finally lets custom validators run client-side. Phase 6 step 7 moved there. + +> **Draft runtime path DONE (2026-07-28):** the last Phase 4 box is closed on the server + RSC side. +> `/json` takes `apply_patches` (default **true**, mirroring `/sources/~`) and replays pending +> patches via the new pure `applyJsonValuesEntryPatches`; the Studio passes `false` because it owns +> in-flight client patches. RSC `fetchValKey`/`fetchValRoute` read drafts through it when enabled and +> fall back to the local thunk. **jsonValues entries now get click-to-edit at all** — `stegaEncode` +> gained an optional `root` seed (entry path + serialized item schema), without which every +> jsonValues stega call was a silent identity transform. Also fixed `getSources` poisoning a +> jsonValues module's whole patch chain (observed live on the example support router). +> Still open: client `useValKey`/`useValRoute` render committed content in draft mode (needs the +> overlay emitter to carry patched sources — same limitation `useValStega` has today), and the +> **manual Studio walkthrough (V1–V9) has still NOT been run.** + +> **Studio hardening DONE (2026-07-27):** the Phase 3 defects are fixed. Renaming an entry now +> commits (`move`/`copy` arms in `ValOps.prepare`), a published edit no longer appears to revert +> (json entry cache is invalidated on publish), a failed `/json` load renders an error instead of a +> forever-spinner, and nested `.jsonValues()` is rejected at startup. Also fixed a **pre-existing** +> bug found on the way: `analyzePatches` pushed one `patchesByModule` entry per op, so `prepare` +> re-applied the whole patch once per op (harmless for `replace`, corrupting for `move`). +> Automated tests green; the **manual Studio walkthrough (V1–V9 below) has NOT been run yet.** + +> **Commit flow DONE (2026-07-03):** `ValOps.prepare` now routes patch ops for `.jsonValues()` +> records. Content edits write only the entry's `*.val.json` (the `.val.ts` is NOT touched); adding +> an entry writes a new `*.val.json` + inserts a `c.json(() => import("..."))` thunk into the +> `.val.ts`; removing an entry deletes the `*.val.json` (via a `patchedSourceFiles[path] = null`) + +> drops the thunk. All three go through the existing `patchedSourceFiles` map, so both `ValOpsFS` +> and `ValOpsHttp` commit them with no new abstraction. Server suite green (166), whole monorepo +> `pnpm test` green (1056), `-r typecheck` clean. + +> **Studio lazy-load DONE (2026-06-30):** opening a jsonValues entry now fetches its content via +> `GET /json` and renders the fields (was: the `resolvePath` guard error). Read path works in both +> production (`fetchValKey`) and the Studio. **Editing** now persists on commit (see above). +> Requires `pnpm --filter @valbuild/ui build` for the Studio bundle to pick up UI changes (it's a +> built bundle, not a live dev-stub). + +- **Phase**: 1 ✅. Phase 2 server: validation + loader + emit primitive + **/json endpoint** + + **commit flow** ✅. Phase 3 UI lazy-load ✅ (Studio reads jsonValues entries). Phase 4: + `fetchValKey`/`useValKey` + **`fetchValRoute`/`useValRoute`** ✅ (production path). Example + (support pages) now uses `fetchValRoute`; `examples/next` `next build` green. + **The `c.json` sha was removed (2026-07-02).** Remaining: end-to-end Studio verify of add/remove/ + edit against a running dev server; **Enabled/Studio draft runtime path** (all the single-entry read + APIs still read the committed local thunk, not draft edits); full Phase 5 CI gate run. +- **Single-entry runtime read API (typecheck-validated, runtime-validation via example pending)**: + - RSC `fetchValKey` — `initFetchValKeyStega` in `next/src/rsc/initValRsc.ts` (returned as + `fetchValKeyStega`). Resolves ONE entry by key from the local module's thunk + stega-encodes. + - Client `useValKey` — `useValKeyStega` in `next/src/client/initValClient.ts` (returned as + `useValKeyStega`). Uses a module-level promise cache + `React.use` (Suspense) to load one entry. + - `fetchValRoute`/`useValRoute` now also load a single entry for `.jsonValues()` routers (map + params → key, then resolve one entry) — see Phase 4. ✅ + - Both: production path resolves the local thunk; Enabled/Studio **draft** path is a TODO (needs + the single-entry endpoint + a sub-selector for stega edit tags). +- **Commit flow — DONE (2026-07-03)**. Persists edits to `*.val.json` instead of the `.val.ts`. + Implementation landed: + - **Key enabler**: `ValOps.prepare`'s `patchedSourceFiles: Record` is written by + the commit loop to **arbitrary paths** relative to rootDir (`null` = delete). `*.val.json` + writes/deletes go through the same map (no new abstract FS method needed). + - **AST analyzer**: `patch/ts/jsonValuesModule.ts` `analyzeJsonValuesEntries(sourceExpr)` → + `Map` (uses `analyzeValModule` to get the source object literal). Tested. + - **Path helpers**: `patch/jsonValuesPatch.ts` — `getNewJsonEntryPaths(mfp, key)` (LOCKED + convention below) + `resolveExistingJsonPath(mfp, importPath)` (existing/hand-placed files use + the analyzer's importPath). Tested (`jsonValuesPatch.test.ts`). + - **Op classifier**: `patch/jsonValuesPatch.ts` `classifyJsonValuesOp(serializedSchema, opPath)` + walks the serialized schema; returns `{kind:"entry", recordPath, entryKey, subPath}` or + `{kind:"normal"}`. Handles root records/routers (recordPath `[]`) AND nested jsonValues records. + - **ts-ops**: `patch/ts/ops.ts` `insertValJsonEntry` / `removeValJsonEntry` insert/remove the + `c.json(() => import(...))` property on the record's object literal (built with + `createValJsonReference`, spliced with the internal `insertAt`/`removeAt`). Tested + (`jsonValuesEntry.test.ts`). + - **Routing in `prepare.applySourceFilePatches`**: fetches `this.getSchemas()` once, serializes per + module, and processes each patch's `sourceFileOps` op-by-op (in order). Per op: + 1. `normal` → `applyPatch(tsSourceFile, tsOps, [op])` (unchanged behavior; `.val.ts` reformatted). + 2. `entry` + empty subPath → STRUCTURAL: `add` = new `*.val.json` + `insertValJsonEntry`; + `remove` = `null` json + `removeValJsonEntry`; `replace` = whole-entry json content. + 3. `entry` + non-empty subPath → CONTENT: load current `*.val.json` (`getSourceFile` + + `JSON.parse`, cached in a per-module map), then replay the **rebased** op via `jsonOps` + (`rebaseContentOp` drops the record+entryKey prefix and rebases any move/copy `from`). + - **`.val.ts` untouched on pure content edits**: `applySourceFilePatches` now returns + `result: string | null` (`null` = ts unchanged) + `extraFiles: Record`. The + caller only writes `patchedSourceFiles[mfp]` when `result !== null`, and always merges + `extraFiles`. So a content-only commit writes ONLY the `*.val.json`. + - **Filename convention (LOCKED)**: new entry mirrors the key under a folder named after the + `.val.ts` (its `.val.ts` suffix becomes the folder). Module `/app/foo/[...slug]/page.val.ts`, + key `/foo/bar/zoo` → jsonPath `/app/foo/[...slug]/page/foo/bar/zoo.val.json`, importPath + `./page/foo/bar/zoo.val.json`. EXISTING entries use the analyzer's importPath (hybrid authoring). + - **Tested**: `ValOpsFS.jsonValues.test.ts` (content edit writes only json; add writes json + + thunk; remove nulls json + drops thunk). Whole `pnpm test` green (1056). + - Validation already handles inline content (`validateJsonValuesEntries` for base thunks; + `executeValidate` validates inline content). `/sources/~` shallow markers already work + (JSON.stringify drops the thunk). +- **Last verified green**: core json suite (14 tests) + server `validateJsonValues`/loader/ + `jsonReference` suites; core + server typecheck. (Earlier: whole-monorepo `-r typecheck` clean + except the pre-existing unrelated `packages/cli` chokidar failure.) +- **Key API note**: `JsonSource` is a phantom-typed pure-JSON marker; the lazy thunk is + runtime-only — read it with `Internal.getJsonImport(source)`, never `source._import` in typed code. +- **Done since**: server-side per-entry json validation (`validateJsonValues.ts`, wired into + `ValOps.validateSources`); core `Internal.resolveJsonValues(source)` (eager resolver for the + `fetchVal`/`useVal` path). **The sha was dropped (2026-07-02)** — `jsonValuesSha.ts` deleted, sha + removed from `c.json`/`JsonSource`/`/json`/analyzer/example. **Discovered gap**: even eager + `fetchVal` must resolve markers before stega-encoding (a jsonValues module's local source is + markers, not content) — use `resolveJsonValues` there. +- **Runtime integration notes (for fetchValKey/fetchVal in next/react)**: disabled/production path + reads the local module (`Internal.getSource`) whose markers still carry thunks → resolve locally + (`getJsonImport` for one key, `resolveJsonValues` for all). Enabled/Studio path gets shallow + markers from `/sources/~` WITHOUT thunks → needs the single-entry fetch endpoint (Phase 2) to load + draft content; until then it can fall back to the local thunk (committed content only). + +--- + +## Goal (one paragraph) + +Let `s.record(...)` and `s.router(...)` (NOT `s.images()` / `s.files()` galleries) opt into +`.jsonValues()`, so each entry's value lives in its own `*.val.json` file referenced by a lazy +thunk `c.json(() => import("./x.val.json"))`. Keeps `.val.ts` tiny at 10K+ +entries; runtime/Studio/validation work one entry at a time; zero overhead when Val is disabled. + +## Locked decisions (do not relitigate) + +1. `fetchVal`/`useVal` stay eager; new `fetchValKey`/`useValKey` + `fetchValRoute`/`useValRoute` + load a single entry. +2. Hybrid authoring: Val generates/maintains json files + thunks; hand-edits re-validated. +3. **No sha (dropped 2026-07-02).** `c.json` takes ONLY the thunk: + `c.json(() => import("./x.val.json"))`. The earlier sha / validation-cache-key idea (a + `-` token) was removed — not worth the complexity. There is no + revalidation token: validation just runs on an entry's content when it is loaded (we accept that + validation/builds take more time). `jsonValuesSha.ts` was deleted. +4. Type precision: keep object/array structure, widen only what JSON can't carry + (literals → base, drop `RawString`/brand, widen `_type` literals). Runtime validation enforces + strictness. Val object-unions are always discriminated, so distribute+recurse suffices. +5. i18n deferred; design json format locale-agnostic. +6. All-or-nothing: every entry of a `.jsonValues()` record is a `c.json` thunk (no mixing). +7. **Root-only (LOCKED 2026-07-27).** `.jsonValues()` is only supported on a module's ROOT + record/router. A nested one is rejected at startup: `ValOps.initSources` reports a `ModulesError` + per offender (via `findNestedJsonValuesRecords`), so `/sources/~` fails with "Val is not correctly + setup" naming the module; the commit flow rejects nested entry ops as defense in depth. + Rationale: the `/json` endpoint keys entries by a single string, the Studio substitutes content at + the top level of the module source, and `validateJsonValuesEntries` only visits a root record — + nested entries would silently get NO content validation. `classifyJsonValuesOp` still reports + `recordPath` truthfully so the door stays open. +8. **A rename relocates the file (LOCKED 2026-07-27).** A `move` writes the destination via + `getNewJsonEntryPaths` — the generated convention path — and deletes the source file. So renaming + a hand-placed `content/faq.val.json` produces `page/support/faq2.val.json`. One invariant; the + accepted cost is that renaming empties a hand-authored directory. + +## Key runtime shapes + +`JsonSource` is a phantom-typed **pure-JSON marker** so `Source` stays JSON-serializable +(no `_sha`): + +```ts +// JsonSource TYPE (what flows through Source/SelectorSource): +{ _type: "json", patch_id?: string } & PhantomType + +// RUNTIME value produced by c.json(thunk) — also carries the thunk, which is +// NOT in the type; read it via Internal.getJsonImport(source): +{ _type: "json", _import: () => Promise<{ default: T }> } + +// Over the wire (/sources/~): the marker only (thunk dropped): +{ _type: "json", patch_id?: string } +``` + +--- + +## Phase 1 — Core (`packages/core`) ✅ + +- [x] `source/json.ts`: `JsonSource` (default `T = unknown` so the unions accept any + content), `_type:"json"` const (`JSON_VAL_EXTENSION_TAG`), `json()` ctor, `isJson()`, + `JsonOf` transform (distribute + recurse + widen leaves). +- [x] `source/index.ts`: `JsonSource` added to `Source` union. +- [x] `selector/index.ts`: `JsonSource` in `SelectorSource`; mapped in `Selector` → + `GenericSelector`. +- [x] `initVal.ts`: `json` added to `c` + `ContentConstructor`. +- [x] `schema/record.ts`: `.jsonValues()` modifier + `isJsonValues` flag; `Src` widened to + `JsonValuesRecordSrc`; serializes `jsonValues`; throws on media galleries; defers value + validation (record-level only asserts `isJson` marker); `validateJsonEntryContent()` helper. +- [x] `schema/record.ts` (`SerializedRecordSchema`) + `schema/deserialize.ts`: carry `jsonValues`. +- [x] `module.ts` `resolvePath` (both variants): descend a json entry → schema becomes `item`; + throws/returns clear error when traversing deeper into an unloaded marker. +- [x] `index.ts`: export `JsonSource`/`JsonOf` types + `Internal.isJson`. +- [x] Tests: `schema/jsonValues.test.ts` — `JsonOf` compile-time, `c.json` unit, serialize/ + deserialize round-trip, router compose, gallery rejection, deferred validation, authoring + surface (`c.define` + `c.json`). +- [x] **Verified**: `pnpm test packages/core` (454 tests) + core typecheck green. + +## Phase 2 — Server (`packages/server`) + +- [x] `loadValModules.ts`: `loadModule` parses `.json` (mirrors Node `require` json); `.json` added + to `RESOLVE_EXTENSIONS`; dynamic `import()` transpiles to a lazy `require` via `customRequire` + so thunks stay lazy. Fixture: `test/jsonValues-fixture/` + `loadValModules.jsonValues.test.ts` + (verifies marker shape, laziness, and thunk-loads-json). ✅ +- [x] `patch/ts/ops.ts`: `createValJsonReference(importPath)` — emits `c.json(() => import("..."))` + (factory-built; uses `createIdentifier("import")` to print a dynamic import without casting the + ImportKeyword token). Tested in `jsonReference.test.ts`. ✅ +- [x] `patch/ts/ops.ts`: `insertValJsonEntry` / `removeValJsonEntry` insert/remove a + `c.json(() => import(...))` entry property on the record object literal (via + `insertAt`/`removeAt` + `createValJsonReference`). Tested (`jsonValuesEntry.test.ts`). ✅ +- [x] **Per-entry validation**: `validateJsonValues.ts` (`validateJsonValuesEntries`) loads each + entry's content via `getJsonImport` and validates against the item schema; wired into + `ValOps.validateSources` (runs before the `res === false` early-continue). Tested in + `validateJsonValues.test.ts` (valid/invalid/load-error/non-jsonValues-skip). ✅ +- [x] `ValOps.ts` commit flow: `prepare.applySourceFilePatches` routes ops via `classifyJsonValuesOp` + and writes `*.val.json` (content edits) + inserts/removes `c.json(...)` thunks for add/remove + through the existing `patchedSourceFiles` map (so `ValOpsFS`/`ValOpsHttp` commit them + unchanged). `.val.ts` is not written on pure content edits. Tested + (`ValOpsFS.jsonValues.test.ts`). ✅ (Still to confirm: shallow `/sources/~` serialization end to + end against the Studio.) +- [x] Core eager resolver `Internal.resolveJsonValues(source)` (for `fetchVal`/`useVal`). ✅ +- [x] `ValServer.ts` `/json`: fetch one entry's content, **draft-aware** (2026-07-28). It takes + `apply_patches` (default TRUE, mirroring `/sources/~`); the Studio passes `false` because it owns + in-flight client patches and would otherwise double-apply. The handler is a thin adapter over + `ValOps.getJsonEntry(path, key, {applyPatches})`, which reads committed content from the import + thunk (fs + http, no extra I/O) and replays pending patches with `applyJsonValuesEntryPatches`. + NOTE: no `patch_id[]` pinning — no caller has patch ids, and `fetchPatches({patchIds})` is + already there if one appears. `/sources/~` shallow markers fall out of `JSON.stringify` dropping + the thunk. ✅ +- [x] `getSources` is jsonValues-aware (2026-07-28). It used to apply entry-content ops with `jsonOps` + against the opaque marker → "Cannot replace object element which does not exist", which then + marked the module poisoned and skipped **every** later patch for it. Content sub-ops are now + skipped (the module source is genuinely unaffected); whole-entry add/replace/move/copy push the + MARKER so the key set stays right for drafts. ✅ +- [x] **Verify**: `pnpm test packages/server/...` green. ✅ + +## Phase 3 — UI (`packages/ui/spa`) — NEXT MILESTONE (do with Studio running) + +The Studio currently throws the (intentional) `resolvePath` guard when opening a jsonValues entry, +because entry content is never loaded into the client source tree. Concrete integration points +(all in the 3376-line `ValSyncEngine.ts` unless noted): + +- [ ] **Content cache + fetch**: add `private jsonEntryContents: Record>` and `private loadingJsonEntries: Set<"mfp\0key">`. Add `requestJsonEntry(mfp, key)` + that, if not loaded/loading, calls `this.client("/json", "GET", { query: { path: mfp, key } })`, + stores `content`, then `invalidatePatchedSourcesCache(mfp)` + clears `cachedSourceSnapshots` + for the module + `emit(this.listeners["source"]?.[mfp])` / `["sources"]` so subscribers + re-render. (Mirror the existing `requestModuleValidation` side-effect pattern.) +- [ ] **Substitution before patches**: in `getPatchedSource(mfp)`, build the effective base by + replacing each loaded json marker at `baseSource[key]` with `jsonEntryContents[mfp][key]` + BEFORE applying patches (so field-level patches at `?p="key"."field"` apply on top). Markers + without loaded content stay as-is (list view only reads keys). Invalidate the patched cache on + load (above) since the effective base changed. +- [ ] **Trigger on open**: the field component that renders a navigated-to entry path must call + `requestJsonEntry(mfp, key)` (effect on mount, keyed by path). Find the entry detail renderer + (AnyField/Field at a path); when the path's parent schema is a `jsonValues` record and the + marker isn't loaded, request it and render a loading state until `getSourceSnapshot` returns + content. `useShallowSourceAtPath(path, "record")` already returns keys only (list is fine). +- [x] **Per-entry patches**: editing an entry field produces a normal patch at the entry path; on + commit the server commit-flow writes the `*.val.json` (Phase 2 commit flow). Add/remove entry = + add/remove the record key (commit flow emits/removes thunk + file). **Rename** (`move`) and + duplicate (`copy`) now supported too — see Session 4. +- [x] **Cache lifecycle + error state** (Session 4): `jsonEntryErrors` memoizes a failed load (no + refetch-on-remount loop; the field renders an error, not a forever-spinner) and + `staleJsonEntries` invalidates loaded entries on publish / `/sources/~` refresh. The stale flag + is cleared when a request STARTS, so an invalidation that lands mid-flight wins over the + in-flight response. `retryJsonEntry` / `ensureJsonEntry` added. +- [ ] **Verify** (Studio running) — **NOT YET RUN**. Walkthrough: + - **V1** open the router module → both keys listed, **zero** `/api/val/json` requests. + ⚠️ **SUPERSEDED by Phase 6**: once the list view loads previews for visible rows, opening a module + legitimately requests the visible window. Verify V16 instead (bounded by visible + overscan, never + by total key count). "Zero on open" only holds while the list renders no per-entry preview. + - **V2** open an entry → exactly one `/json`; revisiting it → no new request. + - **V3** edit `title`, publish → only `content/faq.val.json` modified, `page.val.ts` untouched, and + the new title **survives the publish without a reload**. + - **V4** add `/support/new-page` → new `page/support/new-page.val.json` = `{"title":"","body":"","order":0}` + (from `emptyOf`) + a `c.json` thunk in `page.val.ts`. + - **V5** hand-authored `content/` and generated `page/support/` coexist; re-editing an existing + entry still writes its original path. + - **V6** rename → new file with the same content, old file deleted, thunk key + import path swapped. + - **V7** delete → file deleted, thunk gone (empty dirs are left behind; `deleteFile` doesn't prune). + - **V8** corrupt a `*.val.json` → `/json` 500s → ONE request, an error in the field, no refetch on + navigate-away-and-back; restoring the file + a refresh clears the memo. + - **V9** a nested `.jsonValues()` module → Studio refuses to load, naming the module. + +## Phase 4 — Runtime APIs (`packages/next`, `packages/react`) + +- [x] `rsc/initValRsc.ts`: `fetchValKey` (`initFetchValKeyStega`, returned as `fetchValKeyStega`). +- [x] `client/initValClient.ts`: `useValKey` (`useValKeyStega`, promise cache + `React.use`). +- [x] `fetchValRoute` / `useValRoute`: jsonValues-aware. When the module schema is a `.jsonValues()` + record (`isJsonValuesRecordSchema` in `routeFromVal.ts`), map params → the entry key via + `getValRouteUrlFromVal` (passing the LOCAL source markers as the guard `val`), then load ONLY + that entry (RSC: `loadJsonEntryContent`; client: same `React.use` + `jsonEntryPromiseCache` as + `useValKey`). Non-jsonValues routers keep the eager `fetchVal`/`useValStega` path. Return types + gained a `NonNullable[string] extends JsonSource ? C | null : …` branch. ✅ +- [x] Example wires `fetchValRoute` (support pages) — `next build` green; `/support/[slug]` is a + single-entry dynamic route. (Was `fetchValKey`.) ✅ +- [x] **Edit tags for jsonValues entries** (2026-07-28). `stegaEncode` seeds its recursion only from + the selector branch, so calling it on an entry's RAW content (plain JSON, no `Path`/`GetSchema` + symbols) left `recOpts` undefined and every string hit the `!recOpts` bail — all four jsonValues + call sites were **silent identity transforms**, i.e. entries had no click-to-edit at all. + `stegaEncode` now takes `root?: {path, schema}`; `getJsonEntryStegaRoot` (in `routeFromVal.ts`) + builds it from `Internal.createValPathOfItem(modulePath, key)` + the serialized `item` schema, so + a field is tagged `…?p="/support/faq"."title"` — the shape `findUnloadedJsonEntryKey` walks. + Also needs `SET_AUTO_TAG_JSX_ENABLED(true)`, which only `initFetchValStega` used to call. + **Not** a sub-selector: `newSelectorProxy` isn't exported from core, it needs the private + `RecordSchema["item"]` instance, and re-entering the selector branch makes `getModuleIds` report + the sub-path as a module id. ✅ +- [x] RSC draft path (2026-07-28): `fetchValKey` + the jsonValues branch of `fetchValRoute` read + through the in-process `/json` (`loadDraftJsonEntry`) when enabled, falling back to the local + committed thunk. `initFetchValKeyStega` now takes `valServerPromise` + `getCookies`. ✅ +- [ ] **Client hooks draft path**: `useValKey`/`useValRoute` have the stega tags but still render + COMMITTED content in draft mode — the same limitation `useValStega` has today. The blocker is + that `overlayEmitter` is handed the raw `/sources/~` module (`apply_patches:false`, json entries + still markers), so reading `valOverlayContext.store` would be a no-op. Fixing that means + emitting `getPatchedSource(...)` (ideally from `invalidateSource`, the single choke point) and + having the overlay's engine proactively `requestJsonEntry` for entries that have drafts. That + changes `useValStega` behaviour for ALL client components, so it wants its own change. +- [ ] Draft-added routes are not reachable via `fetchValRoute` (params → key is resolved from the + LOCAL source), and draft-removed routes still route, then 404 → `null`. Revisit by resolving the + key set from `/sources/~` now that `getSources` keeps it correct for drafts. + +## Phase 5 — Example + CI gate + +- [x] Add a `.jsonValues()` router to `examples/next` with a few `*.val.json` entries + (`app/support/[slug]/page.val.ts` + `content/*.val.json`); consumed via `fetchValRoute`. +- [x] `cd examples/next && pnpm run build` green (`/support/[slug]` dynamic route builds). +- [ ] Full CI in one pass: `pnpm run lint`, `pnpm -w run format`, `pnpm run -r typecheck`, + `pnpm test`, `pnpm run build` (root preconstruct+ui; remember `pnpm preconstruct dev` after), + `cd examples/next && pnpm run build`. + +## Phase 6 — Reference integrity + search over un-loaded entries — NEXT MILESTONE + +### The defect (found 2026-07-30, reviewing PR #453) + +Four client-side global scans are blind to un-loaded entry content: + +| Scan | Marker handling | Backs | +| -------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ | ---------------------------------- | +| [getRouteReferences.ts:31](../../packages/ui/spa/components/getRouteReferences.ts#L31) | `isJson` → `return` | route refs | +| [traverseSchemas.ts:38](../../packages/ui/spa/components/traverseSchemas.ts#L38) | `isJson` → `return` | `getKeysOf` + `getReferencedFiles` | +| [traverseSchemaSource.ts](../../packages/ui/spa/utils/traverseSchemaSource.ts) | **none** — marker hits the object/record branch, `_type` matches no sub-schema, `continue` ⇒ indexes nothing | the LIVE search worker | +| [createSearchIndex.ts:25](../../packages/ui/spa/search/createSearchIndex.ts#L25) | `isJson` → `return` | **nothing — file is DEAD** | + +Why it is not cosmetic: route refs and keyOf refs merge into `allRefs` +([ArrayAndRecordTools.tsx:120](../../packages/ui/spa/components/ArrayAndRecordTools.tsx#L120)), which feeds +two mutating decisions — `refs.length > 0` is the ONLY thing that turns `DeleteRecordPopover` into +"Cannot delete", and `existingKeys` is the list of referring fields `ChangeRecordPopover` rewrites on +rename. A missed ref ⇒ delete looks safe (dangling ref left behind) / rename silently leaves the +referrer pointing at a key that no longer exists. Worse, the answer depends on which entries the user +happened to open this session, so it is nondeterministic. + +`traverseSchemaSource`'s accidental skip has one edge: an entry schema with a field literally named +`_type` or `patch_id` would index the marker's own value. + +### Scoping rule — direction decides (LOCKED 2026-07-30) + +The serialized schema names the target module for 2 of the 3 ref kinds: + +| Ref kind | Matcher | Names target module? | +| ---------- | ----------------------------------------------------------------- | --------------------------------------------------------------------- | +| `keyOf` | `schema.path === parentSourcePath` (`SerializedKeyOfSchema.path`) | **yes** | +| image/file | `schema.referencedModule === parent` | **yes** | +| `route` | `source === routeKey` (plain string compare) | **no** — `SerializedRouteSchema` only carries include/exclude regexes | + +Therefore: + +- **Incoming refs → keys only, ZERO loading.** Renaming/deleting key `K` of a jsonValues record `M`: + the referrers are `keyOf(M)` / `route` fields _elsewhere_, and `M`'s own key set is fully present in + the base source (markers preserve keys). `M`'s entry CONTENT is irrelevant to finding them. The one + entry we do need is the one being moved — already handled by the `ensureJsonEntry` await in + `ChangeRecordPopover`. One entry, not all. +- **Outgoing refs → the only case that needs content.** Loading is required only when a jsonValues + entry's own schema can contain a referrer to the thing being edited, e.g. + `s.record(s.object({ test: s.keyOf(otherModule) })).jsonValues()`. +- **The predicate** (schemas only — no sources, no fetching): for a query "refs to `M`", the set to + load = every jsonValues record whose **item schema transitively contains** (through + object/array/record/union) a `keyOf` with `path === M`, or an image/file with + `referencedModule === M`. In the overwhelmingly common case the set is EMPTY: no requests, no + progress UI, guard complete and correct immediately. Self-reference falls out for free. +- **Route refs are the one over-approximation**: since `s.route()` does not record which router it + points into, the predicate degrades to "does this jsonValues item schema contain ANY `route` field". + Still a large cut. Optional later narrowing: test the field's `options.include` regex against the + route key being renamed and skip fields that cannot match. +- **Search cannot be scoped** — it indexes all content by definition. It is the one unconditional + full-load consumer, which is why it (and only it) needs pagination + visible progress. + +### Third consumer: the record LIST view — visible rows only (added 2026-07-30) + +The list view already renders a per-entry preview for EVERY key, unvirtualized: +[RecordFields.tsx:162](../../packages/ui/spa/components/fields/RecordFields.tsx#L162) (default grid) and +[:191](../../packages/ui/spa/components/fields/RecordFields.tsx#L191) (`ListRecordRenderComponent`), both +``. For a jsonValues record that preview reads +the entry path, i.e. an un-loaded marker — so **today a jsonValues record list shows N broken/empty +previews**, and the naive fix (load what the preview needs) would load every entry on open, defeating +the whole feature. + +**Requirement (LOCKED 2026-07-30):** the list must be virtualized with `@tanstack/react-virtual` +(already a dependency — used in `FileGalleryListView` / `MediaPicker`, not yet in record lists) and +entry content loaded **only for the rows the virtualizer actually renders** (visible + overscan). + +- The virtualizer's rendered window IS the key window → **one batch request per window**, not one per + row. This is the strongest argument for the `keys` batch param. +- Debounce on scroll and drop stale windows: flinging through 10k rows must not enqueue 10k keys. + Requests already in flight for keys that scrolled out are harmless (they fill the cache) but must + not block newer windows. +- Rows whose content is not loaded yet render a skeleton/spinner, NOT an error — an un-loaded marker in + a read/render path is normal (see the watch-list note). +- **This changes V1.** "Open the module → zero `/json` requests" is no longer the invariant, because the + visible rows now load. The new invariant: requests are bounded by (visible + overscan) and by the + batch chunk size — never by the record's total key count. +- The list shares the SAME `jsonEntryContents` cache, so scrolling warms it for later ref scans. It must + NOT be mistaken for completeness: the refs guard still runs its own `ensureJsonEntries` for the + modules the predicate names. +- **`.render()` list layouts for jsonValues records** — now its own work item + step below (was a + deferred note). `RecordSchema.executeRender` returns `{}` when `isJsonValues` + ([record.ts:795](../../packages/core/src/schema/record.ts#L795)), so BOTH the per-entry renders and the + record-level `ListRecordRender` are dropped for these records (the early return precedes the + `renderInput` block). + +### Order of work (decided 2026-07-30) + +The list view comes early — it is the most user-visible breakage (a jsonValues record shows N broken +previews today) and it exercises the batch path under real scroll load before anything subtle depends +on it. Steps 1–2 are the only hard prerequisites; 4–6 are independent of each other after that. + +1. ✅ **Batch transport** (2026-07-31) — `ValOps.getJsonEntries` + `/json` batch mode + `ApiRoutes` zod. +2. ✅ **Engine primitives** (2026-07-31) — `requestJsonEntries` + `ensureJsonEntries` + progress store, + `markAllJsonEntriesStale` batched. +3. ✅ **List view** (2026-07-31) — virtualized `RecordFields` (both branches) with + `@tanstack/react-virtual`, loads the rendered window via `requestJsonEntries`, skeletons for un-loaded + rows. Also fixed the N-requests-on-open storm it exposed, by coalescing at the engine level. → **V16** + still to run manually. +4. ✅ **Predicate** (2026-07-31) — `jsonValuesLoadRequirements` + 9 unit tests. Pays off in step 5. +5. **Ref hooks + popover gating** — hooks return a status, destructive popovers gate on `success`. + → **V10–V13, V17**. This is where the data-integrity hole actually closes. +6. **Search** — first-query trigger, debounced re-index, marker skip in `traverseSchemaSource`, delete + the dead `createSearchIndex.ts`. → **V14**. +7. **`.render()` list layouts (windowed)** → **Phase 7 stage 1** (client-side schema instances), verified + by **V18**. Not gated on anything any more; it is simply a different phase's work. Until it lands, + step 3's skeleton + `` fallback IS the list preview. +8. **Verify + gate** — the full manual walkthrough (V1–V18, noting V1 is superseded; V1–V9 have still + never been run) then the Phase 5 CI gate. + +The caching decision (last item below) is deliberately NOT in this order — it is a question for Fredrik, +not a step. + +### Work items + +- [x] **Batch `/json` (transport)** — DONE (2026-07-31). GET `/json` now takes exactly one of `key` + (unchanged single-entry shape), `keys` (**repeated** query param, not a JSON array — `ValClient` + already serialises array query values, and the router's `groupQueryParams` already hands zod a + `string[]`, so no plumbing was needed), or `offset`+`limit`. Batch responses are + `{path, entries, missing, errors, total, offset?, limit?}`. `JSON_ENTRIES_BATCH_MAX = 100` is + exported from `ApiRoutes` so client and server agree; over-cap requests fail zod validation. + Shape violations are 400s, checked in `ValRouter.test.ts` (+5). `ValidQueryParamTypes` gained + `number` (the client stringifies query values anyway). +- [x] **Batch `ValOps.getJsonEntries(mfp, {keys | offset+limit}, {applyPatches})`** — DONE (2026-07-31). + `initSources()` and `fetchPatches()` are hoisted out of the per-entry loop (pinned by a test that + asserts `fetchPatches` is called exactly once for a 2-key batch); thunks resolve via `Promise.all`. + `getJsonEntry` is now a one-key wrapper over it, so there is a single code path. Per-entry problems + stay per-entry: an unknown key lands in `missing`, a corrupt `*.val.json` in `errors`, and only a + missing/non-record MODULE is a whole-request `not-found`. `offset`/`limit` REQUIRES + `applyPatches:false` and errors otherwise — enumerating from the base source would silently omit + draft-added keys, which is the exact bug class this phase exists to kill. Tests: +9 in + `ValOpsFS.jsonValues.test.ts`. +- [x] **`jsonValuesLoadRequirements(schemas, query)`** — DONE (2026-07-31). + `components/jsonValuesLoadRequirements.ts`: pure, schemas-only, no sources and no fetching. `query` + is `{kind:"keyOf"|"file", module}` or `{kind:"route"}`; returns the `ModuleFilePath[]` whose entries + must be loaded. Only modules whose ROOT is a `.jsonValues()` record are candidates (root-only, + locked decision #7), and their `item` schema is walked through object/array/record/union. + An unknown schema type returns TRUE (conservative): wrongly reporting "nothing to load" is exactly + how a guard starts lying. `keyOf.path` (branded `SourcePath`) and the query's `ModuleFilePath` are + compared as plain strings through a helper, so no type assertion is needed. + Tests (+9): empty for the incoming-ref case, empty for a `keyOf` at a different module, empty for + the same shape WITHOUT `.jsonValues()`, non-empty through array/nested-record/tagged-union/deep + nesting, self-reference, file refs matched by `referencedModule` (and not satisfied by a `keyOf` + query), the `route` over-approximation both ways, and multiple modules reported. +- [x] **`ValSyncEngine.ensureJsonEntries(moduleFilePaths)`** — DONE (2026-07-31). Awaitable, + whole-module, and it returns `{complete, errors}` rather than plain `void`: a guard that gates a + delete must be able to tell "loaded everything" from "some entry failed", which is the entire + point. Runs up to 3 passes, because an invalidation landing mid-flight re-marks entries stale and + reporting `complete` while holding pre-invalidation content would be the same lie in a new place. + Never called on boot. +- [x] **Progress store** — DONE (2026-07-31). `{status, loaded, total, percentage}` via + `subscribe("json-entries-progress")` + `getJsonEntriesProgressSnapshot()`. Counts the whole RUN + across modules and batches (not per module) so a percentage never resets at a module boundary; + resets to zero once nothing is in flight. `loaded` counts resolved keys — loaded, missing AND + failed — so a failing entry cannot stall the bar at 99%. The single-entry `ensureJsonEntry` counts + into the same run, so one indicator covers an opened entry and a loading list. +- [x] **`ValSyncEngine.requestJsonEntries(mfp, keys)`** — DONE (2026-07-31). Fire-and-forget window + loader; both public methods delegate to one private `loadJsonEntries` (filter → chunk at 50 → + `loadJsonEntryChunk`), so there is a single cache/in-flight/emit path. Skips + cached/in-flight/errored keys, so re-rendering the same window costs nothing, and **skips keys that + exist only in a pending patch** — they have no committed content, so requesting them would 404 and + wrongly mark the row errored (their value comes from the patch, via `getPatchedSource`). One + `invalidateSource` per batch rather than per entry. +- [x] **Coalesce single-entry requests** — DONE (2026-07-31), and it turned out to be the actual bug. + Each row's `` resolves its own path, and `useSchemaAtPathInternal` fires + `requestJsonEntry` for any un-loaded marker it walks into — so opening a jsonValues record with N + entries fired **N `/json` requests** (the symptom is a wall of spinners, but the cause is a request + storm, not a rendering bug). `requestJsonEntry`/`requestJsonEntries` now queue into + `pendingJsonEntryRequests` and flush on a microtask, so one render pass costs ONE request per + module. The single-entry fetch path was deleted: `ensureJsonEntry` now delegates to the same + `loadJsonEntriesSettled`, so there is one fetch path (and the Studio no longer uses `/json?key=` + at all — only the RSC runtime does). +- [x] **Virtualize the record list + load visible rows only** — DONE (2026-07-31). New + `VirtualizedRecordList` wraps both branches of `RecordFields` (default cards + the + `.render()` list), requests the rendered window via `requestJsonEntries`, and depends on the + window's CONTENT (not the array identity — `Object.keys` returns a fresh array each render, which + would re-fire the effect on every render of a long list). Records at or below + `VIRTUALIZE_THRESHOLD = 50` keys render plainly as before — a nested scroll container is a real UX + change and is not worth imposing on the ordinary small record — and that threshold also bounds the + un-virtualized load to one or two batches. Non-jsonValues records virtualize through the same path. +- [x] **Skeletons for un-loaded jsonValues rows** — DONE (2026-07-31). `RecordRowSkeleton` (fixed + height, so the virtualizer's measurements do not jump as content lands) replaces the preview when + the key's value in the patched source is still a marker; `useUnloadedJsonEntryKeys` computes that + set ONCE per list rather than subscribing per row. + **Still open**: (c) from the original item — a per-row retry affordance for the `errored` case. + A failed row currently falls through to ``, which renders the existing error state. +- [ ] **`.render()` list layouts for jsonValues records (windowed)** — **moved to Phase 7 stage 1**; it is + a client-side-instance change, not a transport one. Summary: replace the `isJsonValues` early return + in `RecordSchema.executeRender` with a per-key `isJson(itemSrc) → continue`, then call + `executeRender(mfp, patchedSource)` on the CLIENT instance. Because `executeRender` iterates + `Object.entries(src)` and un-loaded entries are still markers, the resulting + `ListRecordRender.items` ([render.ts:9](../../packages/core/src/render.ts#L9)) is naturally + **partial** — exactly the loaded keys — and `resolveRefPreview`'s keyed lookup + ([useRefPreview.ts:82](../../packages/ui/spa/components/useRefPreview.ts#L82)) misses on the rest, + which the skeleton item above turns into a skeleton. No `render` field on `/json`, no pagination in + the render path. (Earlier draft of this item assumed only the server could run `select`; wrong — see + Phase 7.) +- [ ] **Ref hooks stop lying** — `useEagerRouteReferences` / `useKeysOf` / `useReferencedFiles` return + a status (`loading` + progress → `success` with COMPLETE refs → `error`) instead of a bare array. + Each hook calls `jsonValuesLoadRequirements` first; empty ⇒ synchronously `success` (today's + behaviour, no request). Non-empty ⇒ `ensureJsonEntries` and report progress. +- [ ] **Popovers gate on completeness** — `DeleteRecordPopover` / `ChangeRecordPopover` render progress + and refuse to act until `success`; on `error` they stay blocked with a retry. Never + "no refs found, go ahead". This is the actual fix for the defect. +- [ ] **Search** — trigger `ensureJsonEntries(all jsonValues modules)` **only on user intent**: search + is lazy, so nothing loads before it is requested. Trigger on the first non-empty query, NOT on + `SearchField` mount / dialog open (radix mounts the content when the dialog opens, so a + mount-effect trigger would load on open). **Debounce the re-index** — + [Search.tsx:121-138](../../packages/ui/spa/components/Search.tsx#L121-L138) rebuilds the whole index + on every `sources` change and would otherwise rebuild once per batch. +- [ ] **Search shows partial results + a percentage while loading** — results appear IMMEDIATELY from + whatever is already indexed (never a blocked/empty dropdown waiting for a full load), and the + dropdown carries a loading indicator with a **percentage** — `Math.round(loaded / total * 100)` off + the progress store, e.g. "Searching… 42% indexed". The list re-renders as each batch lands and the + indicator disappears at 100%. Two details that matter: the percentage must be over the whole + requested set (all jsonValues modules), not per module, or it resets visibly on every module + boundary; and results arriving late must not reorder/jump what the user is already looking at more + than the new matches require. +- [ ] **Fix the live search traversal** — add the marker skip to `traverseSchemaSource` (interim + correctness + kills the `_type`/`patch_id` edge). +- [ ] **Delete the dead `createSearchIndex.ts` + `search.test.ts`** and the unused barrel export in + `search/index.ts`. The marker skip added there was on unreachable code. +- [x] **Fold `markAllJsonEntriesStale` into the batch path** — DONE (2026-07-31). + `markJsonEntriesStale` now marks the module's loaded keys stale and hands them to + `requestJsonEntries` in one call, so a publish with hundreds of cached entries is one batch per + module instead of one request per entry (pinned by a test that counts batches AND per-key + requests). It feeds the same progress store, so the post-publish refresh is visible. + **Still open**: making the consumers re-enter `loading` on that refresh (refs guard, search index, + entry detail) — that lands with those consumers in steps 5–6, since they do not exist yet. +- [ ] **Verify** (Studio running): + - **V10** rename/delete a key in a jsonValues router while another ORDINARY module holds a + `keyOf`/`route` ref to it → refs found, delete blocked, rename rewrites the referrer, and **zero** + `/json` requests (incoming-ref case). + - **V11** same, but the referrer lives inside a jsonValues entry (`keyOf` in the item schema) → + progress shown, refs found after load, delete blocked, rename rewrites it. + - **V12** delete a key in an ordinary record while no jsonValues item schema references it → zero + `/json` requests, guard instant. + - **V13** a batch load that fails mid-way → guard shows an error + retry, destructive action stays + blocked (never silently "no refs"). + - **V14** open search, type nothing → zero `/json`. Type a query → matches from already-indexed + content appear IMMEDIATELY (not after the load), the dropdown shows a percentage that climbs to + 100%, entry content becomes findable as batches land, and the index is NOT rebuilt once per batch. + - **V15** publish with many entries cached → one batched refresh (not N requests), progress visible, + no stale answers from the refs guard in between. + - **V16** open a jsonValues record with many entries → only the visible window (+ overscan) is + requested, in ONE batch, and those rows show real previews; rows below the fold show skeletons and + have NOT been requested. Scroll slowly → one batch per window. Fling to the bottom → the number of + requests is bounded by windows actually rendered, never by total keys. + - **V17** scroll a jsonValues list to the bottom, then rename a key whose only referrer is an entry + that was scrolled past → the refs guard still reports COMPLETE (cache warm ⇒ instant) and the + rename rewrites the referrer. + - **V18** (Phase 7 stage 1) a jsonValues record WITH `.render({layout:"list"})` → visible rows show the + user's title/subtitle/image; rows below the fold are skeletons of the same height (no measurement + jump, no marker reaching a preview component); scrolling fills them in. Then edit a visible row's + title WITHOUT publishing → the row updates as you type (the render is computed from the patched + source on the client). + +### Review findings (self-review of steps 1–3, 2026-07-31) + +Two issues were found and fixed during the pass (a missing React `key` in the un-virtualized branch, +and an effect that re-fired on every render because `Object.keys` returns a fresh array). What is left: + +- [ ] **No component tests for `VirtualizedRecordList`** — the jest preset is `testEnvironment: "node"` + and `jest-environment-jsdom` is not installed, so nothing in the repo renders React. The + virtualization, the window→`requestJsonEntries` wiring and the skeleton swap are therefore covered + only by V16 (manual). Either add a jsdom project to the preset (`@testing-library/react` IS already + a devDependency of `packages/ui`) or accept manual-only and say so. +- [ ] **Nested scroll container needs a real look (V16)** — the virtualized branch introduces an + `overflow-auto` viewport capped at `VIEWPORT_MAX_HEIGHT = 800`, inside the Studio's own scroll + container. Two things to check on a real screen: that scroll chaining feels right (the inner + scroller should not trap the page), and that 800px is not taller than a small viewport. A + `max-h-[70vh]`-style cap may be better than a fixed pixel height. +- [ ] **Row-height estimates are guesses** — `CARD_ROW_HEIGHT = 186` / `RENDER_ROW_HEIGHT = 104` were + derived from the card's `max-h-[170px]` + padding, not measured. `measureElement` corrects the real + heights, so the estimate only affects the initial scrollbar and the first window's size; still + worth checking against V16 so the first window is not visibly wrong. +- [ ] **Skeleton has no retry affordance** — item (c) of the original skeleton task. A row whose entry + FAILED to load falls through to `` and renders the existing error state; there is no + per-row retry button calling `retryJsonEntry`. +- [ ] **`loadJsonEntriesSettled`'s 3-pass bound is arbitrary** — it is a backstop against an + invalidation loop, not a computed limit. If a pathological case ever exhausts it we return + `complete: false` with no errors, which reads as "incomplete for an unknown reason". Consider + logging when the bound is hit so it is diagnosable rather than mysterious. +- [ ] **The un-virtualized path requests up to 50 keys on mount** — for a jsonValues record of 50 + entries, opening it loads all 50 (one batch). That is a deliberate trade (see + `VIRTUALIZE_THRESHOLD`) but it does mean "zero requests on open" is never true for small + jsonValues records. Revisit if a 50-entry record's previews prove expensive to load. +- [ ] **Progress `percentage` is 100 when idle** — so a consumer that renders it without checking + `status` first shows "100%" before anything starts. Documented on the type; a `null` percentage + while idle would be harder to misuse. +- [ ] **DECISION NEEDED — ask Fredrik: efficient browser caching for `/json`.** Deferred from the + Phase 6 design round (2026-07-30). Fredrik's idea: in **http/prod mode there is a stable commit**, + so put it in the request and make the response immutably cacheable by the browser. Open parts: + (1) FS/dev mode has no commit — what is the key there, or do we simply not cache? + (2) `ValClient` sets its own headers, exposes no response headers, and zod-validates a JSON body, + so `ETag`/`If-None-Match`/304 is NOT reachable without extending that layer + ([ValClient.ts:88](../../packages/shared/src/internal/ValClient.ts#L88)). + (3) `apply_patches:false` (what the Studio sends) is a pure function of committed files, so it is + the cacheable case; `apply_patches:true` is mutable. + (4) Alternative if headers stay off-limits: in-payload conditional loading (client sends known + version tokens, server replies "unchanged") — needs a per-entry version token, and + `jsonValuesSha.ts` was deliberately deleted (locked decision #3), so there is none today. + Until this is decided, "caching" means only: the client never refetches what is in + `jsonEntryContents`, and the server hoists `initSources`/`fetchPatches` per batch. + +**Longer-term alternative (not now):** a server-side reference-scan endpoint ("which source paths hold +`keyOf`/`route` value X"), which the server can answer from the `*.val.json` files on disk with no +client download at all. Phase 6 keeps the scan client-side; the schema predicate is what makes that +affordable. Revisit if the outgoing-ref case (V11) turns out to be common in real projects. + +## Phase 7 — Use the CLIENT-SIDE schema instances (added 2026-07-31) + +### The realisation + +The SPA already has the user's REAL schema instances and has been throwing them away. +`` registers the registry on `window.__VAL_MODULES__` +([ValModulesClient.tsx:27](../../packages/next/src/ValModulesClient.tsx#L27)), `useValModules` reads it +([hooks/useValModules.ts](../../packages/ui/spa/hooks/useValModules.ts)), `ValProvider` pushes it into +`syncEngine.setValModules`, and `extractValModules` returns BOTH +`schemas: Record>` (instances) and `serializedSchemas` +([extractValModules.ts:113](../../packages/core/src/extractValModules.ts#L113)) — but `setValModules` +keeps only the serialized ones. Everything downstream then re-derives a lobotomised schema with +`deserializeSchema`, which drops (a) the render `select`, (b) `customValidateFunctions` (every case passes +`[]`), and (c) the router. Cross-bundle identity is already handled: the identity symbols use +`Symbol.for` so the SPA's copy of core and the host bundle's copy agree +([selector/index.ts:57](../../packages/core/src/selector/index.ts#L57)), and protected methods are called +by bracket access (`schema["executeSerialize"]()`) precisely because `instanceof` is unreliable across +copies. So: keep the instances and use them. + +Availability caveat: instances exist only when the host app renders ``. Without it +`window.__VAL_MODULES__` is undefined, `setValModules(null)` runs, and everything must fall back to +today's serialized behaviour. `localModulesStatus` (`absent`/`loading`/`error`/`loaded`) is the flag. + +### Stage 1 — renders from instances (absorbs Phase 6 step 7) → **V18** + +- [ ] Retain `extracted.schemas` on the engine as `localSchemaInstances`. +- [ ] Core: replace the `isJsonValues` early return in `RecordSchema.executeRender` + ([record.ts:795](../../packages/core/src/schema/record.ts#L795)) with a per-key + `isJson(itemSrc) → continue`, so a partially-substituted record renders its loaded keys. +- [ ] Engine `computeRender(mfp)` = `instance["executeRender"](mfp, patchedSource)` → the existing + `renders` map + `invalidateRenders`; memoized per (module, sourceSha); recomputed on source + invalidation. Wrap each key's `select` in try/catch so one throwing user function is one error row, + not a dead Studio. +- [ ] Windowing is free: un-loaded entries are markers, markers are skipped, `items` comes out partial, + `resolveRefPreview` misses → skeleton (Phase 6). +- What this buys beyond jsonValues: renders work again for ALL schema types (they are dead Studio-wide + today) and they are computed from the PATCHED source, so a row's title updates as the user types — + something the server path could never do. + +### Stage 2 — custom validators, worker kept (LOCKED 2026-07-31) + +The worker stays. `executeCustomValidateFunctions` only ever APPENDS errors +([schema/index.ts:91](../../packages/core/src/schema/index.ts#L91)) and `CustomValidateFunction` is +`(src, ctx:{path}) => false | string` — pure, per-node — so custom errors can be merged AFTER the +worker's structural result. No synchronous worker→main round trip is needed. + +- [ ] **Gate**: one boolean per module, memoized by `schemaSha` — does the serialized tree contain any + `customValidate: true`? If not (the common case) nothing changes: no extra message, no main-thread + work. Only modules that actually declare a custom validator pay anything. +- [ ] **Worker reports WHERE**: while it has the module, the worker walks (serialized schema, source) and + collects the paths of flagged nodes it actually visited (so unions/optional branches only report + paths that exist). Response grows `customValidatePaths`. This walk must be separate from + `executeValidate` — the deserialized schema has no validators, so it cannot report that it skipped + any. Walking stays off the main thread; the main thread only EXECUTES. +- [ ] **Main thread executes, time-sliced**: per path, `Internal.resolvePath(modulePath, source, instance)` + (already generic over `Schema | SerializedSchema`, + [module.ts:279](../../packages/core/src/module.ts#L279)) → node instance + src, then run that node's + validators. Slice on a ~5ms budget yielding via `scheduler.postTask({priority:"background"})` → + `requestIdleCallback` → `MessageChannel`. Abort when the module's `latestRequestId` changes + (`ValidationWorkerClient` already tracks it) or the sourceSha moves. + A slow user validator still blocks — accepted: devs who write slow validators see their own slowness. +- [ ] **Error store must MERGE, not replace** — structural errors publish immediately (fast feedback), + custom errors merge per chunk. A wholesale replace would erase the first publish. +- [ ] **API gap**: running one node's validators needs `customValidateFunctions`, which is + `private readonly` on EACH SUBCLASS, so no base-class helper can reach it. Chosen fix: add a one-line + `protected executeCustomValidateAt(path, src)` to each schema class (~15 files, mechanical, matches + the `schema["executeX"]()` convention). Rejected: bracket-accessing a private field from the SPA; + hoisting the field to the base class (cleanest end state, biggest diff — revisit in stage 3). + +**Triggers (LOCKED 2026-07-31)** — custom validation is NOT part of the load path: + +- [ ] **On update only**: `addPatch` → `requestModuleValidation(mfp)` runs structural (as today) plus + custom for THAT module. Note `requestAllModuleValidation()` currently fires from `setValModules` + ([ValSyncEngine.ts:2631](../../packages/ui/spa/ValSyncEngine.ts#L2631)), i.e. on boot and every HMR — + structural may keep doing that, custom must not. +- [ ] **Pre-publish**: validate every module touched by patches (the engine already knows the patch set + per module), custom included, with every needs-keys round resolved before publish is allowed. +- [ ] **"Validate absolutely everything" switch**: one call — + `validateAll({ custom: true, loadAllJsonEntries: true })` — walks every module, resolves every + needs-keys round, reports progress through the Phase 6 progress store. For dev/CI/debugging. + +**needs-keys protocol for `.jsonValues()` (LOCKED 2026-07-31)**: a custom validator cannot run against +opaque markers, so the call returns what it needs instead of guessing: + +``` +runCustomValidation(path) → + | { status: "done", errors } + | { status: "needs-keys", moduleFilePath, keys: string[] } +``` + +- [ ] The keys come from the record's MARKER key set in the module source — always present, no loading + required to compute them. A validator on the RECORD needs every key in its subtree; a validator on + the ITEM schema needs only that entry's key. +- [ ] The caller resolves them with Phase 6's `ensureJsonEntries` / `requestJsonEntries(mfp, keys)` and + retries. Guard the loop: if a retry returns the same `needs-keys` set, report an error instead of + spinning. Validation is thereby the FOURTH consumer of the batch loader (list view, refs guard, + search, validation). +- [ ] **Cost to document for users**: a custom validator on a `.jsonValues()` RECORD forces a full load of + that record whenever it runs (on update of that module, or pre-publish). Prefer putting custom + validators on the ITEM schema, which needs one key. This is inherent, not a bug: a record-level + validator is by definition a statement about all entries. +- [ ] **The server stays the authority for publish.** Client-side custom validation of a jsonValues module + is only ever as complete as what is loaded; `validateJsonValuesEntries` on the server validates every + entry. The client pass is feedback, not proof. +- [ ] **Expect newly-surfaced errors.** Client validation cannot run custom validators AT ALL today, so + existing projects may light up with errors that were previously invisible — and the publish gate + reads validation errors ([DraftChanges.tsx:143](../../packages/ui/spa/components/DraftChanges.tsx#L143)), + so some will block publish. Correct behaviour that will read as a regression; worth a release note. + +### Stage 3 — instances become the primary schema source + +- [ ] Serialized schemas stay for what genuinely must be JSON: `schemaSha` (the change-detection key + against the server), patch-op classification, `/sources/~` comparison. Anything needing BEHAVIOUR + (render, validate, `emptyOf`) reads instances. Honest framing: serialize never fully goes away; + `deserialize` **on the client** does. +- [ ] Ends with `deserializeSchema` removed from the SPA (the worker keeps it only if stage 2's structural + pass still runs there — which it does, so this is "removed from the main thread"). +- [ ] Consider hoisting `customValidateFunctions` (and the router) to the `Schema` base here, which + retires the stage-2 per-class helper. + +### Stage 4 — delete the server render path for the Studio + +- [ ] `getRenders` has exactly ONE caller — `/sources/~` + ([ValServer.ts:1480](../../packages/server/src/ValServer.ts#L1480)) — and only when + `apply_patches` is true, which the Studio never sends. Once stage 1 lands that branch is dead for the + Studio: remove it, and the `render` field on the `/sources/~` response if no other consumer wants it. + +--- + +## sha design — DROPPED (2026-07-02) + +The `c.json` sha (a `-` revalidation-cache key) was **removed** — it wasn't +worth the complexity. `c.json` takes only the thunk; `jsonValuesSha.ts` was deleted. Without a cache +key there is no skip-cache: `validateJsonValuesEntries` validates each loaded entry's content +unconditionally (accepted "validation takes more time" tradeoff). + +## Open questions / watch-list + +- `JsonOf` correctness vs `resolveJsonModule` inference (esp. images inside json: `_type` + widens to `string`, so json must NOT keep the literal `"file"` brand — `JsonOf` widens it). +- ~~Does `getSources()`/`deepClone` preserve the `_import` thunk?~~ **Resolved**: `deepClone` + ([patch/util.ts](packages/core/src/patch/util.ts)) passes functions through unchanged and + `_import` is enumerable, so base-source thunks survive. Patched/draft json entries become thunkless + markers (over-the-wire value) → `validateJsonValuesEntries` skips them by design (validated via the + draft/endpoint path later). So committed validation is correct. +- `vm` loader dynamic `import()` + `.json` resolution. +- resolvePath/selector must never throw on a not-yet-loaded entry. +- **Skipping an un-loaded marker is NOT automatically safe.** It is fine for a read/render path (the + content is simply not there yet) but WRONG for any scan whose answer gates a mutation — see Phase 6. + When adding a new traversal, decide explicitly which of the two it is. +- Browser caching of `/json` — deferred; the last Phase 6 item holds the open questions. +- **Does `jsonEntryContents` need eviction?** With visible-row loading, scrolling a 10k-entry list (or + running search once) eventually caches every entry's content in the client. Nothing evicts today. If + this bites, an LRU keyed by `mfp\0key` is the obvious answer — but eviction must never silently + downgrade a refs guard that reported COMPLETE, so the guard needs to re-check (or pin) the keys it + depended on. Decide only when we see real memory numbers. +- ~~**Where do per-entry renders come from for jsonValues?**~~ **Resolved (2026-07-31)**: from the CLIENT + schema instances — the SPA already has them via `window.__VAL_MODULES__` and discards them in + `setValModules`. See Phase 7. (`executeRender` returning `{}` for jsonValues and Studio renders being + null across the board are both fixed there.) + +## Changelog + +- **Session 6 (2026-07-30)**: planning only — NO code. Reviewing PR #453 surfaced the + reference-integrity defect (the marker skips in `getRouteReferences`/`traverseSchemas` silently make + the delete gate and rename fixup answer "no references", nondeterministically) and that the live + search path never handled markers at all while the file that does (`createSearchIndex.ts`) is dead. + Designed Phase 6: schema-derived scoping rule (direction decides — incoming refs need keys only, + so the common case loads NOTHING), batch `/json` (`keys` + `offset`/`limit`), a progress store, + status-returning ref hooks that gate the destructive popovers, lazy search-triggered full load, and + batched publish invalidation. Browser caching deferred to a decision item at the end of Phase 6. + Then added the **visible-rows-only list requirement** (Fredrik): the list view already renders a + per-entry `` for every key with no virtualization, so jsonValues records show N broken + previews AND the naive fix would load everything on open. Virtualize with `@tanstack/react-virtual` + and request only the rendered window (`requestJsonEntries(mfp, keys)`). Supersedes V1; added V16/V17, + plus watch-list entries for cache eviction and for `executeRender` returning `{}` on jsonValues. + Then promoted the deferred render note to real work: an explicit **Order of work** (8 steps, list view + early because it is the visible breakage), **skeletons on all three preview paths** (a marker must never + reach a preview component), **windowed `.render()` list layouts** (per-key `render` on the batch + response + partial `items`; gated on a DECISION about renders being null Studio-wide — `/sources/~` only + computes them when `apply_patches` is true, and the Studio always sends false), and **search showing + partial results with a percentage** while the index fills. Added V18. +- **Session 7 (2026-07-31)**: planning — **Phase 7 (client-side schema instances)** added, and Phase 6 + step 7 moved into it. The SPA already has the user's real `Schema` instances via + `window.__VAL_MODULES__` → `extractValModules().schemas`; `setValModules` discards them and everything + downstream re-derives a `deserializeSchema` copy with no render `select`, no `customValidateFunctions` + (every case passes `[]`) and no router. Stage 1 renders from instances (all schema types, computed from + the PATCHED source, so draft-accurate). Stage 2 keeps the validation WORKER and adds custom validators: + gate per module on the serialized `customValidate` flag, worker reports the flagged PATHS, main thread + executes them time-sliced, error store merges instead of replacing. Triggers locked: on update only, + all patch-touched modules pre-publish, plus a `validateAll({custom, loadAllJsonEntries})` switch. New + `needs-keys` protocol so a custom validator on a `.jsonValues()` record asks for the keys it needs and + the caller loads them via the Phase 6 batch loader (validation = its fourth consumer). Stages 3–4: + instances become primary (`deserialize` leaves the main thread), then delete the now-dead server render + path. The "renders pincer" decision item is gone — it was premised on a wrong claim that only the server + could run `select`. +- **Session 5 (2026-07-28)**: enabled/draft runtime path (server + RSC) and edit tags. + - Merged `main` (which fixed an unrelated blocker: the publish gate read the RAW + `errors.validationErrors` instead of the surfaced snapshot, so every `s.images()`/`s.files()` + gallery's always-on `check-unique-folder`/`check-all-files` errors blocked publish while the + Studio showed none — `0fcfecf0`). + - `getSources` jsonValues-aware — entry-content ops no longer poison the module's patch chain. + - `applyJsonValuesEntryPatches` + `rebaseContentOp` moved into `patch/jsonValuesPatch.ts`; new + `ValOps.getJsonEntry`; `/json` gained `apply_patches` (default true; Studio opts out). + - `stegaEncode` `root` seed + `getJsonEntryStegaRoot` → jsonValues entries finally get edit tags; + `SET_AUTO_TAG_JSX_ENABLED` now also set by the single-entry readers. + - RSC `fetchValKey`/`fetchValRoute` read drafts via `/json`. + - Test-harness fix: `ValOpsFS.jsonValues.test.ts` evaluated the module in a `vm` whose `require` + resolved `import("./x.val.json")` relative to the TEST FILE, so entry thunks never loaded. + - Tests: +2 `getSources` routing, +6 `ValOps.getJsonEntry`, +11 `rebaseContentOp` / + `applyJsonValuesEntryPatches`, +3 `stegaEncode` root seed (incl. a guard that no seed ⇒ identity). +- **Session 4 (2026-07-27)**: Studio hardening — closes the Phase 3 defects. + - **Rename/duplicate now commit.** `ValOps.prepare` classifies `op.from` as well as `op.path` and + gained `move`/`copy` arms: load the source entry's content, remove the old thunk (move only), + insert the new one at `getNewJsonEntryPaths`, and null the old file. Cross-record and + cross-entry `move`/`copy` are rejected instead of silently corrupting (`rebaseContentOp` sliced + `from` by the destination's prefix without checking they matched). + - **Pre-existing bug fixed**: `analyzePatches` pushed one `patchesByModule` entry per non-file op, + so `prepare` re-applied the _whole_ patch once per op. Idempotent for `replace`, but it + duplicated `add`s and broke `move`. Now at most one entry per (module, patch). + - **Nested `.jsonValues()` rejected** — `findNestedJsonValuesRecords` + a `ModulesError` from + `initSources` (locked decision #7). + - **Studio cache lifecycle**: `jsonEntryErrors` (failure memo + field error state), + `staleJsonEntries` (invalidate on publish and on `/sources/~` refresh), `retryJsonEntry`, + `ensureJsonEntry`. Publish needs its own invalidation because a content-only edit leaves the + module source (bare markers) byte-identical, so `sourcesSha` never changes and nothing refetches. + - **Rename guard**: `ChangeRecordPopover` awaits `ensureJsonEntry` before emitting the `move`, so + the patch carries real content instead of an opaque marker (which would 404 on the new key). + - Tests: `ValOpsFS.jsonValues.test.ts` (+7), `jsonValuesPatch.test.ts` (+6), + `validateJsonValues.test.ts` (+1 pinning the root-only contract), and a new `jsonValues` + describe in `ValSyncEngine.test.ts` (+8, was zero coverage) with `/json` + `/save` added to the + mock client. +- **Session 3 (2026-07-03)**: `fetchValRoute`/`useValRoute` made jsonValues-aware — they now map + route params → the entry key and load ONLY the matched entry (RSC `loadJsonEntryContent`; client + reuses the `useValKey` `React.use` cache), instead of eagerly resolving the whole record. Added + `isJsonValuesRecordSchema` to `routeFromVal.ts`; return types gained the `JsonSource ? C` branch. + Example support page switched to `fetchValRoute`; `examples/next` `next build` green. `-r typecheck` + - next tests green. +- **Session 2 (2026-07-03)**: Commit flow landed. New `patch/jsonValuesPatch.ts` (op classifier + + path helpers), new `insertValJsonEntry`/`removeValJsonEntry` in `patch/ts/ops.ts`, and a rewritten + `ValOps.prepare.applySourceFilePatches` that routes ops into `*.val.json` writes/deletes + + `.val.ts` thunk insert/remove, skipping the `.val.ts` on pure content edits. Tests: + `jsonValuesPatch.test.ts`, `jsonValuesEntry.test.ts`, `ValOpsFS.jsonValues.test.ts`. Whole + `pnpm test` (1056) + `-r typecheck` green. +- **Session 1**: Phase 1 (core) complete + tested; Phase 2 loader done + tested; + `createValJsonReference` primitive done + tested. Whole monorepo typechecks (except pre-existing + unrelated `packages/cli` chokidar error). `JsonSource` redesigned to a phantom-typed pure-JSON + marker (`_import` is runtime-only via `getJsonImport`) so `Source` stays JSON-serializable. +- _(baseline)_ tracker created; design approved. diff --git a/examples/next/app/blogs/[blog]/page.val.ts b/examples/next/app/blogs/[blog]/page.val.ts index f9544a619..a833f4d7e 100644 --- a/examples/next/app/blogs/[blog]/page.val.ts +++ b/examples/next/app/blogs/[blog]/page.val.ts @@ -3,7 +3,7 @@ import authorsVal from "../../../content/authors.val"; import { linkSchema } from "../../../components/link.val"; const blogSchema = s.object({ - title: s.string().maxLength(3), + title: s.string(), content: s.richtext({ inline: { a: s.route(), @@ -228,7 +228,7 @@ export default c.define( }, }, "/blogs/blog-15": { - title: "Blog 1", + title: "Blog dette er jo sinnsyjt! men hvorfor har vi to feil?", content: [ { tag: "p", diff --git a/examples/next/app/support/[slug]/content/faq.val.json b/examples/next/app/support/[slug]/content/faq.val.json new file mode 100644 index 000000000..a929eb5ba --- /dev/null +++ b/examples/next/app/support/[slug]/content/faq.val.json @@ -0,0 +1,5 @@ +{ + "title": "FAQ", + "body": "Frequently asked questions about Val. Each support page is its own *.val.json so the router scales to many pages.", + "order": 2 +} diff --git a/examples/next/app/support/[slug]/content/getting-started.val.json b/examples/next/app/support/[slug]/content/getting-started.val.json new file mode 100644 index 000000000..5eda7f822 --- /dev/null +++ b/examples/next/app/support/[slug]/content/getting-started.val.json @@ -0,0 +1,5 @@ +{ + "title": "Getting started", + "body": "Welcome to Val support. This page is stored as a separate JSON file and loaded lazily via .jsonValues().", + "order": 1 +} diff --git a/examples/next/app/support/[slug]/page.tsx b/examples/next/app/support/[slug]/page.tsx new file mode 100644 index 000000000..044196b35 --- /dev/null +++ b/examples/next/app/support/[slug]/page.tsx @@ -0,0 +1,21 @@ +import { fetchValRoute } from "../../../val/rsc"; +import supportVal from "./page.val"; + +export default async function SupportPage({ + params, +}: { + params: Promise<{ slug: string }>; +}) { + // Maps the route params to the matching entry and loads ONLY that entry's + // backing *.val.json (one dynamic import), not the whole support-pages record. + const page = await fetchValRoute(supportVal, params); + if (!page) { + return
Support page not found.
; + } + return ( +
+

{page.title}

+

{page.body}

+
+ ); +} diff --git a/examples/next/app/support/[slug]/page.val.ts b/examples/next/app/support/[slug]/page.val.ts new file mode 100644 index 000000000..d55ad50ce --- /dev/null +++ b/examples/next/app/support/[slug]/page.val.ts @@ -0,0 +1,25 @@ +import { s, c, type t, nextAppRouter } from "../../../val.config"; + +/** + * Support pages use `.jsonValues()`: each page's content lives in its own + * `*.val.json` file and is loaded lazily via `c.json(() => import(...))`. + * This keeps `page.val.ts` tiny even with thousands of support pages. + */ +export const supportPageSchema = s.object({ + title: s.string().minLength(2), + body: s.string(), + order: s.number(), +}); + +export type SupportPage = t.inferSchema; + +export default c.define( + "/app/support/[slug]/page.val.ts", + s.router(nextAppRouter, supportPageSchema).jsonValues(), + { + "/support/getting-started": c.json( + () => import("./content/getting-started.val.json"), + ), + "/support/faq": c.json(() => import("./content/faq.val.json")), + }, +); diff --git a/examples/next/val.config.ts b/examples/next/val.config.ts index ed2762cdb..cfde56f86 100644 --- a/examples/next/val.config.ts +++ b/examples/next/val.config.ts @@ -1,7 +1,7 @@ import { initVal } from "@valbuild/next"; const { s, c, val, config, nextAppRouter, externalPageRouter } = initVal({ - // project: "valbuild/val-examples-next", + project: "valbuild/val-examples-next", root: "/examples/next", defaultTheme: "dark", ai: { diff --git a/examples/next/val.modules.ts b/examples/next/val.modules.ts index 88a1873ba..6c5e8d1f1 100644 --- a/examples/next/val.modules.ts +++ b/examples/next/val.modules.ts @@ -4,6 +4,7 @@ import { config } from "./val.config"; export default modules(config, [ { def: () => import("./content/authors.val") }, { def: () => import("./app/blogs/[blog]/page.val") }, + { def: () => import("./app/support/[slug]/page.val") }, { def: () => import("./app/generic/[[...path]]/page.val") }, { def: () => import("./content/media.val") }, { def: () => import("./app/page.val") }, diff --git a/examples/next/val/rsc.ts b/examples/next/val/rsc.ts index 122752d11..0d2431e86 100644 --- a/examples/next/val/rsc.ts +++ b/examples/next/val/rsc.ts @@ -6,6 +6,7 @@ import { cookies, draftMode, headers } from "next/headers"; const { fetchValStega: fetchVal, + fetchValKeyStega: fetchValKey, fetchValRouteStega: fetchValRoute, fetchValRouteUrl, } = initValRsc(config, valModules, { @@ -14,4 +15,4 @@ const { cookies, }); -export { fetchVal, fetchValRoute, fetchValRouteUrl }; +export { fetchVal, fetchValKey, fetchValRoute, fetchValRouteUrl }; diff --git a/packages/cli/package.json b/packages/cli/package.json index 7c3491e1f..501e31dc8 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -1,7 +1,7 @@ { "name": "@valbuild/cli", "private": false, - "version": "0.97.1", + "version": "0.97.3", "description": "Val CLI tools", "repository": { "type": "git", diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 1ac1a663f..2f38a0782 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -17,6 +17,7 @@ export type { FileMetadata } from "./schema/file"; export type { ValModule, SerializedModule, InferValModuleType } from "./module"; export type { SourceObject, SourcePrimitive, Source } from "./source"; export type { FileSource } from "./source/file"; +export type { JsonSource, JsonOf, JsonImportThunk } from "./source/json"; export type { RemoteSource, RemoteRef } from "./source/remote"; export { DEFAULT_VAL_REMOTE_HOST } from "./schema/remote"; export type { RawString } from "./schema/string"; @@ -98,6 +99,7 @@ import { import { type ImageMetadata } from "./schema/image"; import { type FileMetadata } from "./schema/file"; import { isFile } from "./source/file"; +import { isJson, getJsonImport, resolveJsonValues } from "./source/json"; import { createRemoteRef } from "./source/remote"; import { getValidationBasis, @@ -197,6 +199,9 @@ const Internal = { }, isVal, isFile, + isJson, + getJsonImport, + resolveJsonValues, createValPathOfItem, getSHA256Hash, initSchema, diff --git a/packages/core/src/initVal.ts b/packages/core/src/initVal.ts index ff738279b..307ebe322 100644 --- a/packages/core/src/initVal.ts +++ b/packages/core/src/initVal.ts @@ -3,6 +3,7 @@ import { InitSchema, initSchema } from "./initSchema"; import { getValPath as getPath } from "./val"; import { initFile } from "./source/file"; import { initImage } from "./source/image"; +import { json } from "./source/json"; import { initRemote } from "./source/remote"; // import { i18n, I18n } from "./source/future/i18n"; // import { remote } from "./source/future/remote"; @@ -13,6 +14,7 @@ export type ContentConstructor = { file: ReturnType; image: ReturnType; remote: ReturnType; + json: typeof json; }; export type ValConstructor = { unstable_getPath: typeof getPath; @@ -96,6 +98,7 @@ InitVal => { remote: initRemote(config), file: initFile(config), image: initImage(config), + json, }, s, config, diff --git a/packages/core/src/module.ts b/packages/core/src/module.ts index 9cae09e7a..9806a0b2f 100644 --- a/packages/core/src/module.ts +++ b/packages/core/src/module.ts @@ -20,6 +20,7 @@ import { SerializedImageSchema, } from "./schema/image"; import { FILE_REF_PROP, FileSource } from "./source/file"; +import { isJson } from "./source/json"; import { AllRichTextOptions, RichTextSource } from "./source/richtext"; import { RecordSchema, SerializedRecordSchema } from "./schema/record"; import { RawString } from "./schema/string"; @@ -342,6 +343,15 @@ export function resolvePath< resolvedSchema instanceof RecordSchema ? resolvedSchema?.["item"] : resolvedSchema.item; + // A `.jsonValues()` entry value is a lazily-loaded JsonSource marker. We + // can resolve the entry itself (schema is now `item`), but we cannot + // descend deeper until the backing `*.val.json` content has been loaded + // and substituted into the source tree (done by the UI/server layer). + if (isJson(resolvedSource) && parts.length > 0) { + throw Error( + `Cannot resolve path into a jsonValues entry until its content is loaded. Path: ${path}`, + ); + } } else if (isObjectSchema(resolvedSchema)) { if (typeof resolvedSource !== "object") { throw Error( @@ -593,6 +603,14 @@ export function safeResolvePath< resolvedSchema instanceof RecordSchema ? resolvedSchema?.["item"] : resolvedSchema.item; + // jsonValues entry: cannot descend deeper until the backing `*.val.json` + // content is loaded into the source tree. + if (isJson(resolvedSource) && parts.length > 0) { + return { + status: "error", + message: `Cannot resolve path into a jsonValues entry until its content is loaded. Path: ${path}`, + }; + } } else if (isObjectSchema(resolvedSchema)) { if (resolvedSource === undefined) { return { diff --git a/packages/core/src/schema/deserialize.ts b/packages/core/src/schema/deserialize.ts index 8da5e0a92..98d593ab5 100644 --- a/packages/core/src/schema/deserialize.ts +++ b/packages/core/src/schema/deserialize.ts @@ -179,6 +179,7 @@ function deserializeSchemaImpl( false, false, serialized.description, + serialized.jsonValues ?? false, ); case "keyOf": return new KeyOfSchema( diff --git a/packages/core/src/schema/jsonValues.test.ts b/packages/core/src/schema/jsonValues.test.ts new file mode 100644 index 000000000..6322113af --- /dev/null +++ b/packages/core/src/schema/jsonValues.test.ts @@ -0,0 +1,196 @@ +import { SourcePath } from "../val"; +import { + json, + isJson, + getJsonImport, + resolveJsonValues, + JsonOf, +} from "../source/json"; +import { Source } from "../source"; +import { VAL_EXTENSION } from "../source"; +import { deserializeSchema } from "./deserialize"; +import { Schema } from "."; +import { SelectorSource } from "../selector"; +import { object } from "./object"; +import { record } from "./record"; +import { router } from "./router"; +import { string } from "./string"; +import { images } from "./images"; +import { nextAppRouter } from "../router"; +import { initVal } from "../initVal"; + +describe("c.json + .jsonValues()", () => { + test("json() returns a JsonSource marker with a thunk", () => { + const thunk = () => Promise.resolve({ default: { title: "hi" } }); + const src = json(thunk); + expect(src[VAL_EXTENSION]).toBe("json"); + expect(getJsonImport(src)).toBe(thunk); + expect(isJson(src)).toBe(true); + expect(isJson({ title: "hi" })).toBe(false); + expect(isJson(null)).toBe(false); + }); + + test(".jsonValues() serializes with jsonValues: true", () => { + const schema = record(object({ title: string() })).jsonValues(); + const serialized = schema["executeSerialize"](); + expect(serialized.type).toBe("record"); + expect(serialized.jsonValues).toBe(true); + // item schema is still serialized so the UI/validation can use it + expect(serialized.item.type).toBe("object"); + }); + + test("non-jsonValues record does not set jsonValues", () => { + const schema = record(object({ title: string() })); + const serialized = schema["executeSerialize"](); + expect(serialized.jsonValues).toBeUndefined(); + }); + + test("deserialize round-trips jsonValues flag", () => { + const schema = record(object({ title: string() })).jsonValues(); + const serialized = schema["executeSerialize"](); + const deserialized = deserializeSchema(serialized); + const reserialized = deserialized["executeSerialize"](); + expect(reserialized.type).toBe("record"); + if (reserialized.type === "record") { + expect(reserialized.jsonValues).toBe(true); + } + }); + + test(".jsonValues() composes with s.router()", () => { + const schema = router( + nextAppRouter, + object({ title: string() }), + ).jsonValues(); + const serialized = schema["executeSerialize"](); + expect(serialized.jsonValues).toBe(true); + expect(serialized.router).toBe("next-app-router"); + }); + + test(".jsonValues() throws on image galleries", () => { + expect(() => images().jsonValues()).toThrow(/jsonValues/); + }); + + describe("validation", () => { + const schema = record(object({ title: string() })).jsonValues(); + // A loosely-typed reference so we can feed untyped values (as the server + // does when validating sources loaded over the wire) without casts. + const looseSchema: Schema = schema; + + test("accepts json markers and never loads content during validation", () => { + let loaded = false; + const src = { + a: json(() => { + // deep content is never inspected at the record level — deferred to + // validateJsonEntryContent once the file is loaded. + loaded = true; + return Promise.resolve({ default: { title: "ok" } }); + }), + }; + const errors = schema["executeValidate"]( + "/test.val.ts" as SourcePath, + src, + ); + expect(errors).toBe(false); + expect(loaded).toBe(false); + }); + + test("validates inline (loaded) content against the item schema", () => { + // valid inline content passes (this is what the UI sees once an entry's + // *.val.json is loaded and substituted for its marker) + expect( + looseSchema["executeValidate"]("/test.val.ts" as SourcePath, { + a: { title: "ok" }, + }), + ).toBe(false); + // invalid inline content (wrong leaf type) is caught + expect( + looseSchema["executeValidate"]("/test.val.ts" as SourcePath, { + a: { title: 123 }, + }), + ).not.toBe(false); + }); + + test("validateJsonEntryContent validates loaded content against item", () => { + const okErrors = schema.validateJsonEntryContent( + '/test.val.ts?p="a"' as SourcePath, + { title: "ok" }, + ); + expect(okErrors).toBe(false); + + const badErrors = schema.validateJsonEntryContent( + '/test.val.ts?p="a"' as SourcePath, + // wrong leaf type + { title: 123 }, + ); + expect(badErrors).not.toBe(false); + }); + }); +}); + +describe("resolveJsonValues", () => { + test("resolves a record of markers into inlined content", async () => { + const source: Source = { + "/a": json(() => Promise.resolve({ default: { title: "A" } })), + "/b": json(() => Promise.resolve({ default: { title: "B" } })), + }; + const resolved = await resolveJsonValues(source); + expect(resolved).toEqual({ "/a": { title: "A" }, "/b": { title: "B" } }); + }); + + test("resolves nested markers recursively", async () => { + const source: Source = { + "/outer": json(() => + Promise.resolve({ + default: { + inner: json(() => Promise.resolve({ default: { deep: "value" } })), + }, + }), + ), + }; + const resolved = await resolveJsonValues(source); + expect(resolved).toEqual({ "/outer": { inner: { deep: "value" } } }); + }); + + test("leaves transport markers (no thunk) as-is", async () => { + const marker = { _type: "json" } as unknown as Source; + const resolved = await resolveJsonValues({ "/a": marker }); + expect(resolved).toEqual({ "/a": { _type: "json" } }); + }); +}); + +describe("c.define authoring surface (compile-time)", () => { + test("define accepts c.json entries for a .jsonValues() router", () => { + const { s, c } = initVal(); + // Simulates `import("./blogs/test.val.json")` whose JSON-inferred type is + // the widened content shape. + const mod = c.define( + "/blogs/[slug]/page.val.ts", + s.router(nextAppRouter, s.object({ title: s.string() })).jsonValues(), + { + "/blogs/test": c.json(() => + Promise.resolve({ default: { title: "Hello" } }), + ), + }, + ); + expect(mod).toBeDefined(); + }); +}); + +describe("JsonOf type transform", () => { + test("widens literals, keeps structure (compile-time)", () => { + // string-literal union -> string + const a: JsonOf<"x" | "y"> = "anything"; + // number literal -> number + const b: JsonOf<1 | 2> = 5; + // object structure preserved, leaves widened + const c: JsonOf<{ kind: "a"; n: 1 }> = { kind: "whatever", n: 42 }; + // array of widened + const d: JsonOf = ["x", "y"]; + // discriminated union (distributes + recurses) + const e: JsonOf<{ k: "a"; x: number } | { k: "b"; y: string }> = { + k: "a", + x: 1, + }; + expect([a, b, c, d, e]).toBeDefined(); + }); +}); diff --git a/packages/core/src/schema/record.ts b/packages/core/src/schema/record.ts index 7b1aa3794..259b20e66 100644 --- a/packages/core/src/schema/record.ts +++ b/packages/core/src/schema/record.ts @@ -14,6 +14,7 @@ import { unsafeCreateSourcePath, } from "../selector/SelectorProxy"; import { ImageSource } from "../source/image"; +import { JsonOf, JsonSource, isJson } from "../source/json"; import { RemoteSource } from "../source/remote"; import { ModuleFilePath, SourcePath } from "../val"; import { ImageMetadata } from "./image"; @@ -44,15 +45,31 @@ export type SerializedRecordSchema = { directory?: string; remote?: boolean; alt?: SerializedSchema; + // When true, entry values are stored in separate lazily-loaded `*.val.json` + // files (see `.jsonValues()`). + jsonValues?: boolean; readonly?: boolean; hidden?: boolean; description?: string; }; +/** + * The source type of a `.jsonValues()` record: every entry value is a lazily + * loaded {@link JsonSource} whose resolved content is the (loosened, see + * {@link JsonOf}) item type. + */ +export type JsonValuesRecordSrc< + T extends Schema, + K extends Schema, +> = Record, JsonSource>>>; + export class RecordSchema< T extends Schema, K extends Schema, - Src extends Record, SelectorOfSchema> | null, + Src extends + | Record, SelectorOfSchema> + | JsonValuesRecordSrc + | null, > extends Schema { constructor( private readonly item: T, @@ -64,6 +81,8 @@ export class RecordSchema< private readonly isReadonly: boolean = false, private readonly isHidden: boolean = false, private readonly description?: string, + /** When true, entry values are lazily loaded {@link JsonSource} thunks. */ + private readonly isJsonValues: boolean = false, ) { super(); } @@ -79,6 +98,7 @@ export class RecordSchema< this.isReadonly, this.isHidden, description ?? undefined, + this.isJsonValues, ); } @@ -95,6 +115,7 @@ export class RecordSchema< this.isReadonly, this.isHidden, this.description, + this.isJsonValues, ); } @@ -238,6 +259,26 @@ export class RecordSchema< this.markKeyErrorsAtPath(entryErr, subPath); error = error ? { ...error, ...entryErr } : entryErr; } + } else if (this.isJsonValues && isJson(elem)) { + // jsonValues record, entry not loaded: the value is a lazy JsonSource + // marker. Deep validation is deferred and run per-entry once the backing + // `*.val.json` is loaded (server: validateJsonEntryContent; UI: the + // loaded content is substituted and validated by the branch below). + } else if (this.isJsonValues) { + // jsonValues record, entry content is inlined (loaded in the UI, or + // hand-authored content) — validate it against the item schema. + const subError = this.item["executeValidate"]( + subPath, + elem as SelectorSource, + ); + if (subError && error) { + error = { + ...subError, + ...error, + }; + } else if (subError) { + error = subError; + } } else { const subError = this.item["executeValidate"]( subPath, @@ -532,6 +573,7 @@ export class RecordSchema< this.isReadonly, this.isHidden, this.description, + this.isJsonValues, ) as RecordSchema; } @@ -546,6 +588,7 @@ export class RecordSchema< true, this.isHidden, this.description, + this.isJsonValues, ); } @@ -560,6 +603,7 @@ export class RecordSchema< this.isReadonly, true, this.description, + this.isJsonValues, ); } @@ -574,6 +618,7 @@ export class RecordSchema< this.isReadonly, this.isHidden, this.description, + this.isJsonValues, ); } @@ -588,9 +633,46 @@ export class RecordSchema< this.isReadonly, this.isHidden, this.description, + this.isJsonValues, ); } + /** + * Store each entry's value in its own lazily-loaded `*.val.json` file instead + * of inlining it in the `.val.ts` module. Entry values become + * {@link JsonSource} thunks (`c.json(() => import("./entry.val.json"))`), + * which lets the runtime, the Studio and validation work one entry at a time + * so a record/router can scale to many thousands of entries. + * + * Not supported on image/file galleries (`s.images()` / `s.files()`). + * + * Only supported on a module's ROOT record/router — a `.jsonValues()` record + * nested inside an object/array/record is rejected at startup with a module + * error, because the single-entry fetch endpoint, the Studio's content + * substitution and content validation are all root-only. + */ + jsonValues(): RecordSchema> { + if (this.mediaOptions) { + throw new Error( + ".jsonValues() cannot be used with image/file galleries (s.images()/s.files())", + ); + } + return new RecordSchema( + this.item, + this.opt, + // custom validate functions are typed against the previous Src; drop them + // since the source shape changes to JsonSource entries. + [], + this.currentRouter, + this.keySchema, + this.mediaOptions, + this.isReadonly, + this.isHidden, + this.description, + true, + ) as RecordSchema>; + } + private getRouterValidations(path: SourcePath, src: Src): ValidationErrors { if (!this.currentRouter) { return false; @@ -663,6 +745,7 @@ export class RecordSchema< customValidate: this.customValidateFunctions && this.customValidateFunctions?.length > 0, + jsonValues: this.isJsonValues ? true : undefined, readonly: this.isReadonly, hidden: this.isHidden, description: this.description, @@ -688,6 +771,19 @@ export class RecordSchema< }; } | null = null; + /** + * Validate the loaded content of a single `.jsonValues()` entry against the + * item schema. The server calls this once it has loaded the backing + * `*.val.json` for an entry (the deep validation that `executeValidate` + * defers). + */ + validateJsonEntryContent( + path: SourcePath, + content: SelectorSource, + ): ValidationErrors { + return this.item["executeValidate"](path, content); + } + protected override executeRender( sourcePath: SourcePath | ModuleFilePath, src: Src, @@ -696,6 +792,11 @@ export class RecordSchema< if (src === null) { return res; } + if (this.isJsonValues) { + // jsonValues entries are not inlined, so there is no per-entry source to + // render at the record level. Rendering happens per entry once loaded. + return res; + } for (const key in src) { const itemSrc = src[key as unknown as SelectorOfSchema]; if (itemSrc === null || itemSrc === undefined) { diff --git a/packages/core/src/selector/index.ts b/packages/core/src/selector/index.ts index aea8ad54d..8519603ac 100644 --- a/packages/core/src/selector/index.ts +++ b/packages/core/src/selector/index.ts @@ -15,6 +15,7 @@ import { ImageSelector } from "./image"; import { RichTextSelector } from "./richtext"; import { ImageSource } from "../source/image"; import { RemoteSource } from "../source/remote"; +import { JsonSource } from "../source/json"; export type Selector = Source extends T ? GenericSelector @@ -22,21 +23,23 @@ export type Selector = Source extends T ? ImageSelector : T extends FileSource ? FileSelector - : T extends RichTextSource - ? RichTextSelector - : T extends SourceObject - ? ObjectSelector - : T extends SourceArray - ? ArraySelector - : T extends string - ? StringSelector - : T extends number - ? NumberSelector - : T extends boolean - ? BooleanSelector - : T extends null - ? PrimitiveSelector - : never; + : T extends JsonSource + ? GenericSelector + : T extends RichTextSource + ? RichTextSelector + : T extends SourceObject + ? ObjectSelector + : T extends SourceArray + ? ArraySelector + : T extends string + ? StringSelector + : T extends number + ? NumberSelector + : T extends boolean + ? BooleanSelector + : T extends null + ? PrimitiveSelector + : never; export type SelectorSource = | SourcePrimitive @@ -48,6 +51,7 @@ export type SelectorSource = | ImageSource | FileSource | RemoteSource + | JsonSource | RichTextSource | GenericSelector; diff --git a/packages/core/src/source/index.ts b/packages/core/src/source/index.ts index 47de4448a..f8a4d0eb8 100644 --- a/packages/core/src/source/index.ts +++ b/packages/core/src/source/index.ts @@ -1,5 +1,6 @@ import { FileSource } from "./file"; import { ImageSource } from "./image"; +import { JsonSource } from "./json"; import { RemoteSource } from "./remote"; import { RichTextOptions, RichTextSource } from "./richtext"; @@ -10,6 +11,7 @@ export type Source = | RemoteSource | FileSource | ImageSource + | JsonSource | RichTextSource; export type SourceObject = { [key in string]: Source } & { diff --git a/packages/core/src/source/json.ts b/packages/core/src/source/json.ts new file mode 100644 index 000000000..6494f5220 --- /dev/null +++ b/packages/core/src/source/json.ts @@ -0,0 +1,136 @@ +import { PhantomType, Source, VAL_EXTENSION } from "."; +import { Json } from "../Json"; + +/** + * The string used as the `_type` discriminator of a {@link JsonSource}. + */ +export const JSON_VAL_EXTENSION_TAG = "json" as const; + +/** + * The lazy import thunk a {@link JsonSource} carries at runtime. It is NOT part + * of the {@link JsonSource} type because `Source` must stay JSON-serializable + * (it is sent over the wire as `Json`); the thunk is a runtime-only detail, + * read via {@link getJsonImport}. + */ +export type JsonImportThunk = () => Promise<{ default: T }>; + +/** + * A JSON source represents a record/router entry whose value is stored in a + * separate `*.val.json` file and loaded lazily via a dynamic `import()` thunk. + * + * This is what `c.json(() => import("./entry.val.json"))` returns. + * + * The *type* is a pure-JSON marker (`{ _type: "json", patch_id? }`) so that + * `Source` stays JSON-serializable. At runtime the value additionally carries a + * lazy import thunk (read via {@link getJsonImport}); over the wire only the + * marker is sent. + * + * The phantom type parameter `T` is the (loosened, see {@link JsonOf}) type of + * the value the JSON file resolves to. It is covariant: a `JsonSource` of a + * narrower value type is assignable to a `JsonSource` of a wider one. + */ +export type JsonSource = { + readonly [VAL_EXTENSION]: typeof JSON_VAL_EXTENSION_TAG; + /** Set on uncommitted/draft entries (mirrors FileSource/RemoteSource). */ + readonly patch_id?: string; +} & PhantomType; + +export function json(importThunk: JsonImportThunk): JsonSource { + return { + [VAL_EXTENSION]: JSON_VAL_EXTENSION_TAG, + // runtime-only: not described by JsonSource so Source stays JSON-serializable + _import: importThunk, + } as unknown as JsonSource; +} + +export function isJson(obj: unknown): obj is JsonSource { + return ( + typeof obj === "object" && + obj !== null && + VAL_EXTENSION in obj && + (obj as { [VAL_EXTENSION]?: unknown })[VAL_EXTENSION] === + JSON_VAL_EXTENSION_TAG + ); +} + +/** + * Reads the runtime-only lazy import thunk from a {@link JsonSource}. Returns + * `undefined` for a transport marker (sent over the wire without the thunk). + */ +export function getJsonImport(source: JsonSource): JsonImportThunk | undefined { + const candidate = (source as { _import?: unknown })._import; + return typeof candidate === "function" + ? (candidate as JsonImportThunk) + : undefined; +} + +/** + * Recursively resolves every {@link JsonSource} marker in a source tree by + * invoking its lazy import thunk, returning a fully-inlined source. This backs + * the eager `fetchVal`/`useVal` path (load everything). + * + * Markers without a runtime thunk (transport markers received over the wire) + * are left as-is — there is nothing to load from them locally. Other branded + * leaf sources (file/image/remote/richtext) are returned untouched. + */ +export async function resolveJsonValues(source: Source): Promise { + if (isJson(source)) { + const thunk = getJsonImport(source); + if (!thunk) { + return source; + } + const loaded = (await thunk()).default; + return resolveJsonValues(loaded as Source); + } + if (Array.isArray(source)) { + return Promise.all(source.map((item) => resolveJsonValues(item))); + } + if (source !== null && typeof source === "object") { + if (VAL_EXTENSION in source) { + // a branded leaf source (file/image/remote) — no nested json markers + return source; + } + const entries = await Promise.all( + Object.entries(source).map( + async ([key, value]) => + [key, await resolveJsonValues(value as Source)] as const, + ), + ); + return Object.fromEntries(entries) as Source; + } + return source; +} + +/** + * Loosens a (strict) schema source type `T` into the type that a + * `resolveJsonModule` import of the backing `*.val.json` can actually satisfy. + * + * Why this is needed: TypeScript widens literals when it infers the type of a + * JSON module (e.g. `"a"` becomes `string`, `1` becomes `number`, and a branded + * `_type: "file"` becomes `_type: string`). A strict schema type (with literal + * unions, `RawString` brands, etc.) would therefore reject a perfectly valid + * JSON file. `JsonOf` performs the same widening at the type level so the + * JSON content typechecks against the schema as strictly as JSON allows. Runtime + * validation enforces the rest. + * + * The transform distributes over unions and recurses through objects/arrays, + * preserving structure while widening every leaf literal. Val object-unions are + * always discriminated, so distribute-and-recurse is sufficient (the discriminant + * literal widens to its base type, but the per-variant shape stays distinct and a + * concrete JSON value still matches exactly one variant). + */ +export type JsonOf = T extends string + ? string + : T extends number + ? number + : T extends boolean + ? boolean + : T extends null + ? null + : T extends undefined + ? undefined + : T extends readonly (infer E)[] + ? JsonOf[] + : T extends object + ? { [K in keyof T]: JsonOf } + : T; diff --git a/packages/next/package.json b/packages/next/package.json index 300a91fb5..2b302f97c 100644 --- a/packages/next/package.json +++ b/packages/next/package.json @@ -12,7 +12,7 @@ "next", "react" ], - "version": "0.97.1", + "version": "0.97.3", "scripts": { "typecheck": "tsc --noEmit", "test": "jest" diff --git a/packages/next/src/client/initValClient.ts b/packages/next/src/client/initValClient.ts index 7dcfcf650..23fbcf93f 100644 Binary files a/packages/next/src/client/initValClient.ts and b/packages/next/src/client/initValClient.ts differ diff --git a/packages/next/src/routeFromVal.ts b/packages/next/src/routeFromVal.ts index 7ad95ce21..b4c072b99 100644 --- a/packages/next/src/routeFromVal.ts +++ b/packages/next/src/routeFromVal.ts @@ -6,6 +6,48 @@ import { RoutePattern, } from "@valbuild/shared/internal"; +/** + * True when the module's schema is a `.jsonValues()` record/router — i.e. each + * entry value is a lazily-loaded `c.json(() => import(...))` marker. Used by the + * route helpers to load ONLY the matched entry instead of the whole record. + */ +export function isJsonValuesRecordSchema(schema: unknown): boolean { + return ( + schema instanceof RecordSchema && + schema["executeSerialize"]().jsonValues === true + ); +} + +/** + * Builds the `stegaEncode` `root` seed for a single `.jsonValues()` entry, so its + * strings get edit tags. Without a seed, `stegaEncode` on raw entry content is an + * identity transform (the content carries no selector path/schema). + * + * Yields e.g. `/app/support/[slug]/page.val.ts?p="/support/faq"`, so a `title` + * field is tagged `…?p="/support/faq"."title"` — the shape the Studio's + * `findUnloadedJsonEntryKey` walks, so click-to-edit + lazy load line up. + */ +export function getJsonEntryStegaRoot( + // eslint-disable-next-line @typescript-eslint/no-explicit-any + selector: any, + key: string, + // eslint-disable-next-line @typescript-eslint/no-explicit-any +): { path: any; schema: any } | undefined { + const modulePath = selector && Internal.getValPath(selector); + if (!modulePath) { + return undefined; + } + const schema = selector && Internal.getSchema(selector); + if (!(schema instanceof RecordSchema)) { + return undefined; + } + const path = Internal.createValPathOfItem(modulePath, key); + if (!path) { + return undefined; + } + return { path, schema: schema["executeSerialize"]().item }; +} + export function getValRouteUrlFromVal( resolvedParams: Record | unknown, methodName: string, diff --git a/packages/next/src/rsc/initValRsc.ts b/packages/next/src/rsc/initValRsc.ts index 8472ed957..ec58bb6ed 100644 --- a/packages/next/src/rsc/initValRsc.ts +++ b/packages/next/src/rsc/initValRsc.ts @@ -14,12 +14,18 @@ import { Internal, ValModule, SourceObject, + JsonSource, } from "@valbuild/core"; import { cookies, draftMode, headers } from "next/headers"; import { VAL_SESSION_COOKIE } from "@valbuild/shared/internal"; import { createValServer, ValServer } from "@valbuild/server"; import { VERSION } from "../version"; -import { getValRouteUrlFromVal, initValRouteFromVal } from "../routeFromVal"; +import { + getJsonEntryStegaRoot, + getValRouteUrlFromVal, + initValRouteFromVal, + isJsonValuesRecordSchema, +} from "../routeFromVal"; SET_RSC(true); const initFetchValStega = @@ -173,7 +179,10 @@ type FetchValRouteReturnType< > = T extends ValModule ? S extends SourceObject - ? StegaOfSource[string]> | null + ? // `.jsonValues()` router: the matched entry resolves to its json content. + NonNullable[string] extends JsonSource + ? C | null + : StegaOfSource[string]> | null : never : never; @@ -197,6 +206,51 @@ const initFetchValRouteStega = | Record | unknown, ): Promise> => { + const resolvedParams = await Promise.resolve(params); + const path = selector && Internal.getValPath(selector); + const schema = selector && Internal.getSchema(selector); + // `.jsonValues()` router: map params → the entry key and load ONLY that + // entry's backing `*.val.json`, instead of eagerly resolving the whole + // record via `fetchVal`. + if (isJsonValuesRecordSchema(schema)) { + const source = selector && Internal.getSource(selector); + const url = getValRouteUrlFromVal( + resolvedParams, + "fetchValRoute", + path, + schema, + source, + ); + if (!url) { + return null as FetchValRouteReturnType; + } + let enabled = false; + try { + enabled = await isEnabled(); + } catch { + // not in a server context where draftMode is readable — treat as disabled + } + let content: unknown = undefined; + if (enabled && path) { + SET_AUTO_TAG_JSX_ENABLED(true); + content = await loadDraftJsonEntry( + valServerPromise, + getCookies, + path as unknown as ModuleFilePath, + url, + ); + } + if (content === undefined) { + content = await loadJsonEntryContent(source, url); + } + if (content === undefined) { + return null as FetchValRouteReturnType; + } + return stegaEncode(content, { + disabled: !enabled, + root: getJsonEntryStegaRoot(selector, url), + }); + } const fetchVal = initFetchValStega( config, valApiEndpoints, @@ -205,9 +259,6 @@ const initFetchValRouteStega = getHeaders, getCookies, ); - const resolvedParams = await Promise.resolve(params); - const path = selector && Internal.getValPath(selector); - const schema = selector && Internal.getSchema(selector); const val = selector && (await fetchVal(selector)); const route = initValRouteFromVal( resolvedParams, @@ -219,6 +270,147 @@ const initFetchValRouteStega = return route; }; +/** + * Resolves a single `.jsonValues()` entry's content by key from a module's local + * source markers (one dynamic import). Returns `undefined` when the key is + * missing or its marker has no runtime thunk (transport marker / draft entry). + */ +async function loadJsonEntryContent( + source: unknown, + key: string, +): Promise { + if (!source || typeof source !== "object") { + return undefined; + } + const marker = (source as Record)[key]; + if (!Internal.isJson(marker)) { + return undefined; + } + const thunk = Internal.getJsonImport(marker); + if (!thunk) { + return undefined; + } + return (await thunk()).default; +} + +/** + * Loads a single `.jsonValues()` entry's DRAFT content via the in-process + * `/json` endpoint (which replays pending patches). Returns `undefined` when the + * entry has no draft content to serve — the caller then falls back to the + * locally-bundled committed content. + */ +async function loadDraftJsonEntry( + valServerPromise: Promise, + getCookies: () => Promise<{ + get(name: string): { name: string; value: string } | undefined; + }>, + moduleFilePath: ModuleFilePath, + key: string, +): Promise { + let cookies; + try { + cookies = await getCookies(); + } catch { + // not in a server context where cookies are readable + return undefined; + } + const valServer = await valServerPromise; + const res = await valServer["/json"]["GET"]({ + query: { + path: moduleFilePath, + key, + keys: undefined, // single-entry shape + offset: undefined, + limit: undefined, + apply_patches: true, + }, + cookies: { + [VAL_SESSION_COOKIE]: cookies?.get(VAL_SESSION_COOKIE)?.value, + }, + }); + if (res.status === 200 && "content" in res.json) { + return res.json.content; + } + if (res.status === 401) { + console.warn("Val: authentication error: ", res.json.message); + return undefined; + } + if (res.status === 404) { + // No such entry in the draft state (e.g. removed by a pending patch). + return undefined; + } + console.error( + "Val: could not load draft JSON entry: ", + "message" in res.json ? res.json.message : `status ${res.status}`, + ); + return undefined; +} + +// The (loosened) content type a single `.jsonValues()` entry resolves to. +type JsonEntryContentOf>> = + T extends ValModule + ? S extends Record + ? V extends JsonSource + ? C + : never + : never + : never; + +/** + * Resolves ONE `.jsonValues()` entry by key, loading only that entry instead of + * the whole record — the runtime-scaling counterpart to the eager `fetchVal`. + * + * Production (Val disabled): resolves the entry's lazy import thunk from the + * locally-bundled module. One dynamic import, no server round-trip. + * + * Enabled (draft mode): reads the entry through `/json`, which replays pending + * patches, so uncommitted Studio edits show up. Falls back to the local thunk if + * the draft read yields nothing. + */ +const initFetchValKeyStega = + ( + valServerPromise: Promise, + isEnabled: () => Promise, + getCookies: () => Promise<{ + get(name: string): { name: string; value: string } | undefined; + }>, + ) => + async >>( + selector: T, + key: string, + ): Promise | undefined> => { + let enabled = false; + try { + enabled = await isEnabled(); + } catch { + // not in a server context where draftMode is readable — treat as disabled + } + const source = selector && Internal.getSource(selector); + const moduleFilePath = + selector && (Internal.getValPath(selector) as unknown as ModuleFilePath); + let content: unknown = undefined; + if (enabled && moduleFilePath) { + SET_AUTO_TAG_JSX_ENABLED(true); + content = await loadDraftJsonEntry( + valServerPromise, + getCookies, + moduleFilePath, + key, + ); + } + if (content === undefined) { + content = await loadJsonEntryContent(source, key); + } + if (content === undefined) { + // missing key, or transport marker without a runtime thunk + return undefined; + } + return stegaEncode(content, { + disabled: !enabled, + root: getJsonEntryStegaRoot(selector, key), + }); + }; + const initFetchValRouteUrl = ( config: ValConfig, @@ -276,6 +468,7 @@ export function initValRsc( rscNextConfig: ValNextRscConfig, ): { fetchValStega: ReturnType; + fetchValKeyStega: ReturnType; fetchValRouteStega: ReturnType; fetchValRouteUrl: ReturnType; } { @@ -326,6 +519,15 @@ export function initValRsc( return await rscNextConfig.cookies(); }, ), + fetchValKeyStega: initFetchValKeyStega( + valServerPromise, + async () => { + return (await rscNextConfig.draftMode()).isEnabled; + }, + async () => { + return await rscNextConfig.cookies(); + }, + ), fetchValRouteStega: initFetchValRouteStega( config, valApiEndpoints, diff --git a/packages/react/package.json b/packages/react/package.json index f2acaaafa..082d242d3 100644 --- a/packages/react/package.json +++ b/packages/react/package.json @@ -1,6 +1,6 @@ { "name": "@valbuild/react", - "version": "0.97.1", + "version": "0.97.3", "private": false, "description": "Val - React internal helpers", "repository": { diff --git a/packages/react/src/stega/stegaEncode.test.ts b/packages/react/src/stega/stegaEncode.test.ts index 3263e9add..e51534b30 100644 --- a/packages/react/src/stega/stegaEncode.test.ts +++ b/packages/react/src/stega/stegaEncode.test.ts @@ -293,5 +293,48 @@ describe("stega transform", () => { }); }); +describe("stegaEncode root seed (jsonValues entries)", () => { + // A `.jsonValues()` entry's content is plain JSON — it carries no selector + // path/schema — so without a `root` seed stegaEncode cannot tag anything. + const itemSchema = s.object({ title: s.string(), body: s.string() }); + const entryPath = '/app/support/[slug]/page.val.ts?p="/support/faq"'; + const content = { title: "FAQ", body: "Body" }; + + test("without a root seed it is an identity transform (the bug)", () => { + const res = stegaEncode(content, {}); + expect(res).toEqual(content); + expect(vercelStegaDecode(res.title)).toBeUndefined(); + }); + + test("with a root seed each string is tagged at the entry sub-path", () => { + const res = stegaEncode(content, { + root: { + path: entryPath, + schema: itemSchema["executeSerialize"](), + }, + }); + expect(vercelStegaSplit(res.title).cleaned).toBe("FAQ"); + expect(vercelStegaDecode(res.title)).toEqual({ + origin: "val.build", + data: { valPath: `${entryPath}."title"` }, + }); + expect(vercelStegaDecode(res.body)).toEqual({ + origin: "val.build", + data: { valPath: `${entryPath}."body"` }, + }); + }); + + test("disabled wins over the root seed", () => { + const res = stegaEncode(content, { + disabled: true, + root: { + path: entryPath, + schema: itemSchema["executeSerialize"](), + }, + }); + expect(res).toEqual(content); + }); +}); + type SchemaOf> = T extends Schema ? S : never; diff --git a/packages/react/src/stega/stegaEncode.ts b/packages/react/src/stega/stegaEncode.ts index fb532aa59..f4f1a6fac 100644 --- a/packages/react/src/stega/stegaEncode.ts +++ b/packages/react/src/stega/stegaEncode.ts @@ -351,6 +351,19 @@ export function stegaEncode( opts: { getModule?: (modulePath: string) => any; disabled?: boolean; + /** + * Seeds the recursion for a RAW source value that is not a selector, so its + * strings still get edit tags. + * + * Needed for a `.jsonValues()` entry loaded by key: its content is plain + * JSON with no `Path`/`GetSchema` symbols, so the selector branch below + * cannot fire and — without this — every string hits the `!recOpts` bail in + * the encoder and the whole call is an identity transform. + * + * `path` is the entry's path (`Internal.createValPathOfItem(modulePath, key)`) + * and `schema` is the SERIALIZED item schema. + */ + root?: { path: any; schema: any }; }, ): any { function rec( @@ -528,7 +541,7 @@ export function stegaEncode( ); return sourceOrSelector; } - return rec(input); + return rec(input, opts.disabled ? undefined : opts.root); } function isRecordSchema( diff --git a/packages/server/package.json b/packages/server/package.json index 59a291060..25b82e686 100644 --- a/packages/server/package.json +++ b/packages/server/package.json @@ -16,7 +16,7 @@ "./package.json": "./package.json" }, "types": "dist/valbuild-server.cjs.d.ts", - "version": "0.97.1", + "version": "0.97.3", "scripts": { "typecheck": "tsc --noEmit", "test": "jest", diff --git a/packages/server/src/ValOps.ts b/packages/server/src/ValOps.ts index 4911d15b0..d4233e344 100644 --- a/packages/server/src/ValOps.ts +++ b/packages/server/src/ValOps.ts @@ -12,6 +12,7 @@ import { RemoteSource, Schema, SelectorSource, + SerializedSchema, Source, SourcePath, VAL_EXTENSION, @@ -21,10 +22,11 @@ import { ValidationErrors, extractValModules, } from "@valbuild/core"; -import { pipe, result } from "@valbuild/core/fp"; +import { array, pipe, result } from "@valbuild/core/fp"; import { JSONOps, JSONValue, + Operation, ParentRef, Patch, PatchError, @@ -32,8 +34,18 @@ import { applyPatch, deepClone, } from "@valbuild/core/patch"; -import { TSOps } from "./patch/ts/ops"; +import { TSOps, insertValJsonEntry, removeValJsonEntry } from "./patch/ts/ops"; import { analyzeValModule } from "./patch/ts/valModule"; +import { analyzeJsonValuesEntries } from "./patch/ts/jsonValuesModule"; +import { + applyJsonValuesEntryPatches, + classifyJsonValuesOp, + findNestedJsonValuesRecords, + getNewJsonEntryPaths, + rebaseContentOp, + resolveExistingJsonPath, +} from "./patch/jsonValuesPatch"; +import { validateJsonValuesEntries } from "./validateJsonValues"; import ts from "typescript"; import { ValSyntaxError, ValSyntaxErrorTree } from "./patch/ts/syntax"; import sizeOf from "image-size"; @@ -70,6 +82,40 @@ const tsOps = new TSOps((document) => { ); }); +/** + * `.jsonValues()` is only supported on a module's ROOT record/router. A nested + * one is broken end to end (the `/json` endpoint keys entries by a single + * string, the Studio substitutes at the top level, and content validation + * silently skips it), so reject it up front as a module error — `/sources/~` + * then fails with "Val is not correctly setup" naming the module. + */ +function findNestedJsonValuesModuleErrors(schemas: Schemas): ModulesError[] { + const errors: ModulesError[] = []; + for (const moduleFilePathS of Object.keys(schemas)) { + const moduleFilePath = moduleFilePathS as ModuleFilePath; + const schema = schemas[moduleFilePath]; + if (!schema) { + continue; + } + let serialized: SerializedSchema; + try { + serialized = schema["executeSerialize"](); + } catch { + // Serialization errors are reported elsewhere (e.g. by extractValModules). + continue; + } + for (const nestedPath of findNestedJsonValuesRecords(serialized)) { + errors.push({ + path: moduleFilePath, + message: `Nested .jsonValues() records are not supported: '${nestedPath.join( + ".", + )}' in ${moduleFilePath}. Use .jsonValues() only on a module's root record/router.`, + }); + } + } + return errors; +} + export type ValOpsOptions = { formatter?: (code: string, filePath: string) => string | Promise; statPollingInterval?: number; @@ -171,13 +217,16 @@ export abstract class ValOps { this.modulesErrors === null ) { const extracted = await extractValModules(this.valModules); + const moduleErrors = extracted.moduleErrors.concat( + findNestedJsonValuesModuleErrors(extracted.schemas), + ); this.sources = extracted.sources; this.schemas = extracted.schemas; this.baseSha = extracted.baseSha as BaseSha; this.schemaSha = extracted.schemaSha as SchemaSha; this.sourcesSha = extracted.sourcesSha as SourcesSha; this.configSha = extracted.configSha as ConfigSha; - this.modulesErrors = extracted.moduleErrors; + this.modulesErrors = moduleErrors; return { baseSha: this.baseSha, schemaSha: this.schemaSha, @@ -185,7 +234,7 @@ export abstract class ValOps { configSha: this.configSha, sources: extracted.sources, schemas: extracted.schemas, - moduleErrors: extracted.moduleErrors, + moduleErrors, }; } return { @@ -207,6 +256,220 @@ export abstract class ValOps { async getBaseSources(): Promise { return this.initSources().then((result) => result.sources); } + + /** + * Resolves the content of ONE `.jsonValues()` entry. + * + * The committed content comes from the entry's import thunk on the base + * source (so it works in both fs and http mode, with no extra I/O). With + * `applyPatches` (the default) any pending patches for that entry are then + * replayed on top, which is what makes draft edits visible to the runtime. + * + * Callers that apply patches themselves (the Studio, which owns + * in-flight client patches the server has not seen) must pass + * `applyPatches: false` or the same edits would be applied twice. + */ + async getJsonEntry( + moduleFilePath: ModuleFilePath, + entryKey: string, + opts?: { applyPatches?: boolean }, + ): Promise< + | { status: "success"; content: JSONValue | null } + | { status: "not-found"; message: string } + | { status: "error"; message: string } + | { status: "unauthorized"; message: string } + > { + const res = await this.getJsonEntries( + moduleFilePath, + { keys: [entryKey] }, + opts, + ); + if (res.status !== "success") { + return res; + } + const entry = res.entries[0]; + if (entry !== undefined) { + return { status: "success", content: entry.content }; + } + const error = res.errors[0]; + if (error !== undefined) { + return { status: "error", message: error.message }; + } + return { + status: "not-found", + message: `Entry not found: ${entryKey} in ${moduleFilePath}`, + }; + } + + /** + * Resolves the content of MANY `.jsonValues()` entries in one pass. + * + * This is the single implementation; {@link getJsonEntry} is a one-key wrapper. + * Batching matters because the expensive parts — `initSources()` and + * `fetchPatches()` — are hoisted OUT of the per-entry loop: resolving 500 + * entries one-by-one would otherwise mean 500 patch fetches. + * + * Per-entry problems stay per-entry (`missing` / `errors`) so one corrupt + * `*.val.json` cannot fail a whole batch. Only a missing or non-record MODULE + * is a whole-request `not-found`. + * + * `selector` is either explicit `keys` or an `offset`/`limit` window over every + * key of the record, in module key order. The window form requires + * `applyPatches: false`: enumerating from the base source would silently omit + * draft-added keys, and a silently-short key list is exactly the class of bug + * this endpoint exists to avoid. + */ + async getJsonEntries( + moduleFilePath: ModuleFilePath, + selector: { keys: string[] } | { offset: number; limit: number }, + opts?: { applyPatches?: boolean }, + ): Promise< + | { + status: "success"; + entries: { key: string; content: JSONValue | null }[]; + missing: string[]; + errors: { key: string; message: string }[]; + total: number; + offset?: number; + limit?: number; + } + | { status: "not-found"; message: string } + | { status: "error"; message: string } + | { status: "unauthorized"; message: string } + > { + const applyPatches = opts?.applyPatches !== false; + const isWindow = !("keys" in selector); + if (isWindow && applyPatches) { + return { + status: "error", + message: + "Cannot enumerate json entries by offset/limit with apply_patches: the base key set would omit draft-added entries. Pass apply_patches=false, or request explicit keys.", + }; + } + const { sources, schemas } = await this.initSources(); + const moduleSource = sources[moduleFilePath]; + if (moduleSource === undefined || moduleSource === null) { + return { + status: "not-found", + message: `Module not found: ${moduleFilePath}`, + }; + } + if (typeof moduleSource !== "object" || Array.isArray(moduleSource)) { + return { + status: "not-found", + message: `Module is not a record: ${moduleFilePath}`, + }; + } + const record = moduleSource as Record; + const allKeys = Object.keys(record); + const requestedKeys = isWindow + ? allKeys.slice(selector.offset, selector.offset + selector.limit) + : selector.keys; + + // Fetched once for the whole batch, not per entry. + let modulePatches: { patchId: PatchId; patch: Patch }[] = []; + let serializedSchema: SerializedSchema | undefined = undefined; + if (applyPatches) { + const patchOps = await this.fetchPatches({ excludePatchOps: false }); + if (patchOps.error) { + return { status: "error", message: patchOps.error.message }; + } + if (patchOps.errors && Object.keys(patchOps.errors).length > 0) { + return { + status: "error", + message: `Could not fetch patches: ${JSON.stringify(patchOps.errors)}`, + }; + } + modulePatches = patchOps.patches + .filter((p) => p.path === moduleFilePath && !p.appliedAt) + .map((p) => ({ patchId: p.patchId, patch: p.patch })); + try { + serializedSchema = schemas[moduleFilePath]?.["executeSerialize"](); + } catch { + // Serialization errors are reported elsewhere; treat as "no schema". + } + } + + const entries: { key: string; content: JSONValue | null }[] = []; + const missing: string[] = []; + const errors: { key: string; message: string }[] = []; + type ResolvedEntry = + | { + entryKey: string; + baseContent: JSONValue | undefined; + /** Set when the value lives in the module source, not a `*.val.json`. */ + inline?: true; + } + | { entryKey: string; message: string }; + const resolved = await Promise.all( + requestedKeys.map(async (entryKey): Promise => { + const marker = record[entryKey]; + if (marker !== undefined && !Internal.isJson(marker)) { + // Not a jsonValues entry — return the inlined value as-is (defensive). + // `inline` skips patch replay: entry patches are expressed against a + // jsonValues entry, and this value is part of the module source proper. + return { entryKey, baseContent: marker as JSONValue, inline: true }; + } + if (marker === undefined) { + return { entryKey, baseContent: undefined }; + } + const thunk = Internal.getJsonImport(marker); + if (!thunk) { + return { entryKey, baseContent: null }; + } + try { + return { + entryKey, + baseContent: ((await thunk()).default ?? null) as JSONValue, + }; + } catch (e) { + return { + entryKey, + message: `Failed to load JSON entry '${entryKey}': ${ + e instanceof Error ? e.message : String(e) + }`, + }; + } + }), + ); + for (const result of resolved) { + const { entryKey } = result; + if ("message" in result) { + errors.push({ key: entryKey, message: result.message }); + continue; + } + const { baseContent } = result; + if (!applyPatches || "inline" in result) { + if (baseContent === undefined) { + missing.push(entryKey); + } else { + entries.push({ key: entryKey, content: baseContent }); + } + continue; + } + const res = applyJsonValuesEntryPatches({ + serializedSchema, + entryKey, + baseContent, + patches: modulePatches, + }); + if (res.kind === "error") { + errors.push({ key: entryKey, message: res.message }); + } else if (res.kind === "deleted") { + missing.push(entryKey); + } else { + entries.push({ key: entryKey, content: res.content }); + } + } + return { + status: "success", + entries, + missing, + errors, + total: allKeys.length, + ...(isWindow ? { offset: selector.offset, limit: selector.limit } : {}), + }; + } async getSchemas(): Promise { return this.initSources().then((result) => result.schemas); } @@ -263,9 +526,18 @@ export abstract class ValOps { if (!patchesByModule[path]) { patchesByModule[path] = []; } - patchesByModule[path].push({ - patchId: patch.patchId, - }); + // At most ONE entry per (module, patch): consumers treat each entry as + // "apply this whole patch". Pushing per-op made a patch with N non-file + // ops be applied N times — idempotent for `replace`, but it duplicates + // `add`s and corrupts non-idempotent ops like `move`. + if ( + patchesByModule[path][patchesByModule[path].length - 1]?.patchId !== + patch.patchId + ) { + patchesByModule[path].push({ + patchId: patch.patchId, + }); + } } } return { @@ -316,6 +588,27 @@ export abstract class ValOps { error: GenericErrorMessage; }[] > = {}; + // Serialized schemas, resolved lazily and only for modules that actually + // have patches, so the common (non-jsonValues) case stays free. + const { schemas } = await this.initSources(); + const serializedSchemaCache = new Map< + ModuleFilePath, + SerializedSchema | undefined + >(); + const jsonValuesSchemaFor = ( + path: ModuleFilePath, + ): SerializedSchema | undefined => { + if (!serializedSchemaCache.has(path)) { + let serialized: SerializedSchema | undefined = undefined; + try { + serialized = schemas[path]?.["executeSerialize"](); + } catch { + // Serialization errors are reported elsewhere; treat as "no schema". + } + serializedSchemaCache.set(path, serialized); + } + return serializedSchemaCache.get(path); + }; for (const patchData of analysis.patches) { const path = patchData.path; if (sources[path] === undefined) { @@ -348,6 +641,13 @@ export abstract class ValOps { } else { const applicableOps: Patch = []; const fileFixOps: Record = {}; + // `.jsonValues()` entry values are opaque `{_type:"json"}` markers in + // the module source — their content lives in the entry's `*.val.json`. + // Ops that reach INTO an entry therefore cannot be applied here (and + // would fail with "Cannot replace object element which does not exist", + // poisoning the rest of this module's patch chain). See the per-op + // routing below. + const serializedSchema = jsonValuesSchemaFor(path); for (const op of patchData.patch) { if (op.op === "file") { if (op.value !== null) { @@ -368,7 +668,46 @@ export abstract class ValOps { // null value = delete: no patch_id to inject; the "remove" op in // the patch already removes the metadata entry from the source } else { - applicableOps.push(op); + const cls = serializedSchema + ? classifyJsonValuesOp(serializedSchema, op.path) + : ({ kind: "normal" } as const); + if (cls.kind === "normal") { + applicableOps.push(op); + } else if (cls.subPath.length > 0) { + // Content edit inside an entry: the module source is genuinely + // unaffected (the content lives in the `*.val.json`), so skip it. + // Draft content is served by the single-entry `/json` endpoint. + } else if (op.op === "add" || op.op === "replace") { + // Whole-entry add/replace: keep the record's KEY SET correct for + // drafts by writing the marker rather than the content. Record + // validation only asserts `isJson`, and + // `validateJsonValuesEntries` skips thunkless markers by design. + applicableOps.push({ + op: op.op, + path: op.path, + value: { + [VAL_EXTENSION]: "json", + patch_id: patchId, + } as JSONValue, + } as Operation); + } else if (op.op === "remove") { + applicableOps.push(op); + } else { + // move/copy of a whole entry: the destination key must appear, and + // for a move the source key must disappear. Both are key-set + // changes we can express with markers. + applicableOps.push({ + op: "add", + path: op.path, + value: { + [VAL_EXTENSION]: "json", + patch_id: patchId, + } as JSONValue, + } as Operation); + if (op.op === "move" && array.isNonEmpty(op.from)) { + applicableOps.push({ op: "remove", path: op.from }); + } + } } } const patchRes = applyPatch( @@ -484,6 +823,25 @@ export abstract class ValOps { path as string as SourcePath, source, ); + // For `.jsonValues()` records, executeValidate only checks the entry + // markers; load + validate each entry's backing `*.val.json` content here. + const jsonValuesErrors = await validateJsonValuesEntries( + schema, + source, + path, + ); + for (const [sourcePathS, entryErrors] of Object.entries( + jsonValuesErrors, + )) { + const sourcePath = sourcePathS as SourcePath; + if (!errors[path]) { + errors[path] = { validations: {} }; + } + if (!errors[path].validations[sourcePath]) { + errors[path].validations[sourcePath] = []; + } + errors[path].validations[sourcePath].push(...entryErrors); + } if (res === false) { continue; } @@ -795,13 +1153,21 @@ export abstract class ValOps { const patchedSourceFiles: Record = {}; const previousSourceFiles: Record = {}; + // Serialized schemas are needed to route ops that target `.jsonValues()` + // entries (their content lives in `*.val.json`, not the `.val.ts`). + const schemas = await this.getSchemas(); + const applySourceFilePatches = async ( path: ModuleFilePath, patches: { patchId: PatchId }[], ): Promise< | { path: ModuleFilePath; - result: string; + // `null` means the `.val.ts` itself was not changed (e.g. pure + // jsonValues content edits only touch `*.val.json`). + result: string | null; + // Extra files to write/delete (jsonValues `*.val.json` entries). + extraFiles: Record; appliedPatches: PatchId[]; errors?: undefined; } @@ -829,11 +1195,93 @@ export abstract class ValOps { } const sourceFile = sourceFileRes.data; previousSourceFiles[path] = sourceFile; - let tsSourceFile = ts.createSourceFile( + const originalSourceFile = ts.createSourceFile( "", sourceFile, ts.ScriptTarget.ES2015, ); + let tsSourceFile = originalSourceFile; + let tsChanged = false; + const serializedSchema = schemas[path]?.["executeSerialize"](); + + // jsonValues entry content, keyed by `*.val.json` path. `null` = delete. + const jsonEntryContents = new Map(); + // Entries added in this commit → their new `*.val.json` path, so later + // content ops in the same commit resolve to the freshly-created file. + const entryKeyToJsonPath = new Map(); + + // Lazily analyzed `c.json(() => import("..."))` entries of the ORIGINAL + // `.val.ts` (import paths are authoritative for existing/hand-placed files). + let analyzerEntries: Map | null = null; + const resolveEntryJsonPath = ( + entryKey: string, + ): result.Result => { + const added = entryKeyToJsonPath.get(entryKey); + if (added !== undefined) { + return result.ok(added); + } + if (analyzerEntries === null) { + const analysis = analyzeValModule(originalSourceFile); + if (result.isErr(analysis)) { + return result.err(analysis.error); + } + analyzerEntries = analyzeJsonValuesEntries(analysis.value.source); + } + const entry = analyzerEntries.get(entryKey); + if (!entry) { + return result.err({ + message: `Could not find jsonValues entry '${entryKey}' in ${path}`, + filePath: path, + }); + } + return result.ok(resolveExistingJsonPath(path, entry.importPath)); + }; + const loadEntryContent = async ( + jsonPath: string, + ): Promise> => { + const current = jsonEntryContents.get(jsonPath); + if (current !== undefined) { + if (current === null) { + return result.err({ + message: `Cannot edit a removed jsonValues entry: ${jsonPath}`, + filePath: jsonPath, + }); + } + return result.ok(current); + } + const res = await this.getSourceFile(jsonPath as ModuleFilePath); + if (res.error) { + return result.err({ message: res.error.message, filePath: jsonPath }); + } + try { + const parsed: JSONValue = JSON.parse(res.data); + jsonEntryContents.set(jsonPath, parsed); + return result.ok(parsed); + } catch (err) { + return result.err({ + message: `Could not parse jsonValues entry ${jsonPath}: ${ + err instanceof Error ? err.message : String(err) + }`, + filePath: jsonPath, + }); + } + }; + const collectPatchError = ( + err: PatchError | ValSyntaxErrorTree, + patchId: PatchId, + op: unknown, + ) => { + console.error( + "Could not patch", + JSON.stringify({ path, patchId, error: err, op }, null, 2), + ); + if (Array.isArray(err)) { + errors.push(...err); + } else { + errors.push(err); + } + }; + const appliedPatches: PatchId[] = []; const triedPatches: PatchId[] = []; for (const { patchId } of patches) { @@ -848,81 +1296,303 @@ export abstract class ValOps { } const patch = patchData.patch; const sourceFileOps = patch.filter((op) => op.op !== "file"); // file is not a valid source file op - const patchRes = applyPatch(tsSourceFile, tsOps, sourceFileOps); - if (result.isErr(patchRes)) { - if (Array.isArray(patchRes.error)) { - for (const error of patchRes.error) { - console.error( - "Could not patch", - JSON.stringify( - { - path, - patchId, - error, - sourceFileOps, - }, - null, - 2, - ), + let patchHadError = false; + for (const op of sourceFileOps) { + const cls = serializedSchema + ? classifyJsonValuesOp(serializedSchema, op.path) + : ({ kind: "normal" } as const); + // `move` / `copy` also READ from a path: classify that too, so an op + // that moves a value out of (or into) a jsonValues entry cannot slip + // through as a plain `.val.ts` op. + const fromCls = + serializedSchema && (op.op === "move" || op.op === "copy") + ? classifyJsonValuesOp(serializedSchema, op.from) + : ({ kind: "normal" } as const); + if (cls.kind === "normal" && fromCls.kind === "normal") { + const patchRes = applyPatch(tsSourceFile, tsOps, [op]); + if (result.isErr(patchRes)) { + collectPatchError(patchRes.error, patchId, op); + patchHadError = true; + break; + } + tsSourceFile = patchRes.value; + tsChanged = true; + continue; + } + if (cls.kind === "normal") { + errors.push({ + message: `Cannot '${op.op}' a value out of a jsonValues entry and into the module source`, + filePath: path, + }); + patchHadError = true; + break; + } + // Nested `.jsonValues()` records are not supported: only the read path + // for a module's ROOT record/router is implemented end to end. This is + // also rejected up front in `initSources`; this is defense in depth. + if ( + cls.recordPath.length > 0 || + (fromCls.kind === "entry" && fromCls.recordPath.length > 0) + ) { + errors.push({ + message: `Nested .jsonValues() records are not supported: '${cls.recordPath.join( + ".", + )}' in ${path}. Use .jsonValues() only on a module's root record/router.`, + filePath: path, + }); + patchHadError = true; + break; + } + // The op targets a `.jsonValues()` entry. + if (cls.subPath.length === 0) { + // Structural / whole-entry op. + if (op.op === "add") { + const { jsonPath, importPath } = getNewJsonEntryPaths( + path, + cls.entryKey, + ); + const insRes = insertValJsonEntry( + tsSourceFile, + cls.recordPath, + cls.entryKey, + importPath, ); + if (result.isErr(insRes)) { + collectPatchError(insRes.error, patchId, op); + patchHadError = true; + break; + } + tsSourceFile = insRes.value; + tsChanged = true; + jsonEntryContents.set(jsonPath, op.value); + entryKeyToJsonPath.set(cls.entryKey, jsonPath); + } else if (op.op === "remove") { + const jsonPathRes = resolveEntryJsonPath(cls.entryKey); + if (result.isErr(jsonPathRes)) { + errors.push(jsonPathRes.error); + patchHadError = true; + break; + } + const remRes = removeValJsonEntry( + tsSourceFile, + cls.recordPath, + cls.entryKey, + ); + if (result.isErr(remRes)) { + collectPatchError(remRes.error, patchId, op); + patchHadError = true; + break; + } + tsSourceFile = remRes.value; + tsChanged = true; + jsonEntryContents.set(jsonPathRes.value, null); + } else if (op.op === "replace") { + const jsonPathRes = resolveEntryJsonPath(cls.entryKey); + if (result.isErr(jsonPathRes)) { + errors.push(jsonPathRes.error); + patchHadError = true; + break; + } + jsonEntryContents.set(jsonPathRes.value, op.value); + } else if (op.op === "move" || op.op === "copy") { + // Rename (move) or duplicate (copy) a whole entry. The new entry + // gets its own `*.val.json` written with the source entry's + // content plus a `c.json(...)` thunk; a move additionally drops + // the old thunk and deletes the old file. + if ( + fromCls.kind !== "entry" || + fromCls.subPath.length !== 0 || + fromCls.recordPath.join("\0") !== cls.recordPath.join("\0") + ) { + errors.push({ + message: `Cannot '${ + op.op + }' a jsonValues entry across records or from a non-entry path (from '${op.from.join( + ".", + )}' to '${op.path.join(".")}')`, + filePath: path, + }); + patchHadError = true; + break; + } + const fromKey = fromCls.entryKey; + const fromPathRes = resolveEntryJsonPath(fromKey); + if (result.isErr(fromPathRes)) { + errors.push(fromPathRes.error); + patchHadError = true; + break; + } + // Load BEFORE marking anything deleted: `loadEntryContent` errors + // on a path that has already been nulled in this commit. + const contentRes = await loadEntryContent(fromPathRes.value); + if (result.isErr(contentRes)) { + errors.push(contentRes.error); + patchHadError = true; + break; + } + const content = deepClone(contentRes.value); + if (op.op === "move") { + const remRes = removeValJsonEntry( + tsSourceFile, + cls.recordPath, + fromKey, + ); + if (result.isErr(remRes)) { + collectPatchError(remRes.error, patchId, op); + patchHadError = true; + break; + } + tsSourceFile = remRes.value; + } + // LOCKED convention: the destination always uses the generated + // path, so renaming a hand-placed file relocates it. + const { jsonPath, importPath } = getNewJsonEntryPaths( + path, + cls.entryKey, + ); + const insRes = insertValJsonEntry( + tsSourceFile, + cls.recordPath, + cls.entryKey, + importPath, + ); + if (result.isErr(insRes)) { + collectPatchError(insRes.error, patchId, op); + patchHadError = true; + break; + } + tsSourceFile = insRes.value; + tsChanged = true; + jsonEntryContents.set(jsonPath, content); + entryKeyToJsonPath.set(cls.entryKey, jsonPath); + if (op.op === "move" && fromPathRes.value !== jsonPath) { + jsonEntryContents.set(fromPathRes.value, null); + } + } else { + errors.push({ + message: `Unsupported op '${op.op}' on jsonValues entry '${cls.entryKey}' (supported: add, remove, replace, move, copy)`, + filePath: path, + }); + patchHadError = true; + break; } - errors.push(...patchRes.error); } else { - console.error( - "Could not patch", - JSON.stringify( - { - path, - patchId, - error: patchRes.error, - sourceFileOps, - }, - null, - 2, - ), - ); - errors.push(patchRes.error); + // Content sub-op: replay against the entry's `*.val.json`. + // `rebaseContentOp` slices `from` by the same prefix as `path`, so a + // cross-entry move/copy would silently corrupt the target entry. + if ( + (op.op === "move" || op.op === "copy") && + (fromCls.kind !== "entry" || fromCls.entryKey !== cls.entryKey) + ) { + errors.push({ + message: `Cannot '${op.op}' between different jsonValues entries`, + filePath: path, + }); + patchHadError = true; + break; + } + const jsonPathRes = resolveEntryJsonPath(cls.entryKey); + if (result.isErr(jsonPathRes)) { + errors.push(jsonPathRes.error); + patchHadError = true; + break; + } + const jsonPath = jsonPathRes.value; + const contentRes = await loadEntryContent(jsonPath); + if (result.isErr(contentRes)) { + errors.push(contentRes.error); + patchHadError = true; + break; + } + const rebasedRes = rebaseContentOp(op, cls.recordPath.length + 1); + if (result.isErr(rebasedRes)) { + errors.push({ + message: rebasedRes.error.message, + filePath: jsonPath, + }); + patchHadError = true; + break; + } + const applied = applyPatch(deepClone(contentRes.value), jsonOps, [ + rebasedRes.value, + ]); + if (result.isErr(applied)) { + collectPatchError(applied.error, patchId, op); + patchHadError = true; + break; + } + jsonEntryContents.set(jsonPath, applied.value); } + } + if (patchHadError) { triedPatches.push(patchId); break; } appliedPatches.push(patchId); - tsSourceFile = patchRes.value; } if (errors.length === 0) { // https://github.com/microsoft/TypeScript/issues/36174 - let sourceFileText = unescape( - tsSourceFile.getText(tsSourceFile).replace(/\\u/g, "%u"), - ); - if (this.options?.formatter) { - try { - sourceFileText = await this.options.formatter(sourceFileText, path); - } catch (err) { - errors.push({ - message: - "Could not format source file: " + - (err instanceof Error ? err.message : "Unknown error"), - }); + let sourceFileText: string | null = null; + if (tsChanged) { + sourceFileText = unescape( + tsSourceFile.getText(tsSourceFile).replace(/\\u/g, "%u"), + ); + if (this.options?.formatter) { + try { + sourceFileText = await this.options.formatter( + sourceFileText, + path, + ); + } catch (err) { + errors.push({ + message: + "Could not format source file: " + + (err instanceof Error ? err.message : "Unknown error"), + }); + } } } - return { - path, - appliedPatches, - result: sourceFileText, - }; - } else { - const skippedPatches = patches - .slice(appliedPatches.length + triedPatches.length) - .map((p) => p.patchId); - - return { - path, - appliedPatches, - triedPatches, - skippedPatches, - errors, - }; + const extraFiles: Record = {}; + for (const [jsonPath, content] of Array.from(jsonEntryContents)) { + if (content === null) { + extraFiles[jsonPath] = null; + continue; + } + let jsonText = JSON.stringify(content, null, 2); + if (this.options?.formatter) { + try { + jsonText = await this.options.formatter(jsonText, jsonPath); + } catch (err) { + errors.push({ + message: + "Could not format jsonValues entry: " + + (err instanceof Error ? err.message : "Unknown error"), + filePath: jsonPath, + }); + } + } + extraFiles[jsonPath] = jsonText; + } + if (errors.length === 0) { + return { + path, + appliedPatches, + result: sourceFileText, + extraFiles, + }; + } } + const skippedPatches = patches + .slice(appliedPatches.length + triedPatches.length) + .map((p) => p.patchId); + + return { + path, + appliedPatches, + triedPatches, + skippedPatches, + errors, + }; }; const allResults = await Promise.all( Object.entries(patchesByModule).map(([path, patches]) => @@ -946,7 +1616,14 @@ export abstract class ValOps { triedPatches[res.path] = res.triedPatches ?? []; skippedPatches[res.path] = res.skippedPatches ?? []; } else { - patchedSourceFiles[res.path] = res.result; + // `result` is null when the `.val.ts` itself was not changed (pure + // jsonValues content edits only write `*.val.json` extraFiles). + if (res.result !== null) { + patchedSourceFiles[res.path] = res.result; + } + for (const [extraPath, data] of Object.entries(res.extraFiles)) { + patchedSourceFiles[extraPath] = data; + } appliedPatches[res.path] = res.appliedPatches ?? []; } for (const patchId of res.appliedPatches ?? []) { diff --git a/packages/server/src/ValOpsFS.jsonValues.test.ts b/packages/server/src/ValOpsFS.jsonValues.test.ts new file mode 100644 index 000000000..c70d6a0cc --- /dev/null +++ b/packages/server/src/ValOpsFS.jsonValues.test.ts @@ -0,0 +1,575 @@ +import { Internal, ModuleFilePath, PatchId, initVal } from "@valbuild/core"; +import { Script } from "node:vm"; +import { transform } from "sucrase"; +import { ValOpsFS } from "./ValOpsFS"; +import fs from "fs"; +import os from "node:os"; +import path from "node:path"; +import synchronizedPrettier from "@prettier/sync"; +import type { OrderedPatches } from "./ValOps"; + +const MODULE_PATH = "/test/pages.val.ts" as ModuleFilePath; + +const MODULE_CODE = ` +import { s, c } from "val.config"; + +export default c.define( + "/test/pages.val.ts", + s.record(s.object({ title: s.string(), order: s.number() })).jsonValues(), + { + "/blog/hello": c.json(() => import("./content/hello.val.json")), + "/blog/world": c.json(() => import("./content/world.val.json")), + } +); +`; + +const JSON_FILES: Record = { + "/test/content/hello.val.json": { title: "Hello", order: 1 }, + "/test/content/world.val.json": { title: "World", order: 2 }, +}; + +function setup() { + const { s, c, config } = initVal(); + // Use the OS temp dir (NOT the repo-local ".tmp", which other suites wipe). + const rootDir = fs.mkdtempSync(path.join(os.tmpdir(), "val-jsonvalues-test")); + + const evalModule = (code: string) => + new Script( + transform(code, { transforms: ["imports"] }).code, + ).runInNewContext({ + exports: {}, + require: (p: string) => { + if (p === "val.config") { + return { s, c, config }; + } + // Resolve the `c.json(() => import("./x.val.json"))` thunks against the + // module's directory in the TEMP root — plain `require` would resolve + // them relative to this test file. + if (p.startsWith("./") || p.startsWith("../")) { + const abs = path.resolve( + path.join(rootDir, path.dirname(MODULE_PATH)), + p, + ); + // eslint-disable-next-line @typescript-eslint/no-require-imports + return require(abs); + } + // eslint-disable-next-line @typescript-eslint/no-require-imports + return require(p); + }, + module: { exports: {} }, + }); + + const moduleAbs = path.join(rootDir, MODULE_PATH); + fs.mkdirSync(path.dirname(moduleAbs), { recursive: true }); + fs.writeFileSync( + moduleAbs, + synchronizedPrettier.format(MODULE_CODE, { parser: "typescript" }), + ); + for (const [filePath, content] of Object.entries(JSON_FILES)) { + const absPath = path.join(rootDir, filePath); + fs.mkdirSync(path.dirname(absPath), { recursive: true }); + fs.writeFileSync(absPath, JSON.stringify(content, null, 2)); + } + + const contentHost = process.env.VAL_CONTENT_URL || "http://localhost:4000"; + const ops = new ValOpsFS( + contentHost, + rootDir, + { + config, + modules: [{ def: async () => ({ default: evalModule(MODULE_CODE) }) }], + }, + { + formatter: (code, filePath) => + synchronizedPrettier.format(code, { filepath: filePath }), + config, + }, + ); + return { ops, rootDir }; +} + +async function prepareSingle( + ops: ValOpsFS, + patch: OrderedPatches["patches"][number]["patch"], +) { + const patches: OrderedPatches["patches"] = [ + { + path: MODULE_PATH, + patchId: crypto.randomUUID() as PatchId, + patch, + createdAt: new Date().toISOString(), + authorId: null, + baseSha: await ops.getBaseSha(), + appliedAt: null, + }, + ]; + const analysis = ops.analyzePatches(patches); + return ops.prepare({ ...analysis, patches }); +} + +async function getSourcesWith( + ops: ValOpsFS, + patch: OrderedPatches["patches"][number]["patch"], +) { + const patches: OrderedPatches["patches"] = [ + { + path: MODULE_PATH, + patchId: crypto.randomUUID() as PatchId, + patch, + createdAt: new Date().toISOString(), + authorId: null, + baseSha: await ops.getBaseSha(), + appliedAt: null, + }, + ]; + const analysis = ops.analyzePatches(patches); + return ops.getSources({ ...analysis, patches }); +} + +/** Creates a real pending patch on disk, as the Studio would. */ +async function createPatch( + ops: ValOpsFS, + patch: OrderedPatches["patches"][number]["patch"], +) { + const existing = await ops.fetchPatches({ excludePatchOps: true }); + const last = existing.patches[existing.patches.length - 1]; + const res = await ops.createPatch( + MODULE_PATH, + patch, + crypto.randomUUID() as PatchId, + last + ? { type: "patch", patchId: last.patchId } + : { type: "head", headBaseSha: await ops.getBaseSha() }, + null, + null, + ); + if ("error" in res) { + throw new Error(`Could not create patch: ${JSON.stringify(res)}`); + } + return res; +} + +describe("ValOps.getJsonEntry", () => { + test("returns the committed content when there are no patches", async () => { + const { ops } = setup(); + const res = await ops.getJsonEntry(MODULE_PATH, "/blog/hello"); + expect(res).toEqual({ + status: "success", + content: { title: "Hello", order: 1 }, + }); + }); + + test("applies a pending content patch (the draft read path)", async () => { + const { ops } = setup(); + await createPatch(ops, [ + { op: "replace", path: ["/blog/hello", "title"], value: "Draft!" }, + ]); + const res = await ops.getJsonEntry(MODULE_PATH, "/blog/hello"); + expect(res).toEqual({ + status: "success", + content: { title: "Draft!", order: 1 }, + }); + }); + + test("applyPatches:false returns the committed content (what the Studio asks for)", async () => { + const { ops } = setup(); + await createPatch(ops, [ + { op: "replace", path: ["/blog/hello", "title"], value: "Draft!" }, + ]); + const res = await ops.getJsonEntry(MODULE_PATH, "/blog/hello", { + applyPatches: false, + }); + expect(res).toEqual({ + status: "success", + content: { title: "Hello", order: 1 }, + }); + }); + + test("resolves an entry that only exists in a pending patch", async () => { + const { ops } = setup(); + await createPatch(ops, [ + { op: "add", path: ["/blog/new"], value: { title: "New", order: 3 } }, + ]); + expect(await ops.getJsonEntry(MODULE_PATH, "/blog/new")).toEqual({ + status: "success", + content: { title: "New", order: 3 }, + }); + // ...but not when the caller wants committed content only. + expect( + ( + await ops.getJsonEntry(MODULE_PATH, "/blog/new", { + applyPatches: false, + }) + ).status, + ).toBe("not-found"); + }); + + test("an entry removed by a pending patch is not-found", async () => { + const { ops } = setup(); + await createPatch(ops, [{ op: "remove", path: ["/blog/world"] }]); + expect((await ops.getJsonEntry(MODULE_PATH, "/blog/world")).status).toBe( + "not-found", + ); + }); + + test("an unknown key is not-found", async () => { + const { ops } = setup(); + expect((await ops.getJsonEntry(MODULE_PATH, "/nope")).status).toBe( + "not-found", + ); + }); +}); + +describe("ValOps.getJsonEntries (batch)", () => { + test("resolves many keys in one call", async () => { + const { ops } = setup(); + const res = await ops.getJsonEntries(MODULE_PATH, { + keys: ["/blog/hello", "/blog/world"], + }); + expect(res).toEqual({ + status: "success", + entries: [ + { key: "/blog/hello", content: { title: "Hello", order: 1 } }, + { key: "/blog/world", content: { title: "World", order: 2 } }, + ], + missing: [], + errors: [], + total: 2, + }); + }); + + test("keeps the requested key order, and echoes it back", async () => { + const { ops } = setup(); + const res = await ops.getJsonEntries(MODULE_PATH, { + keys: ["/blog/world", "/blog/hello"], + }); + expect(res.status === "success" && res.entries.map((e) => e.key)).toEqual([ + "/blog/world", + "/blog/hello", + ]); + }); + + test("an unknown key is `missing`, NOT a failed batch", async () => { + const { ops } = setup(); + const res = await ops.getJsonEntries(MODULE_PATH, { + keys: ["/blog/hello", "/nope"], + }); + expect(res).toMatchObject({ + status: "success", + entries: [{ key: "/blog/hello", content: { title: "Hello", order: 1 } }], + missing: ["/nope"], + errors: [], + }); + }); + + test("a corrupt entry is a per-key error, NOT a failed batch", async () => { + const { ops, rootDir } = setup(); + fs.writeFileSync( + path.join(rootDir, "test/content/world.val.json"), + "{ not json", + "utf-8", + ); + const res = await ops.getJsonEntries(MODULE_PATH, { + keys: ["/blog/hello", "/blog/world"], + }); + expect(res.status).toBe("success"); + if (res.status !== "success") throw new Error("unreachable"); + expect(res.entries).toEqual([ + { key: "/blog/hello", content: { title: "Hello", order: 1 } }, + ]); + expect(res.errors).toHaveLength(1); + expect(res.errors[0].key).toBe("/blog/world"); + }); + + test("applies pending patches per entry", async () => { + const { ops } = setup(); + await createPatch(ops, [ + { op: "replace", path: ["/blog/hello", "title"], value: "Draft!" }, + { op: "remove", path: ["/blog/world"] }, + ]); + const res = await ops.getJsonEntries(MODULE_PATH, { + keys: ["/blog/hello", "/blog/world"], + }); + expect(res).toMatchObject({ + status: "success", + entries: [{ key: "/blog/hello", content: { title: "Draft!", order: 1 } }], + missing: ["/blog/world"], + }); + }); + + test("offset/limit pages over the record in key order", async () => { + const { ops } = setup(); + const first = await ops.getJsonEntries( + MODULE_PATH, + { offset: 0, limit: 1 }, + { applyPatches: false }, + ); + expect(first).toEqual({ + status: "success", + entries: [{ key: "/blog/hello", content: { title: "Hello", order: 1 } }], + missing: [], + errors: [], + offset: 0, + limit: 1, + total: 2, + }); + const second = await ops.getJsonEntries( + MODULE_PATH, + { offset: 1, limit: 10 }, + { applyPatches: false }, + ); + expect(second).toMatchObject({ + status: "success", + entries: [{ key: "/blog/world", content: { title: "World", order: 2 } }], + offset: 1, + limit: 10, + total: 2, + }); + // Past the end: empty, not an error. + expect( + await ops.getJsonEntries( + MODULE_PATH, + { offset: 5, limit: 10 }, + { applyPatches: false }, + ), + ).toMatchObject({ status: "success", entries: [], total: 2 }); + }); + + test("offset/limit is rejected when patches would be applied", async () => { + const { ops } = setup(); + // The base key set cannot represent draft-added keys, so enumerating with + // apply_patches would silently return a short list. + const res = await ops.getJsonEntries(MODULE_PATH, { offset: 0, limit: 10 }); + expect(res.status).toBe("error"); + }); + + test("an unknown module is not-found for the whole request", async () => { + const { ops } = setup(); + expect( + ( + await ops.getJsonEntries("/test/nope.val.ts" as ModuleFilePath, { + keys: ["/blog/hello"], + }) + ).status, + ).toBe("not-found"); + }); + + test("fetches patches ONCE for the whole batch", async () => { + const { ops } = setup(); + await createPatch(ops, [ + { op: "replace", path: ["/blog/hello", "title"], value: "Draft!" }, + ]); + const fetchPatches = jest.spyOn(ops, "fetchPatches"); + await ops.getJsonEntries(MODULE_PATH, { + keys: ["/blog/hello", "/blog/world"], + }); + expect(fetchPatches).toHaveBeenCalledTimes(1); + fetchPatches.mockRestore(); + }); +}); + +describe("ValOpsFS jsonValues commit flow", () => { + test("content edit writes only the *.val.json (not the .val.ts)", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { op: "replace", path: ["/blog/hello", "title"], value: "Hello!" }, + ]); + expect(pc.hasErrors).toBe(false); + // .val.ts is untouched on a pure content edit + expect(pc.patchedSourceFiles[MODULE_PATH]).toBeUndefined(); + const written = pc.patchedSourceFiles["/test/content/hello.val.json"]; + expect(typeof written).toBe("string"); + expect(JSON.parse(written as string)).toEqual({ + title: "Hello!", + order: 1, + }); + // the untouched entry is not written + expect( + pc.patchedSourceFiles["/test/content/world.val.json"], + ).toBeUndefined(); + }); + + test("add entry writes a new *.val.json and inserts a c.json thunk", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { + op: "add", + path: ["/blog/new"], + value: { title: "New", order: 3 }, + }, + ]); + expect(pc.hasErrors).toBe(false); + const newJson = pc.patchedSourceFiles["/test/pages/blog/new.val.json"]; + expect(typeof newJson).toBe("string"); + expect(JSON.parse(newJson as string)).toEqual({ title: "New", order: 3 }); + const ts = pc.patchedSourceFiles[MODULE_PATH]; + expect(typeof ts).toBe("string"); + expect(ts).toContain(`import("./pages/blog/new.val.json")`); + expect(ts).toContain(`"/blog/new"`); + }); + + test("remove entry deletes the *.val.json and drops the thunk", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { op: "remove", path: ["/blog/world"] }, + ]); + expect(pc.hasErrors).toBe(false); + // null signals a file deletion in the commit loop + expect(pc.patchedSourceFiles["/test/content/world.val.json"]).toBeNull(); + const ts = pc.patchedSourceFiles[MODULE_PATH]; + expect(typeof ts).toBe("string"); + expect(ts).not.toContain(`"/blog/world"`); + expect(ts).toContain(`"/blog/hello"`); + }); + + test("move renames a hand-authored entry: relocates the file and swaps the thunk", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { op: "move", from: ["/blog/hello"], path: ["/blog/renamed"] }, + ]); + expect(pc.hasErrors).toBe(false); + // LOCKED convention: the destination uses the generated path, so a rename + // relocates a hand-placed file out of its original directory. + const newJson = pc.patchedSourceFiles["/test/pages/blog/renamed.val.json"]; + expect(typeof newJson).toBe("string"); + expect(JSON.parse(newJson as string)).toEqual({ + title: "Hello", + order: 1, + }); + expect(pc.patchedSourceFiles["/test/content/hello.val.json"]).toBeNull(); + const ts = pc.patchedSourceFiles[MODULE_PATH]; + expect(typeof ts).toBe("string"); + expect(ts).not.toContain(`"/blog/hello"`); + expect(ts).not.toContain(`import("./content/hello.val.json")`); + expect(ts).toContain(`"/blog/renamed"`); + expect(ts).toContain(`import("./pages/blog/renamed.val.json")`); + // the untouched entry survives + expect(ts).toContain(`"/blog/world"`); + }); + + test("move then a content edit on the new key resolves to the new file", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { op: "move", from: ["/blog/hello"], path: ["/blog/renamed"] }, + { op: "replace", path: ["/blog/renamed", "title"], value: "Renamed!" }, + ]); + expect(pc.hasErrors).toBe(false); + const newJson = pc.patchedSourceFiles["/test/pages/blog/renamed.val.json"]; + expect(JSON.parse(newJson as string)).toEqual({ + title: "Renamed!", + order: 1, + }); + expect(pc.patchedSourceFiles["/test/content/hello.val.json"]).toBeNull(); + }); + + test("a content edit on the OLD key after a move is an error", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { op: "move", from: ["/blog/hello"], path: ["/blog/renamed"] }, + { op: "replace", path: ["/blog/hello", "title"], value: "Nope" }, + ]); + expect(pc.hasErrors).toBe(true); + }); + + test("copy duplicates an entry without deleting the source", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { op: "copy", from: ["/blog/hello"], path: ["/blog/copy"] }, + ]); + expect(pc.hasErrors).toBe(false); + const newJson = pc.patchedSourceFiles["/test/pages/blog/copy.val.json"]; + expect(JSON.parse(newJson as string)).toEqual({ + title: "Hello", + order: 1, + }); + // the source file is NOT deleted + expect( + pc.patchedSourceFiles["/test/content/hello.val.json"], + ).not.toBeNull(); + const ts = pc.patchedSourceFiles[MODULE_PATH]; + expect(ts).toContain(`"/blog/hello"`); + expect(ts).toContain(`"/blog/copy"`); + }); + + test("move from a non-entry path into an entry is an error", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + // `from` targets a field INSIDE an entry, `path` targets a whole entry + { + op: "move", + from: ["/blog/hello", "title"], + path: ["/blog/renamed"], + }, + ]); + expect(pc.hasErrors).toBe(true); + }); + + test("a multi-op patch is applied exactly once (not once per op)", async () => { + // Regression: analyzePatches used to push one entry per non-file op, so + // `prepare` re-applied the whole patch once per op. + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { op: "replace", path: ["/blog/hello", "order"], value: 10 }, + { op: "replace", path: ["/blog/world", "order"], value: 20 }, + ]); + expect(pc.hasErrors).toBe(false); + expect( + JSON.parse( + pc.patchedSourceFiles["/test/content/hello.val.json"] as string, + ), + ).toEqual({ title: "Hello", order: 10 }); + expect( + JSON.parse( + pc.patchedSourceFiles["/test/content/world.val.json"] as string, + ), + ).toEqual({ title: "World", order: 20 }); + expect(pc.appliedPatches[MODULE_PATH]).toHaveLength(1); + }); + + test("getSources: a content edit does not poison the module's patch chain", async () => { + // Regression: getSources applied entry-content ops with jsonOps against the + // opaque `{_type:"json"}` marker. That failed with "Cannot replace object + // element which does not exist", after which EVERY later patch for the + // module was skipped with "previous errors exists". + const { ops } = setup(); + const res = await getSourcesWith(ops, [ + { op: "replace", path: ["/blog/hello", "title"], value: "Hello!" }, + { op: "add", path: ["/blog/new"], value: { title: "New", order: 3 } }, + ]); + expect(res.errors[MODULE_PATH]).toBeUndefined(); + const source = res.sources[MODULE_PATH] as Record; + // Content edits leave the marker alone (content lives in the *.val.json). + expect(Internal.isJson(source["/blog/hello"])).toBe(true); + // A newly added entry appears as a marker, so the record's KEY SET is right. + expect(Internal.isJson(source["/blog/new"])).toBe(true); + expect(Object.keys(source).sort()).toEqual([ + "/blog/hello", + "/blog/new", + "/blog/world", + ]); + }); + + test("getSources: remove and rename update the key set", async () => { + const { ops } = setup(); + const res = await getSourcesWith(ops, [ + { op: "remove", path: ["/blog/world"] }, + { op: "move", from: ["/blog/hello"], path: ["/blog/renamed"] }, + ]); + expect(res.errors[MODULE_PATH]).toBeUndefined(); + const source = res.sources[MODULE_PATH] as Record; + expect(Object.keys(source)).toEqual(["/blog/renamed"]); + expect(Internal.isJson(source["/blog/renamed"])).toBe(true); + }); + + test("move between different entries' content is an error", async () => { + const { ops } = setup(); + const pc = await prepareSingle(ops, [ + { + op: "move", + from: ["/blog/hello", "title"], + path: ["/blog/world", "title"], + }, + ]); + expect(pc.hasErrors).toBe(true); + }); +}); diff --git a/packages/server/src/ValRouter.test.ts b/packages/server/src/ValRouter.test.ts index 7c2f8ae9a..c7a2a29d9 100644 --- a/packages/server/src/ValRouter.test.ts +++ b/packages/server/src/ValRouter.test.ts @@ -95,6 +95,55 @@ describe("ValRouter", () => { expect("json" in serverRes && serverRes.json).toBeTruthy(); }); + // `/json` takes exactly one of `key`, `keys` or `offset`+`limit`. These are + // rejected before the ops layer is consulted, so the fixture module (an + // ordinary record) is enough to pin the contract. + describe("/json request shapes", () => { + const jsonRequest = (query: string) => + onRoute( + fakeRequest({ + method: "GET", + url: new URL(`http://localhost:3000/api/val/json?${query}`), + headers: new Headers({ + Cookie: `val_session=${encodeJwt({}, "")}`, + }), + }), + ); + + test("no shape is a 400", async () => { + expect((await jsonRequest("path=/content/authors.val.ts")).status).toBe( + 400, + ); + }); + + test("key AND keys together is a 400", async () => { + expect( + (await jsonRequest("path=/content/authors.val.ts&key=a&keys=b")).status, + ).toBe(400); + }); + + test("offset without limit is a 400", async () => { + expect( + (await jsonRequest("path=/content/authors.val.ts&offset=0")).status, + ).toBe(400); + }); + + test("more keys than the batch cap is rejected", async () => { + const keys = Array.from({ length: 101 }, (_, i) => `keys=k${i}`).join( + "&", + ); + expect( + (await jsonRequest(`path=/content/authors.val.ts&${keys}`)).status, + ).toBe(400); + }); + + test("a valid shape gets past validation (404: no such module)", async () => { + expect( + (await jsonRequest("path=/content/nope.val.ts&key=a")).status, + ).toBe(404); + }); + }); + test("smoke test invalid route", async () => { const serverRes = await onRoute( fakeRequest({ diff --git a/packages/server/src/ValServer.ts b/packages/server/src/ValServer.ts index 4e86832ca..b7ef0f2ac 100644 --- a/packages/server/src/ValServer.ts +++ b/packages/server/src/ValServer.ts @@ -1361,6 +1361,92 @@ export const ValServer = ( }, }, + // #region json + // Loads the content of a single `.jsonValues()` entry by key, so the Studio + // can lazily load just the entry being opened, and the runtime can read draft + // edits. With apply_patches (default true) pending patches for the entry are + // replayed server-side; the Studio passes false and overlays its own. + "/json": { + GET: async (req) => { + const auth = getAuth(req.cookies); + if (auth.error) { + return { status: 401, json: { message: auth.error } }; + } + if (serverOps instanceof ValOpsHttp && !("id" in auth)) { + return { status: 401, json: { message: "Unauthorized" } }; + } + const moduleFilePath = req.query.path as ModuleFilePath; + const { key, keys, offset, limit } = req.query; + // Defaults to true, mirroring /sources/~. The Studio opts out. + const applyPatches = req.query.apply_patches !== false; + const isWindow = offset !== undefined || limit !== undefined; + const shapes = [key !== undefined, keys !== undefined, isWindow].filter( + Boolean, + ).length; + if (shapes !== 1) { + return { + status: 400, + json: { + message: + "Exactly one of 'key', 'keys' or 'offset'+'limit' must be given", + }, + }; + } + if (isWindow && (offset === undefined || limit === undefined)) { + return { + status: 400, + json: { message: "'offset' and 'limit' must be given together" }, + }; + } + if (key !== undefined) { + const res = await serverOps.getJsonEntry(moduleFilePath, key, { + applyPatches, + }); + if (res.status === "unauthorized") { + return { status: 401, json: { message: res.message } }; + } + if (res.status === "not-found") { + return { status: 404, json: { message: res.message } }; + } + if (res.status === "error") { + return { status: 500, json: { message: res.message } }; + } + return { + status: 200, + json: { path: moduleFilePath, key, content: res.content }, + }; + } + const res = await serverOps.getJsonEntries( + moduleFilePath, + keys !== undefined + ? { keys } + : { offset: offset as number, limit: limit as number }, + { applyPatches }, + ); + if (res.status === "unauthorized") { + return { status: 401, json: { message: res.message } }; + } + if (res.status === "not-found") { + return { status: 404, json: { message: res.message } }; + } + if (res.status === "error") { + return { status: 500, json: { message: res.message } }; + } + return { + status: 200, + json: { + path: moduleFilePath, + entries: res.entries, + missing: res.missing, + errors: res.errors, + ...(res.offset !== undefined ? { offset: res.offset } : {}), + ...(res.limit !== undefined ? { limit: res.limit } : {}), + total: res.total, + }, + }; + }, + }, + // #region sources "/sources/~": { PUT: async (req) => { diff --git a/packages/server/src/loadValModules.jsonValues.test.ts b/packages/server/src/loadValModules.jsonValues.test.ts new file mode 100644 index 000000000..4fb27c030 --- /dev/null +++ b/packages/server/src/loadValModules.jsonValues.test.ts @@ -0,0 +1,36 @@ +import path from "path"; +import { Internal } from "@valbuild/core"; +import { loadValModules } from "./loadValModules"; + +const fixtureRoot = path.join(__dirname, "..", "test", "jsonValues-fixture"); + +describe("loadValModules with .jsonValues() + c.json", () => { + test("loads the module without invoking entry thunks (stays lazy)", async () => { + const valModules = loadValModules(fixtureRoot); + expect(valModules.modules).toHaveLength(1); + + const mod = (await valModules.modules[0].def()).default; + const source = Internal.getSource(mod) as Record; + const entry = source["/blogs/test"] as { + _type: string; + _import: () => Promise<{ default: unknown }>; + }; + + // The entry is a json marker, not the loaded content. + expect(Internal.isJson(entry)).toBe(true); + expect(entry._type).toBe("json"); + expect(typeof entry._import).toBe("function"); + }); + + test("invoking the entry thunk loads the backing *.val.json", async () => { + const valModules = loadValModules(fixtureRoot); + const mod = (await valModules.modules[0].def()).default; + const source = Internal.getSource(mod) as Record; + const entry = source["/blogs/test"] as { + _import: () => Promise<{ default: unknown }>; + }; + + const loaded = await entry._import(); + expect(loaded.default).toEqual({ title: "Hello from JSON" }); + }); +}); diff --git a/packages/server/src/loadValModules.ts b/packages/server/src/loadValModules.ts index 51463a1b7..3dcc3bfe2 100644 --- a/packages/server/src/loadValModules.ts +++ b/packages/server/src/loadValModules.ts @@ -56,7 +56,15 @@ function findValModulesPath(projectRoot: string): string | null { return null; } -const RESOLVE_EXTENSIONS = [".ts", ".tsx", ".js", ".jsx", ".cjs", ".mjs"]; +const RESOLVE_EXTENSIONS = [ + ".ts", + ".tsx", + ".js", + ".jsx", + ".cjs", + ".mjs", + ".json", +]; // Specifiers that user val files must not actually use. We stub them so that // importing is fine, but using a value throws a clear error. Real @valbuild @@ -110,6 +118,18 @@ function loadModule( if (cached) { return cached; } + // JSON modules (e.g. the `*.val.json` files backing `.jsonValues()` entries) + // are loaded by parsing, mirroring Node's `require("./x.json")` which returns + // the parsed object as `module.exports`. The importing `.val.ts` wraps this + // with `__importStar` so `import("./x.val.json")` yields `{ default, ... }`. + // These are only loaded when an entry thunk is invoked, never during + // `extractValModules`, so this stays lazy. + if (absPath.endsWith(".json")) { + const parsed = JSON.parse(fs.readFileSync(absPath, "utf-8")); + const jsonModule = { exports: parsed as Record }; + cache[absPath] = jsonModule; + return jsonModule; + } const code = fs.readFileSync(absPath, "utf-8"); const transpiled = ts.transpileModule(code, { compilerOptions: { diff --git a/packages/server/src/patch/jsonValuesPatch.test.ts b/packages/server/src/patch/jsonValuesPatch.test.ts new file mode 100644 index 000000000..f82ef90e5 --- /dev/null +++ b/packages/server/src/patch/jsonValuesPatch.test.ts @@ -0,0 +1,306 @@ +import { initVal, PatchId, type SerializedSchema } from "@valbuild/core"; +import { + applyJsonValuesEntryPatches, + classifyJsonValuesOp, + findNestedJsonValuesRecords, + getNewJsonEntryPaths, + rebaseContentOp, + resolveExistingJsonPath, +} from "./jsonValuesPatch"; +import { result } from "@valbuild/core/fp"; +import type { Patch } from "@valbuild/core/patch"; + +const { s } = initVal(); + +describe("classifyJsonValuesOp", () => { + const rootJsonValues: SerializedSchema = s + .record(s.object({ title: s.string(), order: s.number() })) + .jsonValues() + ["executeSerialize"](); + + test("field edit inside a root jsonValues entry → content sub-op", () => { + const cls = classifyJsonValuesOp(rootJsonValues, ["/blog/hello", "title"]); + expect(cls).toEqual({ + kind: "entry", + recordPath: [], + entryKey: "/blog/hello", + subPath: ["title"], + }); + }); + + test("add/remove of an entry key → entry op with empty subPath", () => { + expect(classifyJsonValuesOp(rootJsonValues, ["/blog/hello"])).toEqual({ + kind: "entry", + recordPath: [], + entryKey: "/blog/hello", + subPath: [], + }); + }); + + test("nested jsonValues record under an object carries recordPath", () => { + const nested: SerializedSchema = s + .object({ + pages: s.record(s.object({ title: s.string() })).jsonValues(), + }) + ["executeSerialize"](); + expect(classifyJsonValuesOp(nested, ["pages", "/a/b", "title"])).toEqual({ + kind: "entry", + recordPath: ["pages"], + entryKey: "/a/b", + subPath: ["title"], + }); + }); + + test("plain (non-jsonValues) record → normal", () => { + const plain: SerializedSchema = s + .record(s.object({ title: s.string() })) + ["executeSerialize"](); + expect(classifyJsonValuesOp(plain, ["/blog/hello", "title"])).toEqual({ + kind: "normal", + }); + }); + + test("op that does not reach the jsonValues record → normal", () => { + const nested: SerializedSchema = s + .object({ + title: s.string(), + pages: s.record(s.object({ title: s.string() })).jsonValues(), + }) + ["executeSerialize"](); + expect(classifyJsonValuesOp(nested, ["title"])).toEqual({ kind: "normal" }); + }); +}); + +describe("getNewJsonEntryPaths", () => { + test("mirrors the entry key under a folder named after the .val.ts", () => { + expect(getNewJsonEntryPaths("/test/pages.val.ts", "/blog/hello")).toEqual({ + jsonPath: "/test/pages/blog/hello.val.json", + importPath: "./pages/blog/hello.val.json", + }); + }); + + test("handles nested module directories", () => { + expect( + getNewJsonEntryPaths("/app/support/[slug]/page.val.ts", "/support/faq"), + ).toEqual({ + jsonPath: "/app/support/[slug]/page/support/faq.val.json", + importPath: "./page/support/faq.val.json", + }); + }); +}); + +describe("findNestedJsonValuesRecords", () => { + test("a root jsonValues record is allowed → no offenders", () => { + const root: SerializedSchema = s + .record(s.object({ title: s.string() })) + .jsonValues() + ["executeSerialize"](); + expect(findNestedJsonValuesRecords(root)).toEqual([]); + }); + + test("a plain schema with no jsonValues → no offenders", () => { + const plain: SerializedSchema = s + .object({ pages: s.record(s.object({ title: s.string() })) }) + ["executeSerialize"](); + expect(findNestedJsonValuesRecords(plain)).toEqual([]); + }); + + test("jsonValues nested under an object is reported", () => { + const nested: SerializedSchema = s + .object({ + title: s.string(), + pages: s.record(s.object({ title: s.string() })).jsonValues(), + }) + ["executeSerialize"](); + expect(findNestedJsonValuesRecords(nested)).toEqual([["pages"]]); + }); + + test("jsonValues nested under an array is reported", () => { + const nested: SerializedSchema = s + .array( + s.object({ + pages: s.record(s.object({ title: s.string() })).jsonValues(), + }), + ) + ["executeSerialize"](); + expect(findNestedJsonValuesRecords(nested)).toEqual([["*", "pages"]]); + }); + + test("jsonValues nested under another record is reported", () => { + const nested: SerializedSchema = s + .record( + s.object({ + pages: s.record(s.object({ title: s.string() })).jsonValues(), + }), + ) + ["executeSerialize"](); + expect(findNestedJsonValuesRecords(nested)).toEqual([["*", "pages"]]); + }); + + test("multiple offenders are all reported", () => { + const nested: SerializedSchema = s + .object({ + a: s.record(s.object({ title: s.string() })).jsonValues(), + b: s.object({ + c: s.record(s.object({ title: s.string() })).jsonValues(), + }), + }) + ["executeSerialize"](); + expect(findNestedJsonValuesRecords(nested)).toEqual([["a"], ["b", "c"]]); + }); +}); + +describe("rebaseContentOp", () => { + test("drops the record + entry-key prefix from path and from", () => { + const res = rebaseContentOp( + { op: "move", from: ["/a", "items", "0"], path: ["/a", "items", "2"] }, + 1, + ); + expect(result.isOk(res) && res.value).toEqual({ + op: "move", + from: ["items", "0"], + path: ["items", "2"], + }); + }); + + test("rejects removing the entry root", () => { + const res = rebaseContentOp({ op: "remove", path: ["/a"] }, 1); + expect(result.isErr(res)).toBe(true); + }); + + test("rejects a file op", () => { + const res = rebaseContentOp( + { + op: "file", + path: ["/a", "img"], + filePath: "/public/val/x.png", + value: "data:...", + remote: false, + }, + 1, + ); + expect(result.isErr(res)).toBe(true); + }); +}); + +describe("applyJsonValuesEntryPatches", () => { + const schema: SerializedSchema = s + .record(s.object({ title: s.string(), order: s.number() })) + .jsonValues() + ["executeSerialize"](); + const patch = (patch: Patch, n = 1) => ({ + patchId: `p${n}` as PatchId, + patch, + }); + + test("replays a content sub-op onto the committed content", () => { + const res = applyJsonValuesEntryPatches({ + serializedSchema: schema, + entryKey: "/a", + baseContent: { title: "A", order: 1 }, + patches: [patch([{ op: "replace", path: ["/a", "title"], value: "A!" }])], + }); + expect(res).toEqual({ + kind: "content", + content: { title: "A!", order: 1 }, + appliedPatchIds: ["p1"], + }); + }); + + test("ignores ops for other entries and other schemas' paths", () => { + const res = applyJsonValuesEntryPatches({ + serializedSchema: schema, + entryKey: "/a", + baseContent: { title: "A", order: 1 }, + patches: [ + patch([{ op: "replace", path: ["/b", "title"], value: "B!" }], 1), + patch([{ op: "replace", path: ["/a", "order"], value: 9 }], 2), + ], + }); + expect(res).toEqual({ + kind: "content", + content: { title: "A", order: 9 }, + appliedPatchIds: ["p2"], + }); + }); + + test("a whole-entry add creates content that did not exist", () => { + const res = applyJsonValuesEntryPatches({ + serializedSchema: schema, + entryKey: "/new", + baseContent: undefined, + patches: [ + patch([{ op: "add", path: ["/new"], value: { title: "N", order: 3 } }]), + ], + }); + expect(res).toEqual({ + kind: "content", + content: { title: "N", order: 3 }, + appliedPatchIds: ["p1"], + }); + }); + + test("a whole-entry remove reports deleted", () => { + const res = applyJsonValuesEntryPatches({ + serializedSchema: schema, + entryKey: "/a", + baseContent: { title: "A", order: 1 }, + patches: [patch([{ op: "remove", path: ["/a"] }])], + }); + expect(res).toEqual({ kind: "deleted", appliedPatchIds: ["p1"] }); + }); + + test("an entry missing from the base with no patches is deleted", () => { + const res = applyJsonValuesEntryPatches({ + serializedSchema: schema, + entryKey: "/nope", + baseContent: undefined, + patches: [], + }); + expect(res).toEqual({ kind: "deleted", appliedPatchIds: [] }); + }); + + test("editing an entry that does not exist is an error", () => { + const res = applyJsonValuesEntryPatches({ + serializedSchema: schema, + entryKey: "/nope", + baseContent: undefined, + patches: [ + patch([{ op: "replace", path: ["/nope", "title"], value: "x" }]), + ], + }); + expect(res.kind).toBe("error"); + }); + + test("a whole-entry move into this key is reported as an error (needs the source entry)", () => { + const res = applyJsonValuesEntryPatches({ + serializedSchema: schema, + entryKey: "/renamed", + baseContent: undefined, + patches: [patch([{ op: "move", from: ["/a"], path: ["/renamed"] }])], + }); + expect(res.kind).toBe("error"); + }); + + test("without a schema nothing is treated as an entry op", () => { + const res = applyJsonValuesEntryPatches({ + serializedSchema: undefined, + entryKey: "/a", + baseContent: { title: "A", order: 1 }, + patches: [patch([{ op: "replace", path: ["/a", "title"], value: "A!" }])], + }); + expect(res).toEqual({ + kind: "content", + content: { title: "A", order: 1 }, + appliedPatchIds: [], + }); + }); +}); + +describe("resolveExistingJsonPath", () => { + test("resolves a hand-placed import path relative to the module dir", () => { + expect( + resolveExistingJsonPath("/test/pages.val.ts", "./content/hello.val.json"), + ).toBe("/test/content/hello.val.json"); + }); +}); diff --git a/packages/server/src/patch/jsonValuesPatch.ts b/packages/server/src/patch/jsonValuesPatch.ts new file mode 100644 index 000000000..f1b052cc0 --- /dev/null +++ b/packages/server/src/patch/jsonValuesPatch.ts @@ -0,0 +1,317 @@ +import * as path from "path"; +import type { PatchId, SerializedSchema } from "@valbuild/core"; +import { array, result } from "@valbuild/core/fp"; +import { + applyPatch, + deepClone, + JSONOps, + JSONValue, + Operation, + Patch, + PatchError, +} from "@valbuild/core/patch"; + +const jsonOps = new JSONOps(); + +/** + * Classification of a single patch op against a module's serialized schema, + * used by the commit flow to route ops for `.jsonValues()` records: + * + * - `normal`: the op does not descend into a `.jsonValues()` entry; apply it to + * the `.val.ts` as usual. + * - `entry`: the op targets a `.jsonValues()` entry. `recordPath` is the path to + * the record within the module source (empty for a root record/router), + * `entryKey` is the entry key, and `subPath` is the remaining path inside the + * entry (empty when the op targets the entry value itself, e.g. add/remove of + * the whole entry). + */ +export type JsonValuesOpClass = + | { kind: "normal" } + | { + kind: "entry"; + recordPath: string[]; + entryKey: string; + subPath: string[]; + }; + +/** + * Walks the serialized schema following the op path. When a `.jsonValues()` + * record is encountered, the next path segment is the entry key and everything + * after it lives inside the entry's `*.val.json` (so it does not touch the + * `.val.ts`). Returns `{ kind: "normal" }` when the op never enters a + * `.jsonValues()` record. + */ +export function classifyJsonValuesOp( + schema: SerializedSchema, + opPath: string[], +): JsonValuesOpClass { + let current: SerializedSchema | undefined = schema; + const recordPath: string[] = []; + for (let i = 0; i < opPath.length; i++) { + if (!current) { + return { kind: "normal" }; + } + if (current.type === "record" && current.jsonValues) { + return { + kind: "entry", + recordPath: recordPath.slice(), + entryKey: opPath[i], + subPath: opPath.slice(i + 1), + }; + } + const seg = opPath[i]; + current = descend(current, seg); + recordPath.push(seg); + } + return { kind: "normal" }; +} + +function descend( + schema: SerializedSchema, + key: string, +): SerializedSchema | undefined { + switch (schema.type) { + case "object": + return schema.items[key]; + case "record": + return schema.item; + case "array": + return schema.item; + default: + // Unions / primitives / leaf schemas: we cannot (or need not) descend + // further to find a jsonValues record. Anything below is a normal + // `.val.ts` edit. + return undefined; + } +} + +/** + * Finds every `.jsonValues()` record in a module's schema that is NOT the + * module's root, returning the path to each within the module source. + * + * `.jsonValues()` is only supported on a module's ROOT record/router: the + * `/json` endpoint keys entries by a single string, the Studio substitutes + * loaded content at the top level of the module source, and + * `validateJsonValuesEntries` only visits a root record. A nested one would + * silently skip content validation and hang the Studio on a 404, so we reject + * it up front instead (see {@link ValOps.initSources}). + */ +export function findNestedJsonValuesRecords( + schema: SerializedSchema, + path: string[] = [], +): string[][] { + const found: string[][] = []; + const rec = (current: SerializedSchema, currentPath: string[]) => { + if ( + current.type === "record" && + current.jsonValues && + currentPath.length > 0 + ) { + found.push(currentPath); + // Do not descend: everything below lives in the entry's `*.val.json`. + return; + } + switch (current.type) { + case "object": + for (const key of Object.keys(current.items)) { + rec(current.items[key], currentPath.concat(key)); + } + return; + case "record": + rec(current.item, currentPath.concat("*")); + return; + case "array": + rec(current.item, currentPath.concat("*")); + return; + case "union": + for (let i = 0; i < current.items.length; i++) { + rec(current.items[i], currentPath.concat(`union[${i}]`)); + } + return; + default: + return; + } + }; + rec(schema, path); + return found; +} + +/** + * The `.val.ts` suffix a module file path ends with. Stripping it yields the + * folder that a new entry's `*.val.json` files are nested under. + */ +const VAL_TS_SUFFIX = ".val.ts"; + +/** + * Computes the `*.val.json` file path (relative to rootDir) and the `import(...)` + * path (relative to the module's directory) for a NEW `.jsonValues()` entry, + * following the locked filename convention: the file mirrors the entry key under + * a folder named after the `.val.ts` (its `.val.ts` suffix becomes the folder). + * + * For module `/app/support/[slug]/page.val.ts` and key `/support/faq`: + * - jsonPath: `/app/support/[slug]/page/support/faq.val.json` + * - importPath: `./page/support/faq.val.json` + */ +export function getNewJsonEntryPaths( + moduleFilePath: string, + entryKey: string, +): { jsonPath: string; importPath: string } { + const base = moduleFilePath.endsWith(VAL_TS_SUFFIX) + ? moduleFilePath.slice(0, -VAL_TS_SUFFIX.length) + : moduleFilePath; + const keyRel = entryKey.replace(/^\//, ""); + const jsonPath = `${base}/${keyRel}.val.json`; + const moduleDir = path.posix.dirname(moduleFilePath); + let importPath = path.posix.relative(moduleDir, jsonPath); + if (!importPath.startsWith(".")) { + importPath = `./${importPath}`; + } + return { jsonPath, importPath }; +} + +/** + * Rebases a patch op that targets a `.jsonValues()` entry's content so its paths + * are relative to the entry's `*.val.json` root (drops the record + entry-key + * prefix). Used to replay the op against the backing JSON file. + */ +export function rebaseContentOp( + op: Operation, + prefixLen: number, +): result.Result { + const path = op.path.slice(prefixLen); + switch (op.op) { + case "add": + case "replace": + case "test": + return result.ok({ ...op, path }); + case "remove": { + if (!array.isNonEmpty(path)) { + return result.err( + new PatchError("Cannot remove the root of a jsonValues entry"), + ); + } + return result.ok({ ...op, path }); + } + case "move": { + const from = op.from.slice(prefixLen); + if (!array.isNonEmpty(from)) { + return result.err( + new PatchError("Cannot move from the root of a jsonValues entry"), + ); + } + return result.ok({ ...op, path, from }); + } + case "copy": + return result.ok({ ...op, path, from: op.from.slice(prefixLen) }); + case "file": + return result.err( + new PatchError("Cannot apply a file op to a jsonValues entry"), + ); + } +} + +/** The outcome of replaying pending patches onto one `.jsonValues()` entry. */ +export type JsonEntryResolution = + | { kind: "content"; content: JSONValue | null; appliedPatchIds: PatchId[] } + | { kind: "deleted"; appliedPatchIds: PatchId[] } + | { kind: "error"; message: string; patchId?: PatchId }; + +/** + * Replays the ops of `patches` that target ONE `.jsonValues()` entry onto its + * committed content, yielding the entry's draft content. + * + * This is the read-side counterpart to the commit flow in `ValOps.prepare`: + * both route ops with {@link classifyJsonValuesOp} and replay content sub-ops + * with {@link rebaseContentOp}, but this one produces a value instead of files + * and never touches the `.val.ts`. + * + * Root-only, like the rest of the `.jsonValues()` machinery: ops targeting a + * nested record are ignored (nested `.jsonValues()` is rejected at startup). + */ +export function applyJsonValuesEntryPatches(args: { + serializedSchema: SerializedSchema | undefined; + entryKey: string; + /** `undefined` when the entry does not exist in the committed source. */ + baseContent: JSONValue | undefined; + /** Ordered, already filtered to the entry's module. */ + patches: { patchId: PatchId; patch: Patch }[]; +}): JsonEntryResolution { + const { serializedSchema, entryKey, baseContent, patches } = args; + let content: JSONValue | undefined = baseContent; + let deleted = false; + const appliedPatchIds: PatchId[] = []; + for (const { patchId, patch } of patches) { + let touched = false; + for (const op of patch) { + if (op.op === "file") { + continue; + } + const cls = serializedSchema + ? classifyJsonValuesOp(serializedSchema, op.path) + : ({ kind: "normal" } as const); + if ( + cls.kind !== "entry" || + cls.recordPath.length > 0 || + cls.entryKey !== entryKey + ) { + continue; + } + touched = true; + if (cls.subPath.length === 0) { + if (op.op === "add" || op.op === "replace") { + content = op.value as JSONValue; + deleted = false; + } else if (op.op === "remove") { + content = undefined; + deleted = true; + } else { + // move/copy INTO this key: the content comes from the source entry, + // which the caller must resolve (it is a different `*.val.json`). + return { + kind: "error", + message: `Cannot resolve '${op.op}' of jsonValues entry '${entryKey}' from its own content`, + patchId, + }; + } + continue; + } + if (content === undefined) { + return { + kind: "error", + message: `Cannot edit jsonValues entry '${entryKey}': it does not exist`, + patchId, + }; + } + const rebased = rebaseContentOp(op, cls.recordPath.length + 1); + if (result.isErr(rebased)) { + return { kind: "error", message: rebased.error.message, patchId }; + } + const applied = applyPatch(deepClone(content), jsonOps, [rebased.value]); + if (result.isErr(applied)) { + return { kind: "error", message: applied.error.message, patchId }; + } + content = applied.value; + } + if (touched) { + appliedPatchIds.push(patchId); + } + } + if (deleted || content === undefined) { + return { kind: "deleted", appliedPatchIds }; + } + return { kind: "content", content, appliedPatchIds }; +} + +/** + * Resolves an EXISTING entry's `*.val.json` path (relative to rootDir) from the + * `import(...)` path recorded in the `.val.ts` thunk (from + * {@link analyzeJsonValuesEntries}). Existing files may have been hand-placed, + * so the import path is authoritative (hybrid authoring). + */ +export function resolveExistingJsonPath( + moduleFilePath: string, + importPath: string, +): string { + const moduleDir = path.posix.dirname(moduleFilePath); + return path.posix.join(moduleDir, importPath); +} diff --git a/packages/server/src/patch/ts/jsonReference.test.ts b/packages/server/src/patch/ts/jsonReference.test.ts new file mode 100644 index 000000000..f11d3c3c6 --- /dev/null +++ b/packages/server/src/patch/ts/jsonReference.test.ts @@ -0,0 +1,31 @@ +import ts from "typescript"; +import { createValJsonReference } from "./ops"; + +const printer = ts.createPrinter({ newLine: ts.NewLineKind.LineFeed }); + +function print(node: ts.Node): string { + const doc = ts.createSourceFile("out.ts", "", ts.ScriptTarget.ES2020, true); + return printer.printNode(ts.EmitHint.Unspecified, node, doc); +} + +describe("createValJsonReference", () => { + test("prints c.json(() => import(...))", () => { + const node = createValJsonReference("./page/blogs/test.val.json"); + const out = print(node); + expect(out).toContain("c.json("); + expect(out).toContain('import("./page/blogs/test.val.json")'); + expect(out).toContain("() =>"); + }); + + test("produces valid, re-parseable output", () => { + const node = createValJsonReference("./a.val.json"); + const out = print(node); + const reparsed = ts.createSourceFile( + "reparse.ts", + `const x = ${out};`, + ts.ScriptTarget.ES2020, + true, + ); + expect(reparsed.statements).toHaveLength(1); + }); +}); diff --git a/packages/server/src/patch/ts/jsonValuesEntry.test.ts b/packages/server/src/patch/ts/jsonValuesEntry.test.ts new file mode 100644 index 000000000..37227a4b9 --- /dev/null +++ b/packages/server/src/patch/ts/jsonValuesEntry.test.ts @@ -0,0 +1,72 @@ +import ts from "typescript"; +import { result } from "@valbuild/core/fp"; +import { insertValJsonEntry, removeValJsonEntry } from "./ops"; + +const MODULE = `import { s, c } from "../val.config"; + +export default c.define( + "/test/pages.val.ts", + s.record(s.object({ title: s.string() })).jsonValues(), + { + "/blog/hello": c.json(() => import("./content/hello.val.json")), + "/blog/world": c.json(() => import("./content/world.val.json")), + }, +); +`; + +function parse(src: string): ts.SourceFile { + return ts.createSourceFile("", src, ts.ScriptTarget.ES2015); +} + +function print(node: ts.SourceFile): string { + return node.getText(node); +} + +describe("insertValJsonEntry", () => { + test("appends a c.json(() => import(...)) property to the root record", () => { + const res = insertValJsonEntry( + parse(MODULE), + [], + "/blog/new", + "./pages/blog/new.val.json", + ); + if (result.isErr(res)) { + throw res.error; + } + const out = print(res.value); + expect(out).toContain( + `"/blog/new": c.json(() => import("./pages/blog/new.val.json"))`, + ); + // existing entries are untouched + expect(out).toContain( + `"/blog/hello": c.json(() => import("./content/hello.val.json"))`, + ); + }); + + test("fails when the entry key already exists", () => { + const res = insertValJsonEntry( + parse(MODULE), + [], + "/blog/hello", + "./pages/blog/hello.val.json", + ); + expect(result.isErr(res)).toBe(true); + }); +}); + +describe("removeValJsonEntry", () => { + test("removes an existing entry property", () => { + const res = removeValJsonEntry(parse(MODULE), [], "/blog/hello"); + if (result.isErr(res)) { + throw res.error; + } + const out = print(res.value); + expect(out).not.toContain(`"/blog/hello"`); + expect(out).toContain(`"/blog/world"`); + }); + + test("fails when the entry key does not exist", () => { + const res = removeValJsonEntry(parse(MODULE), [], "/blog/missing"); + expect(result.isErr(res)).toBe(true); + }); +}); diff --git a/packages/server/src/patch/ts/jsonValuesModule.test.ts b/packages/server/src/patch/ts/jsonValuesModule.test.ts new file mode 100644 index 000000000..0053a5bb2 --- /dev/null +++ b/packages/server/src/patch/ts/jsonValuesModule.test.ts @@ -0,0 +1,63 @@ +import ts from "typescript"; +import { analyzeValModule } from "./valModule"; +import { analyzeJsonValuesEntries } from "./jsonValuesModule"; +import { result } from "@valbuild/core/fp"; + +function analyze(code: string) { + const sf = ts.createSourceFile( + "test.val.ts", + code, + ts.ScriptTarget.ES2020, + true, + ); + const mod = analyzeValModule(sf); + if (result.isErr(mod)) { + throw new Error("analyzeValModule failed: " + JSON.stringify(mod.error)); + } + return analyzeJsonValuesEntries(mod.value.source); +} + +describe("analyzeJsonValuesEntries", () => { + test("extracts import path for each c.json entry", () => { + const entries = analyze(` + import { s, c } from "./val.config"; + export default c.define( + "/blogs.val.ts", + s.router(r, schema).jsonValues(), + { + "/blogs/a": c.json(() => import("./content/a.val.json")), + "/blogs/b": c.json(() => import("./content/b.val.json")), + } + ); + `); + expect(entries.size).toBe(2); + expect(entries.get("/blogs/a")).toEqual({ + importPath: "./content/a.val.json", + }); + expect(entries.get("/blogs/b")).toEqual({ + importPath: "./content/b.val.json", + }); + }); + + test("skips non-c.json entries", () => { + const entries = analyze(` + import { s, c } from "./val.config"; + export default c.define("/x.val.ts", s.record(schema), { + "/a": { title: "inline" }, + }); + `); + expect(entries.size).toBe(0); + }); + + test("handles a block-body thunk", () => { + const entries = analyze(` + import { s, c } from "./val.config"; + export default c.define("/x.val.ts", s.record(schema).jsonValues(), { + "/a": c.json(() => { return import("./a.val.json"); }), + }); + `); + expect(entries.get("/a")).toEqual({ + importPath: "./a.val.json", + }); + }); +}); diff --git a/packages/server/src/patch/ts/jsonValuesModule.ts b/packages/server/src/patch/ts/jsonValuesModule.ts new file mode 100644 index 000000000..7c9d17c45 --- /dev/null +++ b/packages/server/src/patch/ts/jsonValuesModule.ts @@ -0,0 +1,109 @@ +import ts from "typescript"; + +export type JsonValuesEntry = { + /** The string literal inside `import("...")` in the entry's thunk. */ + importPath: string; +}; + +/** + * Given the `source` object-literal expression of a `.jsonValues()` record / + * router (the 3rd argument to `c.define`, via {@link analyzeValModule}), + * extracts each entry's `c.json(() => import("..."))` import path, keyed by + * entry key. Entries whose value is not a `c.json(...)` call are skipped. + */ +export function analyzeJsonValuesEntries( + source: ts.Expression, +): Map { + const entries = new Map(); + if (!ts.isObjectLiteralExpression(source)) { + return entries; + } + for (const prop of source.properties) { + if (!ts.isPropertyAssignment(prop)) { + continue; + } + const key = getPropertyKey(prop.name); + if (key === null) { + continue; + } + const entry = analyzeCJsonCall(prop.initializer); + if (entry) { + entries.set(key, entry); + } + } + return entries; +} + +function getPropertyKey(name: ts.PropertyName): string | null { + if (ts.isStringLiteralLike(name)) { + return name.text; + } + if (ts.isIdentifier(name)) { + return name.text; + } + return null; +} + +function analyzeCJsonCall(node: ts.Expression): JsonValuesEntry | null { + // c.json(() => import("path")) + if (!ts.isCallExpression(node) || !isCJson(node.expression)) { + return null; + } + const [thunk] = node.arguments; + if (!thunk) { + return null; + } + const importPath = getImportPathFromThunk(thunk); + if (importPath === null) { + return null; + } + return { importPath }; +} + +function isCJson(expr: ts.Expression): boolean { + // c.json + return ( + ts.isPropertyAccessExpression(expr) && + ts.isIdentifier(expr.expression) && + expr.expression.text === "c" && + ts.isIdentifier(expr.name) && + expr.name.text === "json" + ); +} + +function getImportPathFromThunk(thunk: ts.Expression): string | null { + // () => import("path") (concise body) OR () => { return import("path"); } + if (!ts.isArrowFunction(thunk)) { + return null; + } + let importCall: ts.Expression | undefined; + if (ts.isCallExpression(thunk.body)) { + importCall = thunk.body; + } else if (ts.isBlock(thunk.body)) { + for (const stmt of thunk.body.statements) { + if ( + ts.isReturnStatement(stmt) && + stmt.expression && + ts.isCallExpression(stmt.expression) + ) { + importCall = stmt.expression; + break; + } + } + } + if (!importCall || !ts.isCallExpression(importCall)) { + return null; + } + const isImport = + importCall.expression.kind === ts.SyntaxKind.ImportKeyword || + (ts.isIdentifier(importCall.expression) && + importCall.expression.text === "import"); + if (!isImport) { + return null; + } + const arg = importCall.arguments[0]; + if (arg && ts.isStringLiteralLike(arg)) { + return arg.text; + } + return null; +} diff --git a/packages/server/src/patch/ts/ops.ts b/packages/server/src/patch/ts/ops.ts index 51380f5fe..a4c9d3f51 100644 --- a/packages/server/src/patch/ts/ops.ts +++ b/packages/server/src/patch/ts/ops.ts @@ -16,6 +16,7 @@ import { findValRemoteNodeArg, findValRemoteMetadataArg, } from "./syntax"; +import { analyzeValModule } from "./valModule"; import { deepEqual, isNotRoot, @@ -126,6 +127,142 @@ function createValRemoteReference(value: RemoteSource) { ); } +/** + * Builds the expression `c.json(() => import(""))` used to reference + * a lazily-loaded `*.val.json` entry of a `.jsonValues()` record. + */ +export function createValJsonReference(importPath: string): ts.Expression { + // () => import("") + // NOTE: an `import` identifier prints as the dynamic-import keyword call, + // which avoids casting the ImportKeyword token (not typed as an Expression). + const importCall = ts.factory.createCallExpression( + ts.factory.createIdentifier("import"), + undefined, + [ts.factory.createStringLiteral(importPath)], + ); + const thunk = ts.factory.createArrowFunction( + undefined, + undefined, + [], + undefined, + ts.factory.createToken(ts.SyntaxKind.EqualsGreaterThanToken), + importCall, + ); + // c.json() + return ts.factory.createCallExpression( + ts.factory.createPropertyAccessExpression( + ts.factory.createIdentifier("c"), + ts.factory.createIdentifier("json"), + ), + undefined, + [thunk], + ); +} + +/** + * Inserts a new `.jsonValues()` entry `"": c.json(() => import(""))` + * into the record's object literal in a `.val.ts` module. `recordPath` is the + * path from the module source to the record (empty for a root record/router). + * Fails if the entry key already exists. + */ +export function insertValJsonEntry( + document: ts.SourceFile, + recordPath: string[], + key: string, + importPath: string, +): TSOpsResult { + return pipe( + analyzeValModule(document), + result.flatMap(({ source }) => getAtPath(source, recordPath)), + result.flatMap((recordNode: ts.Expression): TSOpsResult => { + if (!ts.isObjectLiteralExpression(recordNode)) { + return result.err( + new PatchError( + "Cannot add jsonValues entry: record source is not an object literal", + ), + ); + } + return pipe( + findObjectPropertyAssignment(recordNode, key), + result.flatMap( + ( + assignment: ts.PropertyAssignment | undefined, + ): TSOpsResult => { + if (assignment) { + return result.err( + new PatchError( + `Cannot add jsonValues entry '${key}': it already exists`, + ), + ); + } + const property = ts.factory.createPropertyAssignment( + isValidIdentifier(key) + ? ts.factory.createIdentifier(key) + : ts.factory.createStringLiteral(key), + createValJsonReference(importPath), + ); + return result.ok( + insertAt( + document, + recordNode.properties, + recordNode.properties.length, + property, + ), + ); + }, + ), + ); + }), + ); +} + +/** + * Removes the `.jsonValues()` entry `""` from the record's object literal + * in a `.val.ts` module. `recordPath` is the path from the module source to the + * record (empty for a root record/router). Fails if the entry key is missing. + */ +export function removeValJsonEntry( + document: ts.SourceFile, + recordPath: string[], + key: string, +): TSOpsResult { + return pipe( + analyzeValModule(document), + result.flatMap(({ source }) => getAtPath(source, recordPath)), + result.flatMap((recordNode: ts.Expression): TSOpsResult => { + if (!ts.isObjectLiteralExpression(recordNode)) { + return result.err( + new PatchError( + "Cannot remove jsonValues entry: record source is not an object literal", + ), + ); + } + return pipe( + findObjectPropertyAssignment(recordNode, key), + result.flatMap( + ( + assignment: ts.PropertyAssignment | undefined, + ): TSOpsResult => { + if (!assignment) { + return result.err( + new PatchError( + `Cannot remove jsonValues entry '${key}': it does not exist`, + ), + ); + } + const [doc] = removeAt( + document, + recordNode.properties, + recordNode.properties.indexOf(assignment), + ); + return result.ok(doc); + }, + ), + ); + }), + ); +} + function toExpression(value: JSONValue): ts.Expression { if (typeof value === "string") { // TODO: Use configuration/heuristics to determine use of single quote or double quote diff --git a/packages/server/src/validateJsonValues.test.ts b/packages/server/src/validateJsonValues.test.ts new file mode 100644 index 000000000..33fa75118 --- /dev/null +++ b/packages/server/src/validateJsonValues.test.ts @@ -0,0 +1,87 @@ +import { initVal, ModuleFilePath } from "@valbuild/core"; +import { validateJsonValuesEntries } from "./validateJsonValues"; + +const { s, c } = initVal(); +const modulePath = "/blogs.val.ts" as ModuleFilePath; + +describe("validateJsonValuesEntries", () => { + const schema = s.record(s.object({ title: s.string() })).jsonValues(); + + test("returns no errors when all entry content is valid", async () => { + const source = { + "/a": c.json(() => Promise.resolve({ default: { title: "ok" } })), + "/b": c.json(() => Promise.resolve({ default: { title: "ok2" } })), + }; + const errors = await validateJsonValuesEntries(schema, source, modulePath); + expect(errors).toEqual({}); + }); + + test("reports validation errors for invalid entry content", async () => { + const source = { + "/a": c.json(() => Promise.resolve({ default: { title: "ok" } })), + // wrong leaf type for title — caught by the deferred content validation + "/bad": c.json(() => Promise.resolve({ default: { title: 123 } })), + }; + const errors = await validateJsonValuesEntries(schema, source, modulePath); + const keys = Object.keys(errors); + expect(keys.length).toBeGreaterThan(0); + expect(keys.some((k) => k.includes("/bad"))).toBe(true); + }); + + test("reports a load error when the entry thunk rejects", async () => { + const source = { + "/boom": c.json(() => Promise.reject(new Error("disk gone"))), + }; + const errors = await validateJsonValuesEntries(schema, source, modulePath); + const keys = Object.keys(errors); + expect(keys.length).toBe(1); + expect(errors[keys[0] as keyof typeof errors][0].message).toContain( + "Could not load JSON entry", + ); + }); + + test("skips non-jsonValues records (no content loading)", async () => { + const plainSchema = s.record(s.object({ title: s.string() })); + let loaded = false; + const source = { + "/a": c.json(() => { + loaded = true; + return Promise.resolve({ default: { title: "ok" } }); + }), + }; + const errors = await validateJsonValuesEntries( + plainSchema, + source, + modulePath, + ); + expect(errors).toEqual({}); + expect(loaded).toBe(false); + }); + + test("ROOT-ONLY contract: a nested jsonValues record is not visited", async () => { + // Pins the documented limitation. Nested `.jsonValues()` records are + // rejected up front as module errors (findNestedJsonValuesRecords), so they + // never reach here — but if that guard is relaxed without making this a + // recursive visitor, nested entries silently get NO content validation. + const nestedSchema = s.object({ + pages: s.record(s.object({ title: s.string() })).jsonValues(), + }); + let loaded = false; + const source = { + pages: { + // invalid content: would be an error if it were visited + "/a": c.json(() => { + loaded = true; + return Promise.resolve({ default: { title: 123 } }); + }), + }, + }; + const errors = await validateJsonValuesEntries( + nestedSchema, + source, + modulePath, + ); + expect(errors).toEqual({}); + expect(loaded).toBe(false); + }); +}); diff --git a/packages/server/src/validateJsonValues.ts b/packages/server/src/validateJsonValues.ts new file mode 100644 index 000000000..c38dbf770 --- /dev/null +++ b/packages/server/src/validateJsonValues.ts @@ -0,0 +1,84 @@ +import { + Internal, + ModuleFilePath, + RecordSchema, + Schema, + SelectorSource, + SourcePath, +} from "@valbuild/core"; +import { ValidationError } from "@valbuild/core"; + +/** + * Validates the content of every `.jsonValues()` entry in a module by loading + * each backing `*.val.json` (via its lazy import thunk) and checking it against + * the record's item schema. + * + * The record-level `executeValidate` only asserts the marker shape — content + * validation is deferred (the content isn't inlined). This performs that + * deferred deep validation server-side. Every loadable entry is validated; + * validation is allowed to be slower at scale, so a sha-keyed skip-cache is left + * as a later optimization. + * + * Entries without a runtime import thunk (transport markers / draft entries + * whose content lives in a patch) are skipped here — their content is validated + * where it is loaded (the single-entry fetch path). + * + * ROOT-ONLY BY CONTRACT: only a module whose root schema is a `.jsonValues()` + * record is visited. Nested `.jsonValues()` records are rejected up front as + * module errors (see `findNestedJsonValuesRecords`), so they can never reach + * here — if that guard is ever relaxed, this must become a recursive visitor or + * nested entries silently get no content validation at all. + */ +export async function validateJsonValuesEntries( + schema: Schema, + source: unknown, + modulePath: ModuleFilePath, +): Promise> { + const out: Record = {}; + if (!(schema instanceof RecordSchema)) { + return out; + } + if (!schema["executeSerialize"]().jsonValues) { + return out; + } + if (source === null || typeof source !== "object" || Array.isArray(source)) { + return out; + } + for (const [key, marker] of Object.entries(source)) { + const entryPath = Internal.createValPathOfItem( + modulePath as string as SourcePath, + key, + ); + if (!entryPath) { + continue; + } + if (!Internal.isJson(marker)) { + // a non-marker entry is already reported by the record-level validation + continue; + } + const thunk = Internal.getJsonImport(marker); + if (!thunk) { + continue; + } + let content: SelectorSource; + try { + content = (await thunk()).default as SelectorSource; + } catch (err) { + out[entryPath] = [ + { + message: `Could not load JSON entry '${key}': ${ + err instanceof Error ? err.message : String(err) + }`, + }, + ]; + continue; + } + const entryErrors = schema.validateJsonEntryContent(entryPath, content); + if (entryErrors) { + for (const [p, errs] of Object.entries(entryErrors)) { + out[p as SourcePath] = errs; + } + } + } + return out; +} diff --git a/packages/server/test/jsonValues-fixture/blogs.val.ts b/packages/server/test/jsonValues-fixture/blogs.val.ts new file mode 100644 index 000000000..5a7c0d2ca --- /dev/null +++ b/packages/server/test/jsonValues-fixture/blogs.val.ts @@ -0,0 +1,9 @@ +import { s, c } from "./val.config"; + +export default c.define( + "/blogs.val.ts", + s.record(s.object({ title: s.string() })).jsonValues(), + { + "/blogs/test": c.json(() => import("./page/blogs/test.val.json")), + }, +); diff --git a/packages/server/test/jsonValues-fixture/page/blogs/test.val.json b/packages/server/test/jsonValues-fixture/page/blogs/test.val.json new file mode 100644 index 000000000..ff16dfa0f --- /dev/null +++ b/packages/server/test/jsonValues-fixture/page/blogs/test.val.json @@ -0,0 +1,3 @@ +{ + "title": "Hello from JSON" +} diff --git a/packages/server/test/jsonValues-fixture/tsconfig.json b/packages/server/test/jsonValues-fixture/tsconfig.json new file mode 100644 index 000000000..f08c25e6d --- /dev/null +++ b/packages/server/test/jsonValues-fixture/tsconfig.json @@ -0,0 +1,12 @@ +{ + "compilerOptions": { + "target": "ES2020", + "module": "CommonJS", + "moduleResolution": "Node", + "esModuleInterop": true, + "resolveJsonModule": true, + "strict": true, + "skipLibCheck": true + }, + "include": ["**/*.ts", "**/*.json"] +} diff --git a/packages/server/test/jsonValues-fixture/val.config.ts b/packages/server/test/jsonValues-fixture/val.config.ts new file mode 100644 index 000000000..bb301c8ba --- /dev/null +++ b/packages/server/test/jsonValues-fixture/val.config.ts @@ -0,0 +1,4 @@ +import { initVal } from "@valbuild/core"; + +const { s, c, config } = initVal(); +export { s, c, config }; diff --git a/packages/server/test/jsonValues-fixture/val.modules.ts b/packages/server/test/jsonValues-fixture/val.modules.ts new file mode 100644 index 000000000..ffb0c1b24 --- /dev/null +++ b/packages/server/test/jsonValues-fixture/val.modules.ts @@ -0,0 +1,4 @@ +import { modules } from "@valbuild/core"; +import { config } from "./val.config"; + +export default modules(config, [{ def: () => import("./blogs.val.ts") }]); diff --git a/packages/shared/src/internal/ApiRoutes.ts b/packages/shared/src/internal/ApiRoutes.ts index 7d75f5591..eb13a668b 100644 --- a/packages/shared/src/internal/ApiRoutes.ts +++ b/packages/shared/src/internal/ApiRoutes.ts @@ -128,6 +128,19 @@ const onlyOneBooleanQueryParam = onlyOneStringQueryParam "Value must be true or false", ) .transform((arg) => arg === "true"); +const onlyOneIntQueryParam = onlyOneStringQueryParam + .refine( + (arg) => /^\d+$/.test(arg), + "Value must be a non-negative whole number", + ) + .transform((arg) => parseInt(arg, 10)); + +/** + * Upper bound on how many `.jsonValues()` entries one `/json` request may ask + * for. Callers page through bigger sets; the Studio chunks its key windows to + * this. Shared so client and server agree on the limit. + */ +export const JSON_ENTRIES_BATCH_MAX = 100; export const Api = { "/draft/enable": { @@ -1010,6 +1023,95 @@ export const Api = { ]), }, }, + // Loads the content of `.jsonValues()` record/router entries, so the Studio can + // lazily load just the entries it needs (instead of the whole record), and so + // the runtime can read draft edits in draft mode. + // + // Three request shapes, exactly one of which must be used: + // 1. `key=` — ONE entry; responds `{path, key, content}`. + // 2. `keys=&keys=&…` — a BATCH; responds `{path, entries, missing, errors}`. + // 3. `offset=&limit=` — a PAGE of every entry, in module key order; + // responds as (2) plus `{offset, limit, total}`. + // (2) and (3) are what make loading many entries one round trip instead of N: + // the server resolves sources and pending patches ONCE per request, not per entry. + // + // Batch failures are per entry, never whole-request: a key with no entry lands in + // `missing`, a key whose `*.val.json` fails to load lands in `errors`. Only a + // missing/non-record MODULE is a 404. + // + // `apply_patches` defaults to TRUE (as on `/sources/~`): the server replays + // pending patches for the entry. The Studio passes `false` explicitly, because + // it owns in-flight client patches the server has not seen yet and applies + // them itself — letting the server apply them too would double-apply. + // + // Shape (3) REQUIRES `apply_patches=false`. Enumerating "every entry" from the + // base source would silently omit draft-added keys, and the only all-mode caller + // (the Studio) derives its key set from its own patched source anyway. Callers + // that need draft-aware enumeration pass explicit `keys`. + "/json": { + GET: { + req: { + query: { + path: onlyOneStringQueryParam, + key: onlyOneStringQueryParam.optional(), + keys: z.array(z.string()).max(JSON_ENTRIES_BATCH_MAX).optional(), + offset: onlyOneIntQueryParam.optional(), + limit: onlyOneIntQueryParam + .refine( + (arg) => arg > 0 && arg <= JSON_ENTRIES_BATCH_MAX, + `limit must be between 1 and ${JSON_ENTRIES_BATCH_MAX}`, + ) + .optional(), + apply_patches: onlyOneBooleanQueryParam.optional(), + }, + cookies: { [VAL_SESSION_COOKIE]: z.string().optional() }, + }, + res: z.union([ + unauthorizedResponse, + notFoundResponse, + // Shape (1): single `key`. + z.object({ + status: z.literal(200), + json: z.object({ + path: ModuleFilePath, + key: z.string(), + // The entry's JSON content (or null if the entry has no value). + content: z.any(), + }), + }), + // Shapes (2) and (3): `keys` or `offset`+`limit`. + z.object({ + status: z.literal(200), + json: z.object({ + path: ModuleFilePath, + entries: z.array( + z.object({ + key: z.string(), + // The entry's JSON content (or null if the entry has no value). + content: z.any(), + }), + ), + // Requested keys that have no entry (neither committed nor drafted). + missing: z.array(z.string()), + // Requested keys whose content could not be loaded. + errors: z.array(z.object({ key: z.string(), message: z.string() })), + // All-mode only: the resolved window and the record's total key count. + offset: z.number().optional(), + limit: z.number().optional(), + total: z.number().optional(), + }), + }), + z.object({ + status: z.literal(400), + json: GenericError, + }), + z.object({ + status: z.literal(500), + json: GenericError, + }), + ]), + }, + }, "/ai/initialize": { POST: { req: { @@ -1252,7 +1354,14 @@ type DefinedObject = Pick>; * * Do not change this without updating the ValRouter query parsing logic * */ -export type ValidQueryParamTypes = boolean | string | string[] | undefined; +// `number` is allowed because the client stringifies every query value before it +// goes on the URL (see createValClient), so a numeric param round-trips fine. +export type ValidQueryParamTypes = + | boolean + | number + | string + | string[] + | undefined; export type ApiEndpoint = { req: { path?: z.ZodString | z.ZodOptional; diff --git a/packages/ui/package.json b/packages/ui/package.json index a355590f7..838da425b 100644 --- a/packages/ui/package.json +++ b/packages/ui/package.json @@ -1,6 +1,6 @@ { "name": "@valbuild/ui", - "version": "0.97.1", + "version": "0.97.3", "sideEffects": false, "repository": { "type": "git", diff --git a/packages/ui/spa/ValSyncEngine.test.ts b/packages/ui/spa/ValSyncEngine.test.ts index 3c0e775d8..1cb5ae3e7 100644 --- a/packages/ui/spa/ValSyncEngine.test.ts +++ b/packages/ui/spa/ValSyncEngine.test.ts @@ -449,6 +449,458 @@ describe("ValSyncEngine", () => { "module-schema-not-found", ); }); + + describe("jsonValues", () => { + const PAGES = "/pages.val.ts" as const; + + /** + * A `.jsonValues()` record module: the source only carries lazy markers, so + * the engine must fetch each entry's content via GET /json. + */ + function setupJsonValues() { + const { s, c, config } = initVal(); + const schema = s + .record(s.object({ title: s.string(), order: s.number() })) + .jsonValues(); + const valModule = c.define(PAGES, schema, { + "/a": c.json(() => + Promise.resolve({ default: { title: "A", order: 1 } }), + ), + "/b": c.json(() => + Promise.resolve({ default: { title: "B", order: 2 } }), + ), + }); + const tester = new SyncEngineTester("fs", [valModule as any], config); + tester.fakeJsonEntries[PAGES] = { + "/a": { title: "A", order: 1 }, + "/b": { title: "B", order: 2 }, + }; + return { tester, config }; + } + + /** Lets the `/json` promise chain settle. */ + const flush = () => new Promise((resolve) => setTimeout(resolve, 0)); + + test("requestJsonEntry substitutes the loaded content into the source", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + // Before loading, the entry is an opaque marker. + const before = engine.getSourceSnapshot(toModuleFilePath(PAGES)) + .data as any; + expect(Internal.isJson(before["/a"])).toBe(true); + + engine.requestJsonEntry(toModuleFilePath(PAGES), "/a"); + await flush(); + + const after = engine.getSourceSnapshot(toModuleFilePath(PAGES)) + .data as any; + expect(after["/a"]).toEqual({ title: "A", order: 1 }); + // the un-requested entry is untouched + expect(Internal.isJson(after["/b"])).toBe(true); + expect(tester.jsonRequestCounts[`${PAGES}\0/a`]).toBe(1); + }); + + test("a loaded entry is not refetched", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + engine.requestJsonEntry(toModuleFilePath(PAGES), "/a"); + await flush(); + engine.requestJsonEntry(toModuleFilePath(PAGES), "/a"); + engine.requestJsonEntry(toModuleFilePath(PAGES), "/a"); + await flush(); + expect(tester.jsonRequestCounts[`${PAGES}\0/a`]).toBe(1); + }); + + test("a failed load is memoized: no refetch loop, and an error is exposed", async () => { + // Regression: a failing entry used to be refetched on every remount and + // rendered a spinner forever, because nothing was cached on failure. + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + engine.requestJsonEntry(toModuleFilePath(PAGES), "/missing"); + await flush(); + + expect( + engine.getJsonEntryError(toModuleFilePath(PAGES), "/missing"), + ).toContain("Entry not found"); + expect(tester.jsonRequestCounts[`${PAGES}\0/missing`]).toBe(1); + + // Subsequent requests must NOT hit the endpoint again. + engine.requestJsonEntry(toModuleFilePath(PAGES), "/missing"); + engine.requestJsonEntry(toModuleFilePath(PAGES), "/missing"); + await flush(); + expect(tester.jsonRequestCounts[`${PAGES}\0/missing`]).toBe(1); + }); + + test("retryJsonEntry clears the memo and refetches", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + engine.requestJsonEntry(toModuleFilePath(PAGES), "/late"); + await flush(); + expect( + engine.getJsonEntryError(toModuleFilePath(PAGES), "/late"), + ).not.toBe(null); + + // The entry shows up server-side, then the user retries. + tester.fakeJsonEntries[PAGES]["/late"] = { title: "Late", order: 3 }; + engine.retryJsonEntry(toModuleFilePath(PAGES), "/late"); + await flush(); + + expect(engine.getJsonEntryError(toModuleFilePath(PAGES), "/late")).toBe( + null, + ); + expect(tester.jsonRequestCounts[`${PAGES}\0/late`]).toBe(2); + }); + + test("publish refetches loaded entries (a published edit must not revert)", async () => { + // Regression: jsonEntryContents was only cleared on init/reset. A + // content-only edit does not change the module source (bare markers), so + // sourcesSha is unchanged and nothing refetched — after publish the + // pre-edit content was re-substituted and the edit looked reverted. + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/a"); + expect( + (engine.getSourceSnapshot(toModuleFilePath(PAGES)).data as any)["/a"], + ).toEqual({ title: "A", order: 1 }); + + // Edit the entry, then publish. The fake server commits it: the entry's + // *.val.json now holds the new content, the module source is unchanged. + const res = engine.addPatch( + toSourcePath(PAGES), + "record", + [{ op: "replace", path: ["/a", "title"], value: "A edited" }], + tester.getNextNow(), + ); + expect(res.status).toBe("patch-added"); + tester.simulatePassingOfSeconds(5); + await engine.sync(tester.getNextNow()); + await flush(); + const patchIds = tester.fakePatches.map((p) => p.patchId); + tester.fakeJsonEntries[PAGES]["/a"] = { title: "A edited", order: 1 }; + const requestsBeforePublish = + tester.jsonRequestCounts[`${PAGES}\0/a`] ?? 0; + + expect( + await engine.publish(patchIds, undefined, tester.getNextNow()), + ).toMatchObject({ status: "done" }); + await flush(); + + expect(tester.jsonRequestCounts[`${PAGES}\0/a`]).toBe( + requestsBeforePublish + 1, + ); + expect( + (engine.getSourceSnapshot(toModuleFilePath(PAGES)).data as any)["/a"], + ).toEqual({ title: "A edited", order: 1 }); + }); + + test("an entry invalidated mid-flight is refetched (a stale response must not win)", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/a"); + engine.addPatch( + toSourcePath(PAGES), + "record", + [{ op: "replace", path: ["/a", "title"], value: "A edited" }], + tester.getNextNow(), + ); + tester.simulatePassingOfSeconds(5); + await engine.sync(tester.getNextNow()); + const patchIds = tester.fakePatches.map((p) => p.patchId); + + // Publish WITHOUT letting the sync-triggered refetch settle first, so the + // entry is invalidated while a request is still in flight. That in-flight + // response predates the publish and must not be the final value. + tester.fakeJsonEntries[PAGES]["/a"] = { title: "A edited", order: 1 }; + await engine.publish(patchIds, undefined, tester.getNextNow()); + await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/a"); + await flush(); + + expect( + (engine.getSourceSnapshot(toModuleFilePath(PAGES)).data as any)["/a"], + ).toEqual({ title: "A edited", order: 1 }); + }); + + test("ensureJsonEntry resolves once content is loaded", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/b"); + expect( + (engine.getSourceSnapshot(toModuleFilePath(PAGES)).data as any)["/b"], + ).toEqual({ title: "B", order: 2 }); + + // Already loaded: resolves immediately, no second request. + await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/b"); + expect(tester.jsonRequestCounts[`${PAGES}\0/b`]).toBe(1); + }); + + test("an UNLOADED entry renamed by a pending patch still loads (rename then reload)", async () => { + // Regression: after a reload, a pending rename leaves the *marker* under + // the new key. Requesting that key from /json 404s, because the committed + // source only knows the old key. The engine must map back to the base key + // — loading it there also lets the move patch relocate the content. + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + // Rename WITHOUT ever loading the entry (as after a fresh page load). + engine.addPatch( + toSourcePath(PAGES), + "record", + [{ op: "move", from: ["/a"], path: ["/renamed"] }], + tester.getNextNow(), + ); + expect( + Internal.isJson( + (engine.getSourceSnapshot(toModuleFilePath(PAGES)).data as any)[ + "/renamed" + ], + ), + ).toBe(true); + + await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/renamed"); + await flush(); + + // Fetched under the BASE key, not the renamed one. + expect(tester.jsonRequestCounts[`${PAGES}\0/a`]).toBe(1); + expect(tester.jsonRequestCounts[`${PAGES}\0/renamed`]).toBeUndefined(); + expect( + engine.getJsonEntryError(toModuleFilePath(PAGES), "/renamed"), + ).toBe(null); + const source = engine.getSourceSnapshot(toModuleFilePath(PAGES)) + .data as any; + expect(source["/a"]).toBeUndefined(); + expect(source["/renamed"]).toEqual({ title: "A", order: 1 }); + }); + + test("a move patch on a loaded entry carries real content to the new key", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/a"); + engine.addPatch( + toSourcePath(PAGES), + "record", + [{ op: "move", from: ["/a"], path: ["/renamed"] }], + tester.getNextNow(), + ); + + const source = engine.getSourceSnapshot(toModuleFilePath(PAGES)) + .data as any; + expect(source["/a"]).toBeUndefined(); + // real content, NOT an opaque marker + expect(source["/renamed"]).toEqual({ title: "A", order: 1 }); + }); + + describe("batch loading", () => { + test("requestJsonEntries loads a window in ONE request", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + engine.requestJsonEntries(toModuleFilePath(PAGES), ["/a", "/b"]); + await flush(); + + const source = engine.getSourceSnapshot(toModuleFilePath(PAGES)) + .data as any; + expect(source["/a"]).toEqual({ title: "A", order: 1 }); + expect(source["/b"]).toEqual({ title: "B", order: 2 }); + // Two keys, ONE HTTP request — the point of the batch. + expect(tester.jsonBatchRequestCounts[PAGES]).toBe(1); + }); + + test("already-loaded and in-flight keys are not refetched", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + await engine.ensureJsonEntry(toModuleFilePath(PAGES), "/a"); + const batchesAfterFirstLoad = tester.jsonBatchRequestCounts[PAGES] ?? 0; + // /a is cached; only /b should go out. + engine.requestJsonEntries(toModuleFilePath(PAGES), ["/a", "/b"]); + // Re-rendering the same window while it is in flight must add nothing. + engine.requestJsonEntries(toModuleFilePath(PAGES), ["/a", "/b"]); + await flush(); + + expect(tester.jsonRequestCounts[`${PAGES}\0/a`]).toBe(1); + expect(tester.jsonRequestCounts[`${PAGES}\0/b`]).toBe(1); + // Two window calls, ONE additional request. + expect(tester.jsonBatchRequestCounts[PAGES]).toBe( + batchesAfterFirstLoad + 1, + ); + }); + + test("single-entry requests in the same tick collapse into ONE request", async () => { + // Regression: a record list renders a per key and each one + // triggers requestJsonEntry for its own entry, so opening a record with N + // entries fired N requests. They must coalesce. + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + engine.requestJsonEntry(toModuleFilePath(PAGES), "/a"); + engine.requestJsonEntry(toModuleFilePath(PAGES), "/b"); + await flush(); + + expect(tester.jsonBatchRequestCounts[PAGES]).toBe(1); + const source = engine.getSourceSnapshot(toModuleFilePath(PAGES)) + .data as any; + expect(source["/a"]).toEqual({ title: "A", order: 1 }); + expect(source["/b"]).toEqual({ title: "B", order: 2 }); + }); + + test("keys that exist only in a pending patch are never requested", async () => { + // They have no committed content: their value comes from the patch, so + // asking /json would 404 and wrongly mark the row errored. + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + engine.addPatch( + toSourcePath(PAGES), + "record", + [{ op: "add", path: ["/drafted"], value: { title: "D", order: 3 } }], + tester.getNextNow(), + ); + engine.requestJsonEntries(toModuleFilePath(PAGES), ["/drafted"]); + await flush(); + + expect(tester.jsonRequestCounts[`${PAGES}\0/drafted`]).toBeUndefined(); + expect( + engine.getJsonEntryError(toModuleFilePath(PAGES), "/drafted"), + ).toBe(null); + expect( + (engine.getSourceSnapshot(toModuleFilePath(PAGES)).data as any)[ + "/drafted" + ], + ).toEqual({ title: "D", order: 3 }); + }); + + test("ensureJsonEntries loads every committed entry and reports complete", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + const res = await engine.ensureJsonEntries([toModuleFilePath(PAGES)]); + expect(res).toEqual({ complete: true, errors: [] }); + const source = engine.getSourceSnapshot(toModuleFilePath(PAGES)) + .data as any; + expect(source["/a"]).toEqual({ title: "A", order: 1 }); + expect(source["/b"]).toEqual({ title: "B", order: 2 }); + expect(tester.jsonBatchRequestCounts[PAGES]).toBe(1); + + // Fully cached: resolves without another request. + expect( + await engine.ensureJsonEntries([toModuleFilePath(PAGES)]), + ).toEqual({ complete: true, errors: [] }); + expect(tester.jsonBatchRequestCounts[PAGES]).toBe(1); + }); + + test("ensureJsonEntries reports NOT complete when an entry fails", async () => { + // The whole point: a guard that gates a delete must never read a failed + // load as "no references found". + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + delete tester.fakeJsonEntries[PAGES]["/b"]; + + const res = await engine.ensureJsonEntries([toModuleFilePath(PAGES)]); + expect(res.complete).toBe(false); + expect(res.errors).toHaveLength(1); + expect(res.errors[0]).toMatchObject({ + moduleFilePath: PAGES, + key: "/b", + }); + // The entry that DID load is still usable. + expect( + (engine.getSourceSnapshot(toModuleFilePath(PAGES)).data as any)["/a"], + ).toEqual({ title: "A", order: 1 }); + }); + + test("progress counts up across the run and resets when it settles", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + expect(engine.getJsonEntriesProgressSnapshot()).toEqual({ + status: "idle", + loaded: 0, + total: 0, + percentage: 100, + }); + + const done = engine.ensureJsonEntries([toModuleFilePath(PAGES)]); + const during = engine.getJsonEntriesProgressSnapshot(); + expect(during.status).toBe("loading"); + expect(during.total).toBe(2); + expect(during.loaded).toBe(0); + expect(during.percentage).toBe(0); + + await done; + // Nothing in flight: the run is over and the next one starts from zero. + expect(engine.getJsonEntriesProgressSnapshot()).toEqual({ + status: "idle", + loaded: 0, + total: 0, + percentage: 100, + }); + }); + + test("progress notifies subscribers", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + let notified = 0; + const unsubscribe = engine.subscribe("json-entries-progress")(() => { + notified++; + }); + + await engine.ensureJsonEntries([toModuleFilePath(PAGES)]); + expect(notified).toBeGreaterThan(0); + unsubscribe(); + }); + + test("publish refetches loaded entries in ONE batch, not one request each", async () => { + const { tester } = setupJsonValues(); + const engine = await tester.createInitializedSyncEngine(); + + await engine.ensureJsonEntries([toModuleFilePath(PAGES)]); + + engine.addPatch( + toSourcePath(PAGES), + "record", + [{ op: "replace", path: ["/a", "title"], value: "A edited" }], + tester.getNextNow(), + ); + tester.simulatePassingOfSeconds(5); + await engine.sync(tester.getNextNow()); + await flush(); + const patchIds = tester.fakePatches.map((p) => p.patchId); + tester.fakeJsonEntries[PAGES]["/a"] = { title: "A edited", order: 1 }; + // Measured from just before publish: the sync above also refreshes the + // module's entries (its own single batch). + const batchesBefore = tester.jsonBatchRequestCounts[PAGES] ?? 0; + const keyRequestsBefore = { + a: tester.jsonRequestCounts[`${PAGES}\0/a`] ?? 0, + b: tester.jsonRequestCounts[`${PAGES}\0/b`] ?? 0, + }; + + expect( + await engine.publish(patchIds, undefined, tester.getNextNow()), + ).toMatchObject({ status: "done" }); + await flush(); + + // BOTH cached entries were invalidated and refetched... + expect(tester.jsonRequestCounts[`${PAGES}\0/a`]).toBe( + keyRequestsBefore.a + 1, + ); + expect(tester.jsonRequestCounts[`${PAGES}\0/b`]).toBe( + keyRequestsBefore.b + 1, + ); + // ...in ONE request, not one per entry. + expect(tester.jsonBatchRequestCounts[PAGES]).toBe(batchesBefore + 1); + expect( + (engine.getSourceSnapshot(toModuleFilePath(PAGES)).data as any)["/a"], + ).toEqual({ title: "A edited", order: 1 }); + }); + }); + }); }); function toModuleFilePath(moduleFilePath: `/${string}.val.ts`): ModuleFilePath { @@ -497,6 +949,12 @@ type FakeApi = { "/schema": { GET: z.infer | ClientFetchErrors | null; }; + "/json": { + GET: z.infer | ClientFetchErrors | null; + }; + "/save": { + POST: z.infer | ClientFetchErrors | null; + }; }; class SyncEngineTester { @@ -514,6 +972,12 @@ class SyncEngineTester { fakeSources: Record; now: number; fakeResponses: Partial; + /** Committed `.jsonValues()` entry content, by module then entry key. */ + fakeJsonEntries: Record> = {}; + /** How many times GET /json was called, keyed `${path}\0${key}`. */ + jsonRequestCounts: Record = {}; + /** HTTP requests (not keys) served by the batch shapes of `/json`, per module. */ + jsonBatchRequestCounts: Record = {}; constructor( private mode: "fs" | "http", @@ -597,6 +1061,73 @@ class SyncEngineTester { }; } + getJson( + req: InferReq, + ): z.infer { + const path = req.query.path; + const { key, keys, offset, limit } = req.query; + const countKey = (k: string) => { + this.jsonRequestCounts[`${path}\0${k}`] = + (this.jsonRequestCounts[`${path}\0${k}`] ?? 0) + 1; + }; + const available = this.fakeJsonEntries[path]; + if (key !== undefined) { + countKey(key); + if (available === undefined || !(key in available)) { + return { + status: 404, + json: { message: `Entry not found: ${key} in ${path}` }, + }; + } + return { + status: 200, + json: { + path: path as ModuleFilePath, + key, + content: available[key], + }, + }; + } + // Batch shapes: one HTTP request, per-key outcomes. + this.jsonBatchRequestCounts[path] = + (this.jsonBatchRequestCounts[path] ?? 0) + 1; + const allKeys = Object.keys(available ?? {}); + const isWindow = offset !== undefined && limit !== undefined; + const requestedKeys = isWindow + ? allKeys.slice(offset, offset + limit) + : keys; + if (requestedKeys === undefined) { + return { + status: 400, + json: { + message: + "Exactly one of 'key', 'keys' or 'offset'+'limit' must be given", + }, + }; + } + const entries: { key: string; content: unknown }[] = []; + const missing: string[] = []; + for (const requestedKey of requestedKeys) { + countKey(requestedKey); + if (available !== undefined && requestedKey in available) { + entries.push({ key: requestedKey, content: available[requestedKey] }); + } else { + missing.push(requestedKey); + } + } + return { + status: 200, + json: { + path: path as ModuleFilePath, + entries, + missing, + errors: [], + ...(isWindow ? { offset, limit } : {}), + total: allKeys.length, + }, + }; + } + putPatches( req: InferReq, ): z.infer { @@ -856,6 +1387,16 @@ class SyncEngineTester { if (route === "/schema" && method === "GET") { return this.getSchema(req); } + if (route === "/json" && method === "GET") { + // NOTE: must be a Promise — the sync engine calls `.then()` on it. + return Promise.resolve(this.getJson(req)); + } + if (route === "/save" && method === "POST") { + // Publishing commits the pending patches; the fake server just drops + // them (fs mode behaviour). + this.fakePatches = []; + return { status: 200, json: {} }; + } return { status: 404, json: { diff --git a/packages/ui/spa/ValSyncEngine.ts b/packages/ui/spa/ValSyncEngine.ts index f2d6d9229..8d40a4d71 100644 --- a/packages/ui/spa/ValSyncEngine.ts +++ b/packages/ui/spa/ValSyncEngine.ts @@ -25,6 +25,7 @@ import { JSONValue, } from "@valbuild/core/patch"; import { + JSON_ENTRIES_BATCH_MAX, ParentRef, ValClient, Patch, @@ -33,7 +34,10 @@ import { import { canMerge } from "./utils/mergePatches"; import { PatchSets, SerializedPatchSet } from "./utils/PatchSets"; import { ReifiedRender } from "@valbuild/core"; -import { ValidationWorkerClient } from "./validation/ValidationWorkerClient"; +import { + ValidationWorkerClient, + type ValidationWorkerFactory, +} from "./validation/ValidationWorkerClient"; import { partitionValidationErrors } from "./validation/partitionValidationErrors"; /** @@ -135,6 +139,53 @@ export class ValSyncEngine { } | undefined > | null; + /** + * Loaded content for `.jsonValues()` record entries, keyed by module then + * entry key. The on-disk source only carries lazy `{ _type:"json" }` + * markers; the Studio fetches an entry's content on demand via + * `requestJsonEntry` (GET /json) and `getPatchedSource` substitutes it in so + * field resolution/rendering works. + */ + private jsonEntryContents: Record> = + {}; + /** In-flight json entry loads, keyed `${moduleFilePath}\0${key}`. */ + private loadingJsonEntries: Map> = new Map(); + /** + * Entries whose last load failed, keyed by module then entry key. Memoizing + * the failure is what stops a permanently-failing entry from being refetched + * on every field remount (and from rendering a spinner forever): the field + * hooks surface this as an error state instead. Cleared by `retryJsonEntry` + * and whenever the module's server source is replaced. + */ + private jsonEntryErrors: Record> = {}; + /** + * Loaded entries whose committed content may now be out of date (the module's + * server source was replaced, e.g. after publish), keyed + * `${moduleFilePath}\0${key}`. They keep rendering their old content until the + * refetch lands, so there is no loading flash — but they MUST be refetched, or + * a published edit appears to revert to the pre-edit content. + */ + private staleJsonEntries: Set = new Set(); + /** + * Progress of the CURRENT json-entry load run, spanning every module and every + * batch in flight — deliberately not per module, so a UI percentage does not + * visibly reset at each module boundary. Counts up as keys resolve (loaded, + * missing or failed all count as resolved) and resets to zero once nothing is + * in flight. + */ + private jsonEntriesProgress: { requested: number; resolved: number } = { + requested: 0, + resolved: 0, + }; + /** + * Keys queued by {@link requestJsonEntry} / {@link requestJsonEntries} for the + * next coalesced flush. A record list renders one `` per key and each + * asks for its own entry, so without this a record with N entries fires N + * requests; with it, one render pass costs one request per module. + */ + private pendingJsonEntryRequests: Map> = + new Map(); + private jsonEntryFlushScheduled = false; private renders: Record | null; private schemas: Record | null; private serverSideSchemaSha: string | null; @@ -238,6 +289,12 @@ export class ValSyncEngine { private readonly overlayEmitter: | typeof defaultOverlayEmitter | undefined = undefined, + // Injected by the composition root (ValProvider). Kept out of this file so + // the worker's import.meta reference never reaches the Jest-compiled core. + // When undefined (tests / SSR / stories) validation runs on the main thread. + private readonly createValidationWorker: + | ValidationWorkerFactory + | undefined = undefined, ) { this.initializedAt = null; this.forceSyncAllModules = true; @@ -386,6 +443,16 @@ export class ValSyncEngine { this.sourcesSha = null; this.serverSources = null; this.patchedSourcesCache = null; + this.jsonEntryContents = {}; + this.loadingJsonEntries = new Map(); + this.jsonEntryErrors = {}; + this.staleJsonEntries = new Set(); + this.jsonEntriesProgress = { requested: 0, resolved: 0 }; + this.cachedJsonEntriesProgressSnapshot = null; + this.pendingJsonEntryRequests = new Map(); + // Deliberately NOT clearing jsonEntryFlushScheduled: a flush may already be + // queued, and it must find an empty pending map rather than a second flush + // being scheduled behind it. this.renders = null; this.globalServerSidePatchIds = []; this.syncedServerSidePatchIds = []; @@ -466,6 +533,9 @@ export class ValSyncEngine { type: "all-validation-errors", ): (listener: () => void) => () => void; subscribe(type: "initialized-at"): (listener: () => void) => () => void; + subscribe( + type: "json-entries-progress", + ): (listener: () => void) => () => void; subscribe( type: "sync-status", path: SourcePath, @@ -554,6 +624,11 @@ export class ValSyncEngine { this.emit(this.listeners["initialized-at"]?.[globalNamespace]); } + private invalidateJsonEntriesProgress() { + this.cachedJsonEntriesProgressSnapshot = null; + this.emit(this.listeners["json-entries-progress"]?.[globalNamespace]); + } + private invalidateSource(moduleFilePath: ModuleFilePath) { if (this.cachedSourceSnapshots !== null) { const keysToRemove: string[] = []; @@ -816,11 +891,562 @@ export class ValSyncEngine { * Otherwise we rebuild from `serverSources`, which covers patch deletion, * server-side reorder, and any other non-append change. */ + /** + * Lazily loads the content of a single `.jsonValues()` entry and folds it into + * the source view (re-rendering subscribers). No-op if the entry is already + * loaded or a load is in flight. Called by the field hooks when the Studio + * renders a path that descends into an un-loaded json marker. + * + * COALESCED: every call in the same tick becomes ONE `/json` request per + * module. That matters because a record list renders a `` per key and + * each one asks for its own entry — before coalescing, opening a record with + * N entries fired N requests. + */ + requestJsonEntry(moduleFilePath: ModuleFilePath, key: string): void { + this.coalesceJsonEntryRequests(moduleFilePath, [key]); + } + + /** + * Queues keys for the next coalesced flush. The flush is a microtask, so all + * the field effects of one render pass land in a single request per module. + */ + private coalesceJsonEntryRequests( + moduleFilePath: ModuleFilePath, + keys: string[], + ): void { + let pending = this.pendingJsonEntryRequests.get(moduleFilePath); + if (pending === undefined) { + pending = new Set(); + this.pendingJsonEntryRequests.set(moduleFilePath, pending); + } + for (const key of keys) { + pending.add(key); + } + if (this.jsonEntryFlushScheduled) { + return; + } + this.jsonEntryFlushScheduled = true; + queueMicrotask(() => { + this.jsonEntryFlushScheduled = false; + const requests = Array.from( + this.pendingJsonEntryRequests, + ([path, requestedKeys]) => ({ + moduleFilePath: path, + keys: Array.from(requestedKeys), + }), + ); + this.pendingJsonEntryRequests.clear(); + void this.loadJsonEntriesSettled(requests); + }); + } + + /** + * Maps an entry key as it appears in the PATCHED source back to the key it + * has in the committed base source, by undoing pending whole-entry renames. + * + * `/json` can only resolve keys that exist in the committed source, so a + * pending rename would 404 on the new key. Loading the content under the + * BASE key instead is also what makes it render: `applyJsonEntryContents` + * substitutes into the base source, and the `move` patch then relocates the + * content to the new key on its own. + */ + private resolveBaseJsonEntryKey( + moduleFilePath: ModuleFilePath, + key: string, + ): string { + const baseSource = this.serverSources?.[moduleFilePath]; + if ( + baseSource === undefined || + baseSource === null || + typeof baseSource !== "object" || + Array.isArray(baseSource) || + key in baseSource + ) { + return key; + } + // Walk the pending ops newest-first, undoing renames until we land on a key + // the base source actually has. + const patchIds = this.orderedPatchIdsForModule(moduleFilePath); + let current = key; + for (let i = patchIds.length - 1; i >= 0; i--) { + const patchData = this.patchDataByPatchId[patchIds[i]]; + if (!patchData) { + continue; + } + const ops = patchData.patch; + for (let j = ops.length - 1; j >= 0; j--) { + const op = ops[j]; + if ( + (op.op === "move" || op.op === "copy") && + op.path.length === 1 && + op.path[0] === current && + op.from.length === 1 + ) { + current = op.from[0]; + if (current in baseSource) { + return current; + } + break; + } + } + } + return current; + } + + /** + * Resolves once a `.jsonValues()` entry's content is loaded — or immediately + * if it already is, or if its load previously failed. Awaited before emitting + * a patch that moves a whole entry, so the patch carries real content rather + * than an opaque marker. {@link requestJsonEntry} is the fire-and-forget + * variant used by the field hooks. + */ + async ensureJsonEntry( + moduleFilePath: ModuleFilePath, + requestedKey: string, + ): Promise { + await this.loadJsonEntriesSettled([ + { moduleFilePath, keys: [requestedKey] }, + ]); + } + + /** + * The committed entry keys of a `.jsonValues()` module — i.e. the keys whose + * base-source value is still a lazy `{_type:"json"}` marker. `null` when the + * module's source is not (yet) a record. + * + * Keys that exist ONLY in a pending patch are deliberately excluded: they have + * no committed content to fetch (their value is the patch's own), so asking + * `/json` for them would 404 and poison them with an error state. + */ + private committedJsonEntryKeys( + moduleFilePath: ModuleFilePath, + ): string[] | null { + const baseSource = this.serverSources?.[moduleFilePath]; + if ( + baseSource === undefined || + baseSource === null || + typeof baseSource !== "object" || + Array.isArray(baseSource) + ) { + return null; + } + return Object.keys(baseSource).filter((key) => + Internal.isJson(baseSource[key]), + ); + } + + /** + * Loads the content of an explicit set of `.jsonValues()` entries, batched. + * Fire-and-forget: this is what a virtualized list calls for the window it just + * rendered. Already-loaded, in-flight and previously-failed keys are skipped, + * so re-rendering the same window costs nothing. + */ + requestJsonEntries(moduleFilePath: ModuleFilePath, keys: string[]): void { + this.coalesceJsonEntryRequests(moduleFilePath, keys); + } + + /** + * Resolves once EVERY committed entry of the given modules is loaded (or has + * failed). This is the completeness-critical variant: the reference guards and + * search must not answer from a partially-loaded record, so they await this and + * read `complete`. + * + * Never called on Studio boot — only when something is about to need the + * content (a destructive action's reference check, a search query, or an + * explicit "validate everything"). + */ + async ensureJsonEntries(moduleFilePaths: ModuleFilePath[]): Promise<{ + /** True when every requested key resolved to content. */ + complete: boolean; + errors: { moduleFilePath: ModuleFilePath; key: string; message: string }[]; + }> { + return this.loadJsonEntriesSettled( + moduleFilePaths.map((moduleFilePath) => ({ + moduleFilePath, + keys: this.committedJsonEntryKeys(moduleFilePath) ?? [], + })), + ); + } + + /** + * Loads `requests` and re-passes while anything it asked for is still + * outstanding, then reports whether everything resolved to content. + * + * The re-pass exists because an invalidation that lands mid-flight marks + * entries stale again: the response we were waiting for predates it, so + * reporting `complete` on the strength of it would be exactly the lie this + * method exists to prevent. Bounded, so a pathological invalidation loop + * cannot spin here forever. + */ + private async loadJsonEntriesSettled( + requests: { moduleFilePath: ModuleFilePath; keys: string[] }[], + ): Promise<{ + complete: boolean; + errors: { moduleFilePath: ModuleFilePath; key: string; message: string }[]; + }> { + let errors: { + moduleFilePath: ModuleFilePath; + key: string; + message: string; + }[] = []; + for (let pass = 0; pass < 3; pass++) { + const res = await this.loadJsonEntries(requests); + errors = res.errors; + const outstanding = res.requestedBaseKeys.some( + ({ moduleFilePath, keys }) => + keys.some((key) => { + if (this.staleJsonEntries.has(`${moduleFilePath}\0${key}`)) { + return true; + } + return ( + this.jsonEntryContents[moduleFilePath]?.[key] === undefined && + this.jsonEntryErrors[moduleFilePath]?.[key] === undefined + ); + }), + ); + if (!outstanding) { + break; + } + } + return { complete: errors.length === 0, errors }; + } + + /** + * ONE load pass: filters each module's requested keys down to the ones worth + * fetching, chunks them, and resolves when every chunk has settled. Reports the + * keys it actually took responsibility for, so the caller can tell "resolved" + * from "deliberately skipped". + */ + private async loadJsonEntries( + requests: { moduleFilePath: ModuleFilePath; keys: string[] }[], + ): Promise<{ + errors: { moduleFilePath: ModuleFilePath; key: string; message: string }[]; + requestedBaseKeys: { moduleFilePath: ModuleFilePath; keys: string[] }[]; + }> { + const waits: Promise[] = []; + const requestedBaseKeys: { + moduleFilePath: ModuleFilePath; + keys: string[]; + }[] = []; + for (const { moduleFilePath, keys } of requests) { + const committed = this.committedJsonEntryKeys(moduleFilePath); + if (committed === null) { + continue; + } + const committedKeys = new Set(committed); + const patchedSource = this.getPatchedSource(moduleFilePath); + const draftedKeys = + patchedSource !== undefined && + patchedSource !== null && + typeof patchedSource === "object" && + !Array.isArray(patchedSource) + ? new Set(Object.keys(patchedSource)) + : new Set(); + const baseKeys: string[] = []; + const wanted: string[] = []; + const seen = new Set(); + for (const requestedKey of keys) { + const key = this.resolveBaseJsonEntryKey(moduleFilePath, requestedKey); + if (seen.has(key)) { + continue; + } + // A key the committed source does not have, but the PATCHED source does, + // exists only in a pending patch: its value comes from that patch, so + // there is nothing to fetch and asking would 404 and wrongly mark it + // errored. A key in neither is a genuine miss and IS requested, so the + // caller gets a real error instead of silence. + if (!committedKeys.has(key) && draftedKeys.has(key)) { + continue; + } + seen.add(key); + baseKeys.push(key); + const loadingKey = `${moduleFilePath}\0${key}`; + const inFlight = this.loadingJsonEntries.get(loadingKey); + if (inFlight !== undefined) { + waits.push(inFlight); + continue; + } + const isStale = this.staleJsonEntries.has(loadingKey); + if (isStale) { + wanted.push(key); + continue; + } + if (this.jsonEntryContents[moduleFilePath]?.[key] !== undefined) { + continue; + } + // A memoized failure stops the refetch loop (see `jsonEntryErrors`); + // `retryJsonEntry` clears it. + if (this.jsonEntryErrors[moduleFilePath]?.[key] !== undefined) { + continue; + } + wanted.push(key); + } + requestedBaseKeys.push({ moduleFilePath, keys: baseKeys }); + for (let i = 0; i < wanted.length; i += JSON_ENTRIES_CHUNK_SIZE) { + waits.push( + this.loadJsonEntryChunk( + moduleFilePath, + wanted.slice(i, i + JSON_ENTRIES_CHUNK_SIZE), + ), + ); + } + } + await Promise.all(waits); + const errors: { + moduleFilePath: ModuleFilePath; + key: string; + message: string; + }[] = []; + for (const { moduleFilePath, keys } of requestedBaseKeys) { + for (const key of keys) { + const message = this.jsonEntryErrors[moduleFilePath]?.[key]; + if (message !== undefined) { + errors.push({ moduleFilePath, key, message }); + } + } + } + return { errors, requestedBaseKeys }; + } + + /** Loads ONE batch of entries: a single `/json` request for many keys. */ + private loadJsonEntryChunk( + moduleFilePath: ModuleFilePath, + keys: string[], + ): Promise { + const setJsonEntryError = (key: string, message: string) => { + if (this.jsonEntryErrors[moduleFilePath] === undefined) { + this.jsonEntryErrors[moduleFilePath] = {}; + } + this.jsonEntryErrors[moduleFilePath][key] = message; + }; + // Cleared at request START, not on success: anything that marks an entry + // stale while this request is in flight must win, since the response we are + // about to get was produced before that invalidation. + for (const key of keys) { + this.staleJsonEntries.delete(`${moduleFilePath}\0${key}`); + } + this.noteJsonEntriesRequested(keys.length); + const promise = this.client("/json", "GET", { + // apply_patches=false: we own in-flight client patches the server has not + // seen yet and apply them ourselves in `getPatchedSource`. Letting the + // server apply them too would double-apply. + query: { + path: moduleFilePath, + key: undefined, + keys, + offset: undefined, + limit: undefined, + apply_patches: false, + }, + }) + .then((res) => { + if (res.status === 200 && "entries" in res.json) { + if (this.jsonEntryContents[moduleFilePath] === undefined) { + this.jsonEntryContents[moduleFilePath] = {}; + } + for (const entry of res.json.entries) { + this.jsonEntryContents[moduleFilePath][entry.key] = + entry.content ?? null; + if (this.jsonEntryErrors[moduleFilePath] !== undefined) { + delete this.jsonEntryErrors[moduleFilePath][entry.key]; + } + } + // A committed key that the server cannot resolve (deleted on disk + // between our source sync and this request) is an error, not silence. + for (const key of res.json.missing) { + setJsonEntryError( + key, + `Entry not found: ${key} in ${moduleFilePath}`, + ); + } + for (const { key, message } of res.json.errors) { + setJsonEntryError(key, message); + } + } else { + const message = + "message" in res.json + ? res.json.message + : `Request failed with status ${res.status}`; + console.error("Val: SyncEngine: failed to load json entries", { + moduleFilePath, + keys, + res, + }); + for (const key of keys) { + setJsonEntryError(key, message); + } + } + }) + .catch((err) => { + console.error("Val: SyncEngine: error loading json entries", { + moduleFilePath, + keys, + err, + }); + const message = err instanceof Error ? err.message : String(err); + for (const key of keys) { + setJsonEntryError(key, message); + } + }) + .finally(() => { + for (const key of keys) { + this.loadingJsonEntries.delete(`${moduleFilePath}\0${key}`); + } + // ONE invalidation for the whole batch, not one per entry. + this.invalidateSource(moduleFilePath); + // After the in-flight deletes above, so "nothing left in flight" is + // accurate and the run can reset. + this.noteJsonEntriesResolved(keys.length); + }); + for (const key of keys) { + this.loadingJsonEntries.set(`${moduleFilePath}\0${key}`, promise); + } + return promise; + } + + private noteJsonEntriesRequested(count: number): void { + if (count <= 0) { + return; + } + this.jsonEntriesProgress = { + requested: this.jsonEntriesProgress.requested + count, + resolved: this.jsonEntriesProgress.resolved, + }; + this.invalidateJsonEntriesProgress(); + } + + private noteJsonEntriesResolved(count: number): void { + if (count <= 0) { + return; + } + const { requested, resolved } = this.jsonEntriesProgress; + this.jsonEntriesProgress = + this.loadingJsonEntries.size === 0 + ? // The run is over — reset so the next one starts from 0%. + { requested: 0, resolved: 0 } + : { requested, resolved: Math.min(resolved + count, requested) }; + this.invalidateJsonEntriesProgress(); + } + + private cachedJsonEntriesProgressSnapshot: JsonEntriesProgress | null = null; + /** + * Progress of the current json-entry load run, for UI that must show it rather + * than pretend it has a complete answer. Spans every module in flight, so the + * percentage does not reset at module boundaries. + */ + getJsonEntriesProgressSnapshot(): JsonEntriesProgress { + if (this.cachedJsonEntriesProgressSnapshot === null) { + const { requested, resolved } = this.jsonEntriesProgress; + this.cachedJsonEntriesProgressSnapshot = { + status: requested === 0 ? "idle" : "loading", + loaded: resolved, + total: requested, + // 100 while idle: check `status` first if "nothing requested yet" and + // "everything done" must look different. + percentage: + requested === 0 ? 100 : Math.round((resolved / requested) * 100), + }; + } + return this.cachedJsonEntriesProgressSnapshot; + } + + /** + * The error message of the last failed load of a `.jsonValues()` entry, or + * `null`. Returns a primitive so it is safe to read from a + * `useSyncExternalStore` snapshot getter. + */ + getJsonEntryError( + moduleFilePath: ModuleFilePath, + requestedKey: string, + ): string | null { + const key = this.resolveBaseJsonEntryKey(moduleFilePath, requestedKey); + return this.jsonEntryErrors[moduleFilePath]?.[key] ?? null; + } + + /** Clears a memoized json entry failure and loads it again. */ + retryJsonEntry(moduleFilePath: ModuleFilePath, requestedKey: string): void { + const key = this.resolveBaseJsonEntryKey(moduleFilePath, requestedKey); + if (this.jsonEntryErrors[moduleFilePath] !== undefined) { + delete this.jsonEntryErrors[moduleFilePath][key]; + } + this.requestJsonEntry(moduleFilePath, key); + } + + /** + * Marks every loaded entry of a module stale, so the next request refetches + * its committed content. Called when the module's server source is replaced + * (e.g. after publish): without this the pre-edit content is re-substituted + * and a published edit looks like it reverted. + */ + private markJsonEntriesStale(moduleFilePath: ModuleFilePath): void { + const contents = this.jsonEntryContents[moduleFilePath]; + delete this.jsonEntryErrors[moduleFilePath]; + if (contents === undefined) { + return; + } + const keys = Object.keys(contents); + for (const key of keys) { + this.staleJsonEntries.add(`${moduleFilePath}\0${key}`); + } + // Batched: with hundreds of entries cached, refetching one-by-one made a + // publish a request storm. + this.requestJsonEntries(moduleFilePath, keys); + } + + /** + * Marks every loaded `.jsonValues()` entry of every module stale. + * + * Needed on publish: a content-only edit rewrites the entry's `*.val.json` + * but NOT the `.val.ts`, so the module source (bare `{_type:"json"}` markers) + * is byte-identical and `sourcesSha` does not change — no `/sources/~` + * refresh is triggered. Without this the just-published content is served + * from the pre-edit cache and the edit looks like it reverted. + */ + private markAllJsonEntriesStale(): void { + for (const moduleFilePathS of Object.keys(this.jsonEntryContents)) { + this.markJsonEntriesStale(moduleFilePathS as ModuleFilePath); + } + } + + /** + * Returns `baseSource` with any loaded `.jsonValues()` entry content + * substituted in place of its lazy marker, so downstream resolution/patching + * sees real content. Markers without loaded content are left untouched. + */ + private applyJsonEntryContents( + moduleFilePath: ModuleFilePath, + baseSource: JSONValue, + ): JSONValue { + const contents = this.jsonEntryContents[moduleFilePath]; + if ( + contents === undefined || + baseSource === null || + typeof baseSource !== "object" || + Array.isArray(baseSource) + ) { + return baseSource; + } + let result: Record | null = null; + for (const key in contents) { + if (Internal.isJson(baseSource[key])) { + if (result === null) { + result = { ...baseSource }; + } + result[key] = contents[key]; + } + } + return result ?? baseSource; + } + private getPatchedSource( moduleFilePath: ModuleFilePath, ): JSONValue | undefined { - const baseSource = this.serverSources?.[moduleFilePath]; - if (baseSource === undefined) return undefined; + const rawBaseSource = this.serverSources?.[moduleFilePath]; + if (rawBaseSource === undefined) return undefined; + const baseSource = this.applyJsonEntryContents( + moduleFilePath, + rawBaseSource, + ); const nextIds = this.orderedPatchIdsForModule(moduleFilePath); if (nextIds.length === 0) return baseSource; @@ -1436,6 +2062,7 @@ export class ValSyncEngine { (moduleFilePath, errors) => { this.applyValidationResult(moduleFilePath, errors); }, + this.createValidationWorker, ); } return this.validationWorker; @@ -2924,6 +3551,11 @@ export class ValSyncEngine { // the un-patched source. The patched view is computed by // getPatchedSource folding the known patch chain on top. this.serverSources[moduleFilePath] = valModule.source; + // The committed content of any loaded `.jsonValues()` entry may + // have changed with it (e.g. after publish) — refetch, or the + // stale pre-edit content is re-substituted and the edit looks + // like it reverted. + this.markJsonEntriesStale(moduleFilePath); // render is always null in the new mode; keep the renders map // up-to-date for any downstream code that still subscribes. if (this.renders === null) { @@ -2989,17 +3621,10 @@ export class ValSyncEngine { this.globalServerSidePatchIds && this.globalServerSidePatchIds.length > 0 ) { - let hasValidationError = false; - for (const sourcePathS in this.errors.validationErrors || {}) { - const sourcePath = sourcePathS as SourcePath; - if ( - this.errors?.validationErrors?.[sourcePath] && - this.errors?.validationErrors?.[sourcePath]!.length > 0 - ) { - hasValidationError = true; - break; - } - } + const surfacedValidationErrors = this.getAllValidationErrorsSnapshot(); + const hasValidationError = Object.values( + surfacedValidationErrors || {}, + ).some((errors) => errors && errors.length > 0); if (!hasValidationError) { await this.publish( this.globalServerSidePatchIds.concat( @@ -3012,7 +3637,7 @@ export class ValSyncEngine { } else { console.debug( "Skip auto-publish since there's validation errors", - this.errors.validationErrors, + surfacedValidationErrors, ); } } @@ -3055,14 +3680,15 @@ export class ValSyncEngine { this.publishDisabled = true; this.invalidatePublishDisabled(); + const surfacedValidationErrors = this.getAllValidationErrorsSnapshot(); const hasValidationError = - Object.values(this.errors.validationErrors || {}).flatMap( + Object.values(surfacedValidationErrors || {}).flatMap( (errors) => errors || [], ).length > 0; if (hasValidationError) { console.debug( "Skipping publish since there's validation errors", - this.errors.validationErrors, + surfacedValidationErrors, ); this.addGlobalTransientError( "Could not publish changes, since there are validation errors", @@ -3118,6 +3744,9 @@ export class ValSyncEngine { this.patchIdsByModuleFilePath = new Map(); this.patchDataByPatchId = {}; this.patchSets = new PatchSets(); + // The published content is now the committed content: any loaded + // `.jsonValues()` entry must be refetched (see markAllJsonEntriesStale). + this.markAllJsonEntriesStale(); const fullReset = true; await this.syncPatches(fullReset, now); this.invalidatePatchSets(); @@ -3273,6 +3902,23 @@ export class ValSyncEngine { // #region Supporting code const ops = new JSONOps(); const globalNamespace = "global"; +/** + * How many `.jsonValues()` entries one `/json` request asks for. Smaller than + * the server's hard cap so a chunk stays a modest URL and lands incrementally + * (each landed chunk re-renders the rows it covers). + */ +const JSON_ENTRIES_CHUNK_SIZE = Math.min(50, JSON_ENTRIES_BATCH_MAX); + +/** Progress of the current `.jsonValues()` entry load run. */ +export type JsonEntriesProgress = { + status: "idle" | "loading"; + /** Keys resolved so far — loaded, missing or failed all count as resolved. */ + loaded: number; + /** Keys requested in this run; 0 when idle. */ + total: number; + /** 0-100, and 100 when idle. */ + percentage: number; +}; /** * These are types where we can be 100% certain that a change in this type, will not result in validations failing in some other module. * We use this to determine if syncing 1 module is enough or if we need to sync all modules. @@ -3350,6 +3996,7 @@ type SyncEngineListenerType = | "schema-out-of-date" | "local-modules-status" | "pending-ops-count" + | "json-entries-progress" | "all-sources" | "all-renders" | "render" diff --git a/packages/ui/spa/components/ChangeRecordPopover.tsx b/packages/ui/spa/components/ChangeRecordPopover.tsx index 83975d791..ae2e879e8 100644 --- a/packages/ui/spa/components/ChangeRecordPopover.tsx +++ b/packages/ui/spa/components/ChangeRecordPopover.tsx @@ -1,7 +1,12 @@ import { Internal, ModuleFilePath, SourcePath } from "@valbuild/core"; import { useState, useEffect, useCallback, useMemo } from "react"; import { Button } from "./designSystem/button"; -import { useAddPatch, useShallowSourceAtPath } from "./ValFieldProvider"; +import { + useAddPatch, + useSchemaAtPath, + useShallowSourceAtPath, + useSyncEngine, +} from "./ValFieldProvider"; import { useValPortal } from "./ValPortalProvider"; import { useNavigation } from "./ValRouter"; import { @@ -70,8 +75,21 @@ export function ChangeRecordPopover({ } return []; }, [parentSource]); + const syncEngine = useSyncEngine(); + const parentSchema = useSchemaAtPath(parentPath); + // A `.jsonValues()` entry's content is lazily loaded. If we move an entry that + // is still an opaque marker, the marker (not the content) lands on the new key + // and opening it would fetch `/json?key=` — which 404s, since the base + // source still only has the old key. Load it first. + const isJsonValuesRecord = + parentSchema.status === "success" && + parentSchema.data.type === "record" && + parentSchema.data.jsonValues === true; const onSubmit = useCallback( - (key: string) => { + async (key: string) => { + if (isJsonValuesRecord) { + await syncEngine.ensureJsonEntry(moduleFilePath, defaultValue); + } const patchOps: Patch = [ { op: "move", @@ -116,6 +134,10 @@ export function ChangeRecordPopover({ parentPatchPath, navigate, onComplete, + syncEngine, + isJsonValuesRecord, + defaultValue, + existingKeys, ], ); diff --git a/packages/ui/spa/components/ValFieldProvider.tsx b/packages/ui/spa/components/ValFieldProvider.tsx index 0264aef7b..6939eabbb 100644 --- a/packages/ui/spa/components/ValFieldProvider.tsx +++ b/packages/ui/spa/components/ValFieldProvider.tsx @@ -533,16 +533,51 @@ function useSchemaAtPathInternal( () => syncEngine.getSourceSnapshot(moduleFilePath), () => syncEngine.getSourceSnapshot(moduleFilePath), ); - const resolvedSchemaAtPathRes = useMemo(() => { - if (schemaRes.status !== "success") { - return schemaRes; - } - const sourceData = + const sourceData = useMemo( + () => sourceOverride && sourceOverride.moduleFilePath === moduleFilePath ? sourceOverride.moduleSource : sourcesRes.status === "success" ? sourcesRes.data - : undefined; + : undefined, + [sourceOverride, moduleFilePath, sourcesRes], + ); + // Lazily load `.jsonValues()` entry content when the path descends into an + // un-loaded marker, and treat the schema as loading until it resolves. + const unloadedJsonKey = useMemo( + () => findUnloadedJsonEntryKey(modulePath, sourceData), + [modulePath, sourceData], + ); + useEffect(() => { + if (unloadedJsonKey !== null) { + syncEngine.requestJsonEntry(moduleFilePath, unloadedJsonKey); + } + }, [syncEngine, moduleFilePath, unloadedJsonKey]); + const jsonEntryError = useSyncExternalStore( + syncEngine.subscribe("source", moduleFilePath), + () => + unloadedJsonKey === null + ? null + : syncEngine.getJsonEntryError(moduleFilePath, unloadedJsonKey), + () => + unloadedJsonKey === null + ? null + : syncEngine.getJsonEntryError(moduleFilePath, unloadedJsonKey), + ); + const resolvedSchemaAtPathRes = useMemo(() => { + if (schemaRes.status !== "success") { + return schemaRes; + } + if (unloadedJsonKey !== null) { + // A failed load must not render as a perpetual spinner. + if (jsonEntryError !== null) { + return { + status: "error" as const, + error: `Could not load entry '${unloadedJsonKey}': ${jsonEntryError}`, + }; + } + return { status: "loading" as const }; + } if (sourceData === undefined) { if (sourcesRes.status !== "success") { return sourcesRes; @@ -601,7 +636,15 @@ function useSchemaAtPathInternal( }`, }; } - }, [schemaRes, sourcesRes, moduleFilePath, modulePath, sourceOverride]); + }, [ + schemaRes, + sourcesRes, + moduleFilePath, + modulePath, + sourceData, + unloadedJsonKey, + jsonEntryError, + ]); const initializedAt = useSyncEngineInitializedAt(syncEngine); if (initializedAt === null) { return { status: "loading" }; @@ -715,6 +758,40 @@ export function useAllRenders() { return renders; } +/** + * Walks `modulePath` against `sourceData` and returns the record key at which + * the path descends into a `.jsonValues()` entry whose content has NOT been + * loaded yet (the value is still a lazy json marker), or `null` otherwise. + * + * The sync engine substitutes loaded entry content in place of the marker, so a + * marker still present here means the entry isn't loaded — the caller should + * trigger `requestJsonEntry` and render a loading state until it resolves. + */ +function findUnloadedJsonEntryKey( + modulePath: ModulePath, + sourceData: Json | undefined, +): string | null { + if (sourceData === undefined) { + return null; + } + let current: Json = sourceData; + for (const part of Internal.splitModulePath(modulePath)) { + if ( + current === null || + typeof current !== "object" || + isJsonArray(current) + ) { + return null; + } + const next: Json = current[part]; + if (Internal.isJson(next)) { + return part; + } + current = next; + } + return null; +} + function walkSourcePath( modulePath: ModulePath, sources?: Json, @@ -1161,6 +1238,31 @@ export function useSourceAtPath( syncEngine ? () => syncEngine.getInitializedAtSnapshot() : getNull, syncEngine ? () => syncEngine.getInitializedAtSnapshot() : getNull, ); + // A `.jsonValues()` entry's content is loaded lazily: if this path descends + // into an un-loaded marker, request it and render loading until it resolves. + const unloadedJsonKey = useMemo( + () => + sourceSnapshot && sourceSnapshot.status === "success" + ? findUnloadedJsonEntryKey(modulePath, sourceSnapshot.data) + : null, + [modulePath, sourceSnapshot], + ); + useEffect(() => { + if (syncEngine && unloadedJsonKey !== null) { + syncEngine.requestJsonEntry(moduleFilePath, unloadedJsonKey); + } + }, [syncEngine, moduleFilePath, unloadedJsonKey]); + const jsonEntryError = useSyncExternalStore( + syncEngine ? syncEngine.subscribe("source", moduleFilePath) : noopSubscribe, + () => + syncEngine && unloadedJsonKey !== null + ? syncEngine.getJsonEntryError(moduleFilePath, unloadedJsonKey) + : null, + () => + syncEngine && unloadedJsonKey !== null + ? syncEngine.getJsonEntryError(moduleFilePath, unloadedJsonKey) + : null, + ); return useMemo(() => { if (!syncEngine) { return NOT_FOUND; @@ -1171,6 +1273,16 @@ export function useSourceAtPath( if (sourceOverride && sourceOverride.moduleFilePath === moduleFilePath) { return walkSourcePath(modulePath, sourceOverride.moduleSource); } + if (unloadedJsonKey !== null) { + // A failed load must not render as a perpetual spinner. + if (jsonEntryError !== null) { + return { + status: "error", + error: `Could not load entry '${unloadedJsonKey}': ${jsonEntryError}`, + }; + } + return { status: "loading" }; + } if (sourceSnapshot && sourceSnapshot.status === "success") { return walkSourcePath(modulePath, sourceSnapshot.data); } @@ -1185,6 +1297,8 @@ export function useSourceAtPath( modulePath, moduleFilePath, sourceOverride, + unloadedJsonKey, + jsonEntryError, ]); } diff --git a/packages/ui/spa/components/ValProvider.tsx b/packages/ui/spa/components/ValProvider.tsx index ab052ce5b..4f67ac1c2 100644 --- a/packages/ui/spa/components/ValProvider.tsx +++ b/packages/ui/spa/components/ValProvider.tsx @@ -34,6 +34,7 @@ import { isJsonArray } from "../utils/isJsonArray"; import { AuthenticationState, useStatus } from "../hooks/useStatus"; import { findRequiredRemoteFiles } from "../utils/findRequiredRemoteFiles"; import { defaultOverlayEmitter, ValSyncEngine } from "../ValSyncEngine"; +import { createValidationWorker } from "../validation/createValidationWorker"; import { SerializedPatchSet } from "../utils/PatchSets"; import { z } from "zod"; import { @@ -314,11 +315,15 @@ export function ValProvider({ const syncEngine = useMemo( () => - new ValSyncEngine(client, (moduleFilePath, newSource) => { - if (dispatchValEvents) { - defaultOverlayEmitter(moduleFilePath, newSource); - } - }), + new ValSyncEngine( + client, + (moduleFilePath, newSource) => { + if (dispatchValEvents) { + defaultOverlayEmitter(moduleFilePath, newSource); + } + }, + createValidationWorker, + ), // TODO: add client to dependency array NOTE: we need to make sure syncing works if when syncEngine is instantiated anew [dispatchValEvents], ); diff --git a/packages/ui/spa/components/fields/RecordFields.tsx b/packages/ui/spa/components/fields/RecordFields.tsx index 00392d9b1..e7d27ecf5 100644 --- a/packages/ui/spa/components/fields/RecordFields.tsx +++ b/packages/ui/spa/components/fields/RecordFields.tsx @@ -1,12 +1,20 @@ import { + Internal, ListRecordRender as ListRecordRender, + ModuleFilePath, SourcePath, } from "@valbuild/core"; +import { useMemo } from "react"; import { useRenderOverrideAtPath, useSchemaAtPath, useShallowSourceAtPath, + useSourceAtPath, } from "../ValFieldProvider"; +import { + RecordRowSkeleton, + VirtualizedRecordList, +} from "./VirtualizedRecordList"; import { ModuleGallery } from "./ModuleGallery"; import { useAllValidationErrors } from "../ValErrorProvider"; import { sourcePathOfItem } from "../../utils/sourcePathOfItem"; @@ -19,6 +27,7 @@ import { useNavigation } from "../../components/ValRouter"; import { PreviewLoading, PreviewNull } from "../../components/Preview"; import { PreviewWithRender } from "../../components/PreviewWithRender"; import { ValidationErrors } from "../../components/ValidationError"; +import type { ValidationError } from "@valbuild/core"; import { isParentError } from "../../utils/isParentError"; import { ErrorIndicator } from "../ErrorIndicator"; import classNames from "classnames"; @@ -41,7 +50,6 @@ export function RecordFields({ }) { const type = "record"; const validationErrors = useAllValidationErrors() || {}; - const { navigate } = useNavigation(); const schemaAtPath = useSchemaAtPath(path); const renderAtPath = useRenderOverrideAtPath(path); const sourceAtPath = useShallowSourceAtPath(path, type); @@ -137,61 +145,159 @@ export function RecordFields({ )} {renderListAtPathData && ( - + )} - {!renderListAtPathData && ( -
- {source && - Object.entries(source).map(([key]) => ( -
navigate(sourcePathOfItem(path, key))} - className={classNames( - "bg-primary-foreground cursor-pointer min-w-[320px] max-h-[170px] overflow-hidden rounded-md border border-border-primary p-4", - "hover:bg-bg-secondary-hover", - )} - > -
-
{key}
- {isParentError( - sourcePathOfItem(path, key), - validationErrors, - ) && } -
-
- -
-
- ))} -
+ {!renderListAtPathData && source && ( + )} ); } +/** Row height estimate for the default card layout (`max-h-[170px]` + gap). */ +const CARD_ROW_HEIGHT = 186; +/** Row height estimate for a `.render({layout:"list"})` row. */ +const RENDER_ROW_HEIGHT = 104; + +function RecordCardList({ + path, + keys, + jsonValues, + validationErrors, +}: { + path: SourcePath; + keys: string[]; + jsonValues: boolean; + validationErrors: Record; +}) { + const { navigate } = useNavigation(); + const [moduleFilePath] = Internal.splitModuleFilePathAndModulePath(path); + const unloadedKeys = useUnloadedJsonEntryKeys(moduleFilePath, jsonValues); + return ( + ( +
+
navigate(sourcePathOfItem(path, key))} + className={classNames( + "bg-primary-foreground cursor-pointer min-w-[320px] max-h-[170px] overflow-hidden rounded-md border border-border-primary p-4", + "hover:bg-bg-secondary-hover", + )} + > +
+
{key}
+ {isParentError(sourcePathOfItem(path, key), validationErrors) && ( + + )} +
+
+ {unloadedKeys.has(key) ? ( + // An un-loaded `.jsonValues()` entry: a preview here would read + // the opaque marker, which is what made these lists a wall of + // spinners. + + ) : ( + + )} +
+
+
+ )} + /> + ); +} + +/** + * The record keys of a `.jsonValues()` module whose entry content has not been + * loaded yet — i.e. whose value in the patched source is still a lazy marker. + * + * Computed once for the whole list rather than per row: one source subscription + * instead of one per visible row. + */ +function useUnloadedJsonEntryKeys( + moduleFilePath: ModuleFilePath, + jsonValues: boolean, +): ReadonlySet { + const moduleSource = useSourceAtPath(moduleFilePath); + const data = "data" in moduleSource ? moduleSource.data : undefined; + return useMemo(() => { + const unloaded = new Set(); + if ( + !jsonValues || + data === undefined || + data === null || + typeof data !== "object" || + Array.isArray(data) + ) { + return unloaded; + } + for (const [key, value] of Object.entries(data)) { + if (Internal.isJson(value)) { + unloaded.add(key); + } + } + return unloaded; + }, [jsonValues, data]); +} + function ListRecordRenderComponent({ path, items, + jsonValues, }: { path: SourcePath; items: ListRecordRender["items"]; + jsonValues: boolean; }) { const { navigate } = useNavigation(); + const [moduleFilePath] = Internal.splitModuleFilePathAndModulePath(path); + const unloadedKeys = useUnloadedJsonEntryKeys(moduleFilePath, jsonValues); + const keys = useMemo(() => items.map(([key]) => key), [items]); return ( -
- {items.map(([key]) => ( - - ))} -
+ ( +
+ +
+ )} + /> ); } diff --git a/packages/ui/spa/components/fields/VirtualizedRecordList.tsx b/packages/ui/spa/components/fields/VirtualizedRecordList.tsx new file mode 100644 index 000000000..0bb9e3c30 --- /dev/null +++ b/packages/ui/spa/components/fields/VirtualizedRecordList.tsx @@ -0,0 +1,166 @@ +import { ModuleFilePath, SourcePath } from "@valbuild/core"; +import { useVirtualizer } from "@tanstack/react-virtual"; +import { Fragment, ReactNode, useEffect, useMemo, useRef } from "react"; +import { useSyncEngine } from "../ValFieldProvider"; + +/** + * Records below this many keys render plainly, exactly as before: a nested + * scroll container is a real UX change, and it is not worth imposing on the + * ordinary small record. Above it the list virtualizes. + * + * It also bounds the un-virtualized `.jsonValues()` load: at most this many keys + * are requested at once, which is one or two batches. + */ +const VIRTUALIZE_THRESHOLD = 50; +/** Height of the virtualized scroll viewport. */ +const VIEWPORT_MAX_HEIGHT = 800; + +/** + * Renders a record's rows, virtualizing once there are enough of them, and — for + * a `.jsonValues()` record — loading the content of ONLY the rows currently + * rendered. + * + * Why this exists: every row renders a preview of its entry, and for a + * jsonValues record that preview needs the entry's content. Rendering all rows + * therefore loaded every entry, which is exactly what `.jsonValues()` is meant + * to avoid. Virtualizing means un-rendered rows cost nothing, and the visible + * window is requested in one batch. + */ +export function VirtualizedRecordList({ + moduleFilePath, + keys, + estimatedRowHeight, + jsonValues, + className, + renderRow, +}: { + moduleFilePath: ModuleFilePath; + /** Record keys, in source order. */ + keys: string[]; + estimatedRowHeight: number; + /** True when this is a `.jsonValues()` record, whose rows must be loaded. */ + jsonValues: boolean; + className?: string; + renderRow: (key: string) => ReactNode; +}) { + const syncEngine = useSyncEngine(); + const parentRef = useRef(null); + const shouldVirtualize = keys.length > VIRTUALIZE_THRESHOLD; + + const virtualizer = useVirtualizer({ + count: shouldVirtualize ? keys.length : 0, + getScrollElement: () => parentRef.current, + estimateSize: () => estimatedRowHeight, + overscan: 8, + }); + const virtualItems = virtualizer.getVirtualItems(); + // The rendered window, as plain indices: depending on these rather than on the + // virtual-item objects keeps the effect below from re-running on every scroll + // frame that happens to render the same rows. + const firstIndex = virtualItems.length > 0 ? virtualItems[0].index : -1; + const lastIndex = + virtualItems.length > 0 ? virtualItems[virtualItems.length - 1].index : -1; + + /** The keys actually rendered — the window whose content we need. */ + const windowKeys = useMemo(() => { + if (!shouldVirtualize) { + return keys; + } + if (firstIndex < 0) { + return []; + } + return keys.slice(firstIndex, lastIndex + 1); + }, [shouldVirtualize, keys, firstIndex, lastIndex]); + + // `keys` is a fresh array on every render (Object.keys), so the effect below + // depends on the window's CONTENT, not its identity — otherwise it re-runs on + // every render of a long list. The keys themselves are read through a ref so + // they never have to be a dependency (joining and re-splitting would corrupt + // any key containing the separator). + const windowKeysId = windowKeys.join("\u0000"); + const windowKeysRef = useRef(windowKeys); + windowKeysRef.current = windowKeys; + useEffect(() => { + if (!jsonValues || windowKeysRef.current.length === 0) { + return; + } + // Coalesced by the engine into one request per module per tick, so a fast + // scroll that crosses several windows does not fan out. + syncEngine.requestJsonEntries(moduleFilePath, windowKeysRef.current); + }, [syncEngine, moduleFilePath, jsonValues, windowKeysId]); + + if (!shouldVirtualize) { + return ( +
+ {keys.map((key) => ( + {renderRow(key)} + ))} +
+ ); + } + + return ( +
+
+ {virtualItems.map((virtualRow) => { + const key = keys[virtualRow.index]; + if (key === undefined) { + return null; + } + return ( +
+ {renderRow(key)} +
+ ); + })} +
+
+ ); +} + +/** + * Placeholder for a `.jsonValues()` row whose content has not loaded yet. + * + * Deliberately NOT the row's real preview: a preview reading an un-loaded marker + * is what made jsonValues record lists render a wall of spinners. The height is + * fixed to the row estimate so the virtualizer's measurements do not jump as + * content lands. + */ +export function RecordRowSkeleton({ + path, + height, +}: { + path: SourcePath; + height: number; +}) { + return ( +
+
+
+
+ ); +} diff --git a/packages/ui/spa/components/getRouteReferences.ts b/packages/ui/spa/components/getRouteReferences.ts index 16a3db334..34f671e74 100644 --- a/packages/ui/spa/components/getRouteReferences.ts +++ b/packages/ui/spa/components/getRouteReferences.ts @@ -1,4 +1,5 @@ import { + Internal, ModuleFilePath, ModuleFilePathSep, Source, @@ -27,6 +28,10 @@ export function getRouteReferences( if (schema === undefined) { return; } + if (Internal.isJson(source)) { + // Un-loaded `.jsonValues()` entry marker — opaque until loaded. + return; + } if (schema.type === "route") { // Check if the source value matches the route key we're looking for if (typeof source === "string" && source === routeKey) { diff --git a/packages/ui/spa/components/jsonValuesLoadRequirements.test.ts b/packages/ui/spa/components/jsonValuesLoadRequirements.test.ts new file mode 100644 index 000000000..b5c5b9853 --- /dev/null +++ b/packages/ui/spa/components/jsonValuesLoadRequirements.test.ts @@ -0,0 +1,297 @@ +import { + Internal, + ModuleFilePath, + SerializedSchema, + Source, + SourcePath, + ValModule, + initVal, +} from "@valbuild/core"; +import { jsonValuesLoadRequirements } from "./jsonValuesLoadRequirements"; + +const { s, c } = initVal(); + +const AUTHORS = "/authors.val.ts" as ModuleFilePath; +const PAGES = "/pages.val.ts" as ModuleFilePath; +const GALLERY = "/gallery.val.ts" as ModuleFilePath; +// The same string, branded as a SourcePath: that is what a `keyOf` schema stores +// when it points at a module-level record. +const PAGES_AS_SOURCE_PATH = "/pages.val.ts" as SourcePath; + +describe("jsonValuesLoadRequirements", () => { + test("nothing to load when the referrers live in ORDINARY modules", () => { + // The incoming-ref case: renaming a key of `authors` needs the referrers, + // which are keyOf fields in a normal module whose source is fully loaded. + // `pages` is jsonValues but points at nothing, so none of its entries matter. + const authors = c.define( + AUTHORS, + s.record(s.object({ name: s.string() })), + { ["ada"]: { name: "Ada" } }, + ); + const schemas = getSchemas([ + authors, + c.define("/featured.val.ts", s.keyOf(authors), "ada"), + c.define( + PAGES, + s.record(s.object({ title: s.string() })).jsonValues(), + {}, + ), + ]); + + expect( + jsonValuesLoadRequirements(schemas, { kind: "keyOf", module: AUTHORS }), + ).toEqual([]); + }); + + test("a jsonValues record whose ITEM holds a keyOf must be loaded", () => { + const authors = c.define( + AUTHORS, + s.record(s.object({ name: s.string() })), + { ["ada"]: { name: "Ada" } }, + ); + const schemas = getSchemas([ + authors, + c.define( + PAGES, + s + .record(s.object({ title: s.string(), author: s.keyOf(authors) })) + .jsonValues(), + {}, + ), + ]); + + expect( + jsonValuesLoadRequirements(schemas, { kind: "keyOf", module: AUTHORS }), + ).toEqual([PAGES]); + }); + + test("a keyOf pointing at a DIFFERENT module does not require loading", () => { + const authors = c.define( + AUTHORS, + s.record(s.object({ name: s.string() })), + { ["ada"]: { name: "Ada" } }, + ); + const other = c.define( + "/tags.val.ts", + s.record(s.object({ label: s.string() })), + { ["x"]: { label: "X" } }, + ); + const schemas = getSchemas([ + authors, + other, + c.define( + PAGES, + s.record(s.object({ tag: s.keyOf(other) })).jsonValues(), + {}, + ), + ]); + + expect( + jsonValuesLoadRequirements(schemas, { kind: "keyOf", module: AUTHORS }), + ).toEqual([]); + }); + + test("the match is transitive through object, array, record and union", () => { + const authors = c.define( + AUTHORS, + s.record(s.object({ name: s.string() })), + { ["ada"]: { name: "Ada" } }, + ); + const query = { kind: "keyOf", module: AUTHORS } as const; + + const inArray = getSchemas([ + authors, + c.define( + PAGES, + s.record(s.object({ list: s.array(s.keyOf(authors)) })).jsonValues(), + {}, + ), + ]); + expect(jsonValuesLoadRequirements(inArray, query)).toEqual([PAGES]); + + const inNestedRecord = getSchemas([ + authors, + c.define( + PAGES, + s.record(s.object({ byId: s.record(s.keyOf(authors)) })).jsonValues(), + {}, + ), + ]); + expect(jsonValuesLoadRequirements(inNestedRecord, query)).toEqual([PAGES]); + + const inUnion = getSchemas([ + authors, + c.define( + PAGES, + s + .record( + s.union( + "type", + s.object({ type: s.literal("plain"), text: s.string() }), + s.object({ type: s.literal("ref"), author: s.keyOf(authors) }), + ), + ) + .jsonValues(), + {}, + ), + ]); + expect(jsonValuesLoadRequirements(inUnion, query)).toEqual([PAGES]); + + const deeplyNested = getSchemas([ + authors, + c.define( + PAGES, + s + .record( + s.object({ + blocks: s.array(s.object({ author: s.keyOf(authors) })), + }), + ) + .jsonValues(), + {}, + ), + ]); + expect(jsonValuesLoadRequirements(deeplyNested, query)).toEqual([PAGES]); + }); + + test("only `.jsonValues()` records are candidates", () => { + const authors = c.define( + AUTHORS, + s.record(s.object({ name: s.string() })), + { ["ada"]: { name: "Ada" } }, + ); + // Same shape, WITHOUT .jsonValues(): its content is already in the source, so + // there is nothing to load. + const schemas = getSchemas([ + authors, + c.define(PAGES, s.record(s.object({ author: s.keyOf(authors) })), { + ["p"]: { author: "ada" }, + }), + ]); + + expect( + jsonValuesLoadRequirements(schemas, { kind: "keyOf", module: AUTHORS }), + ).toEqual([]); + }); + + test("a self-referencing jsonValues record requires loading itself", () => { + const pages: ValModule = c.define( + PAGES, + s.record(s.object({ title: s.string() })).jsonValues(), + {}, + ); + // keyOf pointing back at the same module. + const schemas = getSchemas([pages]); + const schema = schemas[PAGES]; + if (schema.type !== "record") { + throw new Error("expected a record"); + } + // Hand-built, because `s.keyOf(pages)` inside `pages` is not expressible. + schemas[PAGES] = { + ...schema, + item: { + type: "object", + items: { + related: { + type: "keyOf", + path: PAGES_AS_SOURCE_PATH, + opt: false, + values: "string", + }, + }, + opt: false, + }, + }; + + expect( + jsonValuesLoadRequirements(schemas, { kind: "keyOf", module: PAGES }), + ).toEqual([PAGES]); + }); + + test("file refs match the referenced gallery module", () => { + const gallery = c.define(GALLERY, s.images(), {}); + const schemas = getSchemas([ + gallery, + c.define( + PAGES, + s.record(s.object({ hero: s.image(gallery) })).jsonValues(), + {}, + ), + ]); + + expect( + jsonValuesLoadRequirements(schemas, { kind: "file", module: GALLERY }), + ).toEqual([PAGES]); + // A file query must not be satisfied by a keyOf, or vice versa. + expect( + jsonValuesLoadRequirements(schemas, { kind: "keyOf", module: GALLERY }), + ).toEqual([]); + }); + + test("route is an over-approximation: ANY route field counts", () => { + // `s.route()` records no target module, so we cannot tell which router a + // field points into and must load every jsonValues record that has one. + const withRoute = getSchemas([ + c.define(PAGES, s.record(s.object({ link: s.route() })).jsonValues(), {}), + ]); + expect(jsonValuesLoadRequirements(withRoute, { kind: "route" })).toEqual([ + PAGES, + ]); + + const withoutRoute = getSchemas([ + c.define( + PAGES, + s.record(s.object({ title: s.string() })).jsonValues(), + {}, + ), + ]); + expect(jsonValuesLoadRequirements(withoutRoute, { kind: "route" })).toEqual( + [], + ); + }); + + test("reports every module that needs loading", () => { + const authors = c.define( + AUTHORS, + s.record(s.object({ name: s.string() })), + { ["ada"]: { name: "Ada" } }, + ); + const schemas = getSchemas([ + authors, + c.define( + PAGES, + s.record(s.object({ author: s.keyOf(authors) })).jsonValues(), + {}, + ), + c.define( + "/posts.val.ts", + s.record(s.object({ author: s.keyOf(authors) })).jsonValues(), + {}, + ), + ]); + + expect( + jsonValuesLoadRequirements(schemas, { + kind: "keyOf", + module: AUTHORS, + }).sort(), + ).toEqual([PAGES, "/posts.val.ts"]); + }); +}); + +function getSchemas( + valModules: ValModule[], +): Record { + const schemas: Record = {}; + for (const valModule of valModules) { + const moduleFilePath = Internal.getValPath( + valModule, + ) as unknown as ModuleFilePath; + const schema = Internal.getSchema(valModule)?.["executeSerialize"](); + if (!schema) { + throw new Error(`Schema not found for ${moduleFilePath}`); + } + schemas[moduleFilePath] = schema; + } + return schemas; +} diff --git a/packages/ui/spa/components/jsonValuesLoadRequirements.ts b/packages/ui/spa/components/jsonValuesLoadRequirements.ts new file mode 100644 index 000000000..b28107a44 --- /dev/null +++ b/packages/ui/spa/components/jsonValuesLoadRequirements.ts @@ -0,0 +1,133 @@ +import { + ModuleFilePath, + SerializedObjectSchema, + SerializedSchema, +} from "@valbuild/core"; + +/** + * What a reference scan is looking for. Note the asymmetry: `keyOf` and + * `file` name the module they point AT, so they can be matched exactly, while a + * `route` field records no target module at all (`SerializedRouteSchema` only + * carries include/exclude patterns) and `getRouteReferences` matches by comparing + * the field's string VALUE to the route key. + */ +export type JsonValuesLoadQuery = + | { kind: "keyOf"; module: ModuleFilePath } + | { kind: "file"; module: ModuleFilePath } + | { kind: "route" }; + +/** + * Which `.jsonValues()` modules must have their entry CONTENT loaded before a + * reference scan for `query` can be trusted — decided from the schemas alone, so + * it costs nothing and needs no sources. + * + * Direction is what makes this cheap. A scan for references TO a module finds + * referrers, which are `keyOf`/`route`/file fields living somewhere else. The + * scanned record's own key set is always available (an un-loaded entry is still a + * marker under its key), so the only content a scan can be blind to is content + * that itself POINTS OUTWARD — i.e. a jsonValues record whose item schema + * contains a matching referrer, as in: + * + * ```ts + * s.record(s.object({ test: s.keyOf(otherModule) })).jsonValues() + * ``` + * + * In the overwhelmingly common case the result is empty: nothing to load, and the + * scan is complete and correct immediately. + * + * `route` is the one over-approximation — since the schema does not say which + * router a `s.route()` field points into, ANY jsonValues record containing a + * route field has to be loaded. + */ +export function jsonValuesLoadRequirements( + schemas: Record, + query: JsonValuesLoadQuery, +): ModuleFilePath[] { + const required: ModuleFilePath[] = []; + for (const moduleFilePathS in schemas) { + const moduleFilePath = moduleFilePathS as ModuleFilePath; + const schema = schemas[moduleFilePath]; + // `.jsonValues()` is root-only (locked decision #7), so only a module whose + // ROOT is a jsonValues record can hold un-loaded content. + if ( + schema === undefined || + schema.type !== "record" || + schema.jsonValues !== true + ) { + continue; + } + if (containsReferrer(schema.item, query, new Set())) { + required.push(moduleFilePath); + } + } + return required; +} + +/** + * True when `schema` can hold a field that {@link JsonValuesLoadQuery} would + * match, looking through objects, arrays, records and unions. + * + * `seen` guards against a schema that (however unusually) refers to itself + * structurally, so this cannot recurse forever. + */ +function containsReferrer( + schema: SerializedSchema | undefined, + query: JsonValuesLoadQuery, + seen: Set, +): boolean { + if (schema === undefined || seen.has(schema)) { + return false; + } + seen.add(schema); + switch (schema.type) { + case "keyOf": + // `keyOf.path` is the referenced record's path; for a module-level record + // that is the module file path itself (the same comparison `getKeysOf` + // makes). + // Compared as plain strings: `keyOf.path` is branded `SourcePath` and the + // query carries a `ModuleFilePath`, but for a module-level record they are + // the same string (which is the comparison `getKeysOf` makes too). + return query.kind === "keyOf" && sameString(schema.path, query.module); + case "image": + case "file": + return query.kind === "file" && schema.referencedModule === query.module; + case "route": + return query.kind === "route"; + case "object": + return Object.values(schema.items).some((item) => + containsReferrer(item, query, seen), + ); + case "array": + case "record": + return containsReferrer(schema.item, query, seen); + case "union": + // Covers both the tagged form (object variants) and the literal form, + // whose items are literals and match nothing. + return ( + schema.items as (SerializedObjectSchema | SerializedSchema)[] + ).some((item) => containsReferrer(item, query, seen)); + case "string": + case "number": + case "boolean": + case "literal": + case "date": + case "dateTime": + case "richtext": + return false; + default: { + const exhaustiveCheck: never = schema; + console.error( + "Could not compute jsonValues load requirements. Unhandled schema type", + exhaustiveCheck, + ); + // Conservative: an unknown schema type might hold a referrer, and + // wrongly reporting "nothing to load" is what makes a guard lie. + return true; + } + } +} + +/** Compares two branded strings without asserting one into the other's brand. */ +function sameString(a: string, b: string): boolean { + return a === b; +} diff --git a/packages/ui/spa/components/traverseSchemas.ts b/packages/ui/spa/components/traverseSchemas.ts index 2d1a5439d..8508f1a09 100644 --- a/packages/ui/spa/components/traverseSchemas.ts +++ b/packages/ui/spa/components/traverseSchemas.ts @@ -1,4 +1,5 @@ import { + Internal, ModuleFilePath, ModuleFilePathSep, SerializedArraySchema, @@ -34,6 +35,12 @@ export function traverseSchemas( schema: SerializedSchema | undefined, source: Source, ) => { + if (Internal.isJson(source)) { + // An un-loaded `.jsonValues()` entry is an opaque lazy marker + // ({ _type:"json" }). Skip it: once its content is loaded the + // substituted value is the real content and is traversed normally. + return; + } if (schema === undefined) { console.error(`Schema not found for ${sourcePath}`); return; diff --git a/packages/ui/spa/search/createSearchIndex.ts b/packages/ui/spa/search/createSearchIndex.ts index 6797b37d3..76e9a2b13 100644 --- a/packages/ui/spa/search/createSearchIndex.ts +++ b/packages/ui/spa/search/createSearchIndex.ts @@ -22,6 +22,11 @@ function rec( ) { addTokenizedSourcePath(sourcePathIndex, path); } + if (Internal.isJson(source)) { + // Un-loaded `.jsonValues()` entry marker — its content isn't available to + // index yet (the entry path was indexed above). Indexed once loaded. + return; + } if (!schema?.type) { throw new Error("Schema not found for " + path); } else if (source === null) { diff --git a/packages/ui/spa/validation/ValidationWorkerClient.ts b/packages/ui/spa/validation/ValidationWorkerClient.ts index 129eb5906..9c78dbfc5 100644 --- a/packages/ui/spa/validation/ValidationWorkerClient.ts +++ b/packages/ui/spa/validation/ValidationWorkerClient.ts @@ -1,16 +1,14 @@ -import { - deserializeSchema, +import type { ModuleFilePath, - SelectorSource, SerializedSchema, Source, - SourcePath, ValidationErrors, } from "@valbuild/core"; import type { ValidationWorkerRequest, ValidationWorkerResponse, } from "./worker-types"; +import { SchemaValidator } from "./validateModule"; const supportsWorker = typeof window !== "undefined" && typeof Worker !== "undefined"; @@ -20,40 +18,40 @@ export type ValidationResultCallback = ( errors: ValidationErrors, ) => void; +// The factory is injected by the composition root (ValProvider) so the +// `new URL(..., import.meta.url)` worker reference — and thus import.meta — stays +// out of this file and ValSyncEngine, which are compiled by Jest. See +// createValidationWorker.ts. +export type ValidationWorkerFactory = () => Worker; + export class ValidationWorkerClient { private worker: Worker | null = null; - // Worker setup is async (dynamic import) so we can keep the import.meta.url - // reference out of files parsed by Jest. Requests posted before the worker - // resolves are buffered here. - private pending: ValidationWorkerRequest[] | null = supportsWorker - ? [] - : null; + // Requests posted before the worker is installed are buffered here. Null when + // no worker will be installed (no factory, or unsupported env) — validation + // then runs on the main thread via the fallback validator. + private pending: ValidationWorkerRequest[] | null = null; private requestIdCounter = 0; private disposed = false; private latestRequestId = new Map(); - // Fallback cache used when the worker is unavailable (jsdom / SSR). - private fallbackSchemaCache = new Map< - ModuleFilePath, - { - schemaSha: string; - schema: ReturnType; - } - >(); + // Fallback validator used when the worker is unavailable (jsdom / SSR / node). + private fallbackValidator = new SchemaValidator(); - constructor(private onResult: ValidationResultCallback) { - if (supportsWorker) { - void this.setupWorker(); + constructor( + private onResult: ValidationResultCallback, + private createWorker?: ValidationWorkerFactory, + ) { + if (supportsWorker && this.createWorker) { + this.pending = []; + this.setupWorker(this.createWorker); } } - private async setupWorker(): Promise { + private setupWorker(createWorker: ValidationWorkerFactory): void { try { - const { createValidationWorker } = - await import("./createValidationWorker"); - const worker = createValidationWorker(); - // dispose() may have been called while the dynamic import was in flight — - // don't install a worker the client no longer owns (it would leak the - // thread and keep handlers alive). + const worker = createWorker(); + // dispose() may have been called before setup completed — don't install a + // worker the client no longer owns (it would leak the thread and keep + // handlers alive). if (this.disposed) { worker.terminate(); return; @@ -130,14 +128,11 @@ export class ValidationWorkerClient { } // Main-thread fallback (tests, SSR, or worker construction failed). try { - let cached = this.fallbackSchemaCache.get(moduleFilePath); - if (!cached || cached.schemaSha !== schemaSha) { - cached = { schemaSha, schema: deserializeSchema(serializedSchema) }; - this.fallbackSchemaCache.set(moduleFilePath, cached); - } - const errors = cached.schema["executeValidate"]( - moduleFilePath as string as SourcePath, - source as SelectorSource, + const errors = this.fallbackValidator.validate( + moduleFilePath, + source, + serializedSchema, + schemaSha, ); this.onResult(moduleFilePath, errors); } catch (error) { @@ -158,6 +153,6 @@ export class ValidationWorkerClient { } this.pending = null; this.latestRequestId.clear(); - this.fallbackSchemaCache.clear(); + this.fallbackValidator = new SchemaValidator(); } } diff --git a/packages/ui/spa/validation/createValidationWorker.ts b/packages/ui/spa/validation/createValidationWorker.ts index c491636ef..a5bed576c 100644 --- a/packages/ui/spa/validation/createValidationWorker.ts +++ b/packages/ui/spa/validation/createValidationWorker.ts @@ -1,5 +1,13 @@ -export function createValidationWorker(): Worker { - return new Worker(new URL("./validation.worker.ts", import.meta.url), { +import type { ValidationWorkerFactory } from "./ValidationWorkerClient"; + +// Constructed the same way as the search and patchsets workers: Vite rewrites +// new URL(..., import.meta.url) to an absolute, base-prefixed asset URL, so the +// worker resolves regardless of where the SPA entry is served. +// +// This is the only file that references import.meta. It is imported solely by +// the app composition root (ValProvider) and passed into ValSyncEngine, keeping +// import.meta out of the test-compiled core (ValSyncEngine / ValidationWorkerClient). +export const createValidationWorker: ValidationWorkerFactory = () => + new Worker(new URL("./validation.worker.ts", import.meta.url), { type: "module", }); -} diff --git a/packages/ui/spa/validation/validateModule.test.ts b/packages/ui/spa/validation/validateModule.test.ts new file mode 100644 index 000000000..744edb2f1 --- /dev/null +++ b/packages/ui/spa/validation/validateModule.test.ts @@ -0,0 +1,79 @@ +import { initVal, Internal, ModuleFilePath } from "@valbuild/core"; +import { SchemaValidator } from "./validateModule"; + +const { s, c } = initVal(); + +function serialize(module: ReturnType) { + const path = Internal.getValPath(module) as unknown as ModuleFilePath; + const schema = Internal.getSchema(module)!; + const serializedSchema = schema["executeSerialize"](); + const source = Internal.getSource(module); + return { path, serializedSchema, source }; +} + +describe("SchemaValidator", () => { + test("returns no errors for a valid source", () => { + const validator = new SchemaValidator(); + const { path, serializedSchema, source } = serialize( + c.define("/test.val.ts", s.string().minLength(2), "valid"), + ); + const errors = validator.validate(path, source, serializedSchema, "sha1"); + expect(errors).toBe(false); + }); + + test("returns validation errors for an invalid source", () => { + const validator = new SchemaValidator(); + const { path, serializedSchema, source } = serialize( + c.define("/test.val.ts", s.string().minLength(2), "a"), + ); + const errors = validator.validate(path, source, serializedSchema, "sha1"); + expect(errors).not.toBe(false); + expect( + Object.keys(errors as Record).length, + ).toBeGreaterThan(0); + }); + + test("reuses the cached schema while the sha is unchanged", () => { + const validator = new SchemaValidator(); + // Lenient schema: "valid" (5 chars) passes minLength(2). + const lenient = serialize( + c.define("/test.val.ts", s.string().minLength(2), "valid"), + ); + // Strict schema for the same path/source: "valid" fails minLength(10). + const strict = serialize( + c.define("/test.val.ts", s.string().minLength(10), "valid"), + ); + + // First call deserializes and caches the lenient schema under "sha1". + expect( + validator.validate( + lenient.path, + lenient.source, + lenient.serializedSchema, + "sha1", + ), + ).toBe(false); + + // Same sha → cached lenient schema is reused even though a different + // serialized schema is passed, so the source is still considered valid. + expect( + validator.validate( + lenient.path, + lenient.source, + strict.serializedSchema, + "sha1", + ), + ).toBe(false); + + // New sha → schema is re-derived from the strict definition and the source + // now fails validation. + expect( + validator.validate( + lenient.path, + lenient.source, + strict.serializedSchema, + "sha2", + ), + ).not.toBe(false); + }); +}); diff --git a/packages/ui/spa/validation/validateModule.ts b/packages/ui/spa/validation/validateModule.ts new file mode 100644 index 000000000..70e894de6 --- /dev/null +++ b/packages/ui/spa/validation/validateModule.ts @@ -0,0 +1,40 @@ +import { deserializeSchema } from "@valbuild/core"; +import type { + ModuleFilePath, + Schema, + SelectorSource, + SerializedSchema, + Source, + SourcePath, + ValidationErrors, +} from "@valbuild/core"; + +/** + * Framework-free validation logic shared by the validation Web Worker and the + * main-thread fallback in ValidationWorkerClient. Keeps a per-instance cache of + * deserialized schemas keyed by module path, re-deriving only when the schema + * sha changes. + */ +export class SchemaValidator { + private cache = new Map< + ModuleFilePath, + { schemaSha: string; schema: Schema } + >(); + + validate( + moduleFilePath: ModuleFilePath, + source: Source, + serializedSchema: SerializedSchema, + schemaSha: string, + ): ValidationErrors { + let cached = this.cache.get(moduleFilePath); + if (!cached || cached.schemaSha !== schemaSha) { + cached = { schemaSha, schema: deserializeSchema(serializedSchema) }; + this.cache.set(moduleFilePath, cached); + } + return cached.schema["executeValidate"]( + moduleFilePath as string as SourcePath, + source as SelectorSource, + ); + } +} diff --git a/packages/ui/spa/validation/validation.worker.ts b/packages/ui/spa/validation/validation.worker.ts index 13949c859..11fab956f 100644 --- a/packages/ui/spa/validation/validation.worker.ts +++ b/packages/ui/spa/validation/validation.worker.ts @@ -1,39 +1,24 @@ /// -import { - deserializeSchema, - ModuleFilePath, - Schema, - SelectorSource, - SourcePath, -} from "@valbuild/core"; import type { ValidationWorkerRequest, ValidationWorkerResponse, } from "./worker-types"; +import { SchemaValidator } from "./validateModule"; const ctx: DedicatedWorkerGlobalScope = self as unknown as DedicatedWorkerGlobalScope; -const schemaCache = new Map< - ModuleFilePath, - { schemaSha: string; schema: Schema } ->(); +const validator = new SchemaValidator(); ctx.onmessage = (event: MessageEvent) => { const request = event.data; if (request.type !== "validate") return; try { - let cached = schemaCache.get(request.moduleFilePath); - if (!cached || cached.schemaSha !== request.schemaSha) { - cached = { - schemaSha: request.schemaSha, - schema: deserializeSchema(request.serializedSchema), - }; - schemaCache.set(request.moduleFilePath, cached); - } - const errors = cached.schema["executeValidate"]( - request.moduleFilePath as string as SourcePath, - request.source as SelectorSource, + const errors = validator.validate( + request.moduleFilePath, + request.source, + request.serializedSchema, + request.schemaSha, ); const response: ValidationWorkerResponse = { type: "result",