From b3b26e2bb951976dbb3de17eba61c8d9c7dc927b Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Sat, 10 Oct 2026 23:21:52 -0500 Subject: [PATCH 1/3] feat(records): add absent fields and partial updates prototype Prototype draft #398 behind a default-off flag. Preserve append-only duplicates and apply named mutable-state updates as complete validated snapshots on SQLite and PostgreSQL. Record the retained-history limitation and connector emission conflicts. This prototype is not for merge until the specification is final. Refs: PDP-Connect/pdpp#398 Assisted-by: AI Signed-off-by: Tim Nunamaker --- PG-PROFILE-51-REPORT.md | 4 +- docs/261010-ri-absent-fields-prototype.md | 124 +++ .../scripts/postgres-template-eligibility.ts | 2 + .../scripts/scan-append-only-streams.ts | 106 ++ .../server/experimental-record-updates.ts | 221 +++++ .../server/postgres-records.ts | 36 +- reference-implementation/server/records.ts | 61 +- .../test/append-only-manifest-scan.test.ts | 137 +++ .../experimental-record-updates-http.test.ts | 282 ++++++ .../test/experimental-record-updates.test.ts | 903 ++++++++++++++++++ 10 files changed, 1865 insertions(+), 11 deletions(-) create mode 100644 docs/261010-ri-absent-fields-prototype.md create mode 100644 reference-implementation/scripts/scan-append-only-streams.ts create mode 100644 reference-implementation/server/experimental-record-updates.ts create mode 100644 reference-implementation/test/append-only-manifest-scan.test.ts create mode 100644 reference-implementation/test/experimental-record-updates-http.test.ts create mode 100644 reference-implementation/test/experimental-record-updates.test.ts diff --git a/PG-PROFILE-51-REPORT.md b/PG-PROFILE-51-REPORT.md index 34d3311ab..969210206 100644 --- a/PG-PROFILE-51-REPORT.md +++ b/PG-PROFILE-51-REPORT.md @@ -167,8 +167,8 @@ tail: ## Scope counts and comparison baseline `scripts/run-tests.ts` discovers 1,034 profile files. The template equivalence -registry is a different measure: 131 eligible plus 30 cold-required files, -for **161** total. No current tracked source contains a 170-file claim. +registry is a different measure: 133 eligible plus 30 cold-required files, +for **163** total. The two #398 prototype files test record behavior, not schema bootstrap, and are template-eligible. No current tracked source contains a 170-file claim. The sibling report supplies only a memory-default cap-2 baseline (10,252 assertions, 9,690 passed, 209 failed, 353 skipped; about 10 minutes). It is not diff --git a/docs/261010-ri-absent-fields-prototype.md b/docs/261010-ri-absent-fields-prototype.md new file mode 100644 index 000000000..23977dce0 --- /dev/null +++ b/docs/261010-ri-absent-fields-prototype.md @@ -0,0 +1,124 @@ +# Absent fields, partial updates and duplicate keys: RI prototype + +Prototype of [PDPP draft #398](https://github.com/PDP-Connect/pdpp/pull/398). Not for merge until the specification is final. Baseline: `4e69b5dff8a52e3547444a6ac57d7d12dd6940b8`. + +Behavior changes require `PDPP_EXPERIMENTAL_RECORD_UPDATES=1`; the default is off. The experimental profile instruction is `{type:"PDPP_EXPERIMENTAL_PARTIAL_UPDATE_V1", key, set:{...}, remove:[...]}`. The ingest route supplies the stream. Direct storage calls also supply `stream`. `set` replaces named top-level values; `remove` deletes named top-level fields. It cannot include snapshot `data` or `op`. Core defines no such syntax. + +## Requirement evidence + +The named subtests below run on both SQLite and actual PostgreSQL in `reference-implementation/test/experimental-record-updates.test.ts`. **Held** means the tested scenario satisfies the draft. **Wrong** means the tested RI behavior contradicts the draft, not that the draft must change. **Underspecified** identifies a choice or precision bound the draft does not fix. These are bounded prototype findings, not a Core conformance certificate. + +| Requirement | Verdict | Test evidence and result | +| --- | --- | --- | +| Omitted top-level field inside the effective projection has no stored value in that version | Held | `partial set/null/remove preserve unnamed values and projected reads`: removed optional field is absent, not null. | +| Omission outside the projection makes no assertion | Held | Same subtest: a client grant with frozen fields omits the stored required value. HTTP tests exercise owner `fields` and resolved `view` projections. | +| Null is distinct from absence and allowed only by the schema | Held | Same subtest preserves explicit nullable null; `required fields, nullability, identity and malformed instructions roll back atomically` rejects nonnullable null. | +| Required fields and record identity remain mandatory | Held | Atomic rollback subtest rejects required/identity removal and identity mutation. `removing required constructor cannot be satisfied by inherited properties and is atomic` verifies own-property validation. | +| Returned snapshots replace a client's copy within the same projection | Held for the wire representation | `snapshot replacement and complete version history`: an omitted formerly present field disappears; a test-side replacement copy also loses it. There is no production client cache in this RI test. | +| Partial updates are distinct, name set/remove fields and preserve unnamed fields | Held | `partial set/null/remove preserve unnamed values and projected reads` and `concurrent instructions preserve both writers and array order is significant`. | +| Applying an instruction is atomic and yields a complete schema/identity-valid snapshot | Held | Atomic rollback and concurrency subtests compare current records, history and version counters; schema-ID, inherited-property and `overflowing JSON numbers cannot become stored null and reject atomically` regressions are covered. | +| Reads, time constraints, history and incremental results consume complete snapshots | Held | `time constraints and changes_since use complete resulting snapshots` and `snapshot replacement and complete version history`; instructions are never served as records. | +| A partial instruction cannot modify an existing append-only record | Held | `JSON equality and append_only schema/identity validation before duplicate conflict` rejects it before storage. | +| Core specifies no partial-update syntax | Underspecified by design | `flag off retains RECORD legacy rewrites and rejects prototype instructions` verifies the boundary; marker, no-op instructions, missing-record rejection and overlapping set/remove rejection are profile choices. | +| A duplicate is scoped to source, stream, instance and canonical key; identity validation runs first | Held | JSON equality/identity subtest rejects mismatched and incomplete identities; `instance scoping, delete and post-delete reinsert` permits another instance. | +| Equal append-only retry leaves data, emitted_at, version, history and notifications unchanged | Held | JSON equality subtest compares all storage rows/counters and hook calls; `beginning and retries; pruned append_only history anchor stays unchanged` covers the existing self-heal path. | +| JSON-value equality ignores object order and whitespace; arrays retain order | Held | JSON equality subtest uses nested reordering and rejects reordered arrays. | +| Numbers compare by numeric value regardless of spelling | Underspecified precision | JSON equality subtest covers `2.0`/`1e0`. `observed precision limit: distinct unsafe JSON integers collapse before append_only equality` shows that exact numeric comparison beyond JavaScript binary64 precision is not established. | +| Different append-only data reports a producer conflict and should be visible to the owner | Held for the ingest interface | `: HTTP prototype conflicts and malformed instructions have permanent replayable receipts` in `experimental-record-updates-http.test.ts`: permanent conflict code, stable receipts and unchanged owner read. UI presentation of those receipts is unverified. | +| A duplicate creates no incremental result; the existing record appears from beginning | Held with retained history; wrong after pruning | Beginning/retry subtest returns the original record with no duplicate result. `observed residual: beginning after anchor pruning expires although current append_only record survives` instead returns `cursor_expired` on both backends. | +| Explicit deletion remains allowed and produces tombstones | Held | Instance/delete/reinsert subtest asserts a tombstone with no `data` snapshot, then permits reinsertion. Owner deletion uses the existing delete path; this test drives ingest deletion. | +| Collection freshness does not establish reobservation of every returned field | Held as a counterexample | `new collection freshness does not establish reobservation of preserved unnamed fields`: a later partial collection produces current freshness while an unnamed value survives from an earlier write. Freshness derivation is tested directly, not via a collection scheduler. | + +`derived lexical index receives complete snapshots after partial instructions` additionally verifies retained unnamed index fields and deletion of a named indexed field. `schema IDs are scoped to schema contents and do not collide across streams` covers legal schemas with reused `$id` values. + +## Backend findings + +The baseline SQLite no-op check compares serialized JSON. PostgreSQL compares `jsonb`; reordered object members therefore differ on SQLite but not PostgreSQL. The flag-on prototype uses JSON-value equality for append-only retries on both and for SQLite mutable-state retries. Both reject changed append-only data before allocating a version. Mutable-state instructions apply under the existing writer transaction/lock and are indexed from the resulting complete snapshot with a version guard. + +Baseline reproductions failed with a missing append-only rejection and a SQLite reordered-object rewrite. Both pass after the patch. Independent review also reproduced a shared AJV registry collision for reused schema IDs and acceptance of removed required `constructor` through inheritance. The prototype isolates schema registration and requires own properties; both regressions now pass on both backends. Default-off tests also preserve legacy RECORD envelopes with ignored `set`/`remove` extensions and reject the explicit experimental marker. + +The retained-history requirement is not a blanket pass. The existing RI repairs missing append-only history anchors by making an equal retry a changed write. That violates #398's no-new-version requirement. This prototype prevents that repair; the current record remains readable, but a new beginning session expires. Resolving the feed behavior requires the later change-feed step; changing the draft is not implied by this finding. + +Numeric precision is a second scope limit. `operations/rs-records-ingest/index.ts` parses NDJSON with native `JSON.parse`, so distinct JSON integers `9007199254740992` and `9007199254740993` both become `9007199254740992`. An ordinary `type: "integer"` schema accepts the parsed value; neither backend can recover the original digits after storage serialization. No global safe-integer or I-JSON restriction was found in the installed contract or the local Core draft inspected. The prototype compares parsed JavaScript numeric values, not arbitrary-precision decimals. #398 does not specify a numeric precision bound; the large-integer test records this limit without rewriting the ingest parser. + +Overflow is a separate failure from integer precision. Native parsing turns `1e400` into Infinity, which permissive schema validation accepted and serialization replaced with null. Both backends reproduced this nonnullable-field violation. The prototype now rejects non-finite numbers recursively before validation, including nested objects, arrays and fields without a numeric type constraint; snapshot and partial-write rejection leave storage unchanged. This restriction is a prototype representability choice, not a new Core numeric bound. + +## Connector scan + +Run `node reference-implementation/scripts/scan-append-only-streams.ts `. The scanner inventories declared streams and marks field-name candidates. Implementation evidence is required to call a candidate a conflict. `append-only-manifest-scan.test.ts` verifies discovery, exclusions and a same-content Apple Photos metadata change using the shipped parser. + +| Input | Manifests | Streams | append_only streams | +| --- | ---: | ---: | ---: | +| RI pinned connector source `06ed284e1b7ee92b2ebab6bc8ef25f0ae392f84a` | 52 | 211 | 31 | +| RI shipped polyfill tarball | 45 | 173 | 31 | +| RI seed manifests | 4 | 11 | 3 | +| Newer connector source `302363c0cf52c916d196ee6b1ee979ef67866f3e` | 52 | 212 | 31 | + +These inventories overlap; the counts are not additive. Pinned Chase balances include run-time `fetched_at` under the same account/as-of key, and transactions explicitly re-emit changed business data under the same transaction key. Newer Chase source removes that run-time data field; the pinned RI still has it. Apple Photos uses a content-only key but includes filename/mtime/hydration metadata. ChatGPT message IDs also carry mutable conversation-graph fields. The complete pinned table follows below. + +## Validation + +The combined three-file run passes 41 tests with zero failures or skips, including actual SQLite and PostgreSQL the numeric precision observation and overflow rejection. Typecheck and scoped checks pass for the five new TypeScript files without warnings. The existing PostgreSQL template registry includes both new application-behavior test files; its three inventory checks pass. Repository-wide `check` fails on the untouched baseline as well; shared files are not reformatted wholesale. + +The full RI suite completed with the ambient flag off: 1,124 file outputs, 11,259 assertions, 10,865 passed, zero failed and 394 declared skips; runner exit 0. It used `memory-default` with concurrency four. PostgreSQL-dependent tests in this full run are declared skips; the separate 41-test run used the actual PostgreSQL oracle without skips. The overflow guard was added after the prototype files had completed in the full run; the final focused run covers that flag-on-only change and both flag-off storage cases. Final typecheck and scoped formatting checks pass. + +Earlier full attempts were interrupted by console setup, a T3 restart and a stalled integration file. The stalled `pdpp.test.ts` passed all 118 tests in isolation on both the current revision and untouched baseline. The final full run also completed that file. Its success does not establish the earlier stall cause. The new test files needed template-registry entries; those two entries are included and all three inventory checks pass. + +Live-account connector replay, cross-process concurrent writers, semantic embedding updates and a production client cache are not verified. Independent review also notes that existing SQLite rows are JSON-parsed before the disabled append-only helper returns; default-off performance overhead is unmeasured. PostgreSQL tests use disposable databases and the existing JSONB record oracle; they do not require the optional vector extension. + +## Pinned implementation audit + +All 31 pinned append-only streams were inspected: 14 have concrete conflicting-emission paths, 11 have possible changes and six have no conflict found. The following paths refer to the pinned connector source named above. No live account replay was performed. + +Classification: + +- **Confirmed:** implementation contains a concrete changed-data/same-key emission path. This is static code evidence, not an observed account incident. +- **Possible:** source metadata outside the key can change; no concrete occurrence was established. +- **No found:** no conflicting rewrite path found in the inspected implementation; not a guarantee. + +| Append-only stream | Classification | Evidence | +|---|---|---| +| `anthropic.messages` | Possible | `connectors/anthropic/parsers.ts:203` uses message UUID; `:211–219` stores content, update time and attachments. `index.ts:1393–1402` emits messages again for in-window conversations. Changed exports can preserve UUID. | +| `apple-health.records` | Possible | `connectors/apple_health/parsers.ts:235` hashes type/source/start/value; `:242–253` adds source version, device, end date and metadata outside that identity. `index.ts:265` emits it, subject to start-date cursor. | +| `apple-health.workouts` | Possible | `connectors/apple_health/parsers.ts:269` hashes activity/source/start; `:276–297` stores duration, totals, events, statistics and metadata outside identity. `index.ts:286` emits it, subject to cursor. | +| `apple-photos.photos` | **Confirmed** | `connectors/apple_photos/parsers.ts:88–105`: ID depends on content hash, while data includes filename, mtime and hydration outcome. `index.ts:218–233` emits each file newer than prior cursor. Identical bytes under another filename or later export mtime produce changed data with the same ID—even within one directory walk. | +| `chase.transactions` | **Confirmed, conditional** | `connectors/chase/index.ts:1795–1825`: ID is account ID + FITID; payload includes `fetched_at` and acquisition `source`. `:3120–3128` fingerprints exclude those fields. Ordinary unchanged repeats are suppressed, **but reset/legacy fingerprint state re-emits changed run metadata**, and real transaction-field changes explicitly trigger same-key emission. Do not report unconditional per-run churn here. | +| `chase.balances` | **Confirmed** | `connectors/chase/index.ts:2471–2477`: ID is account ID + balance `as_of`; every emitted payload includes current run `fetched_at`, with no fingerprint gate. Re-fetching unchanged balance data changes the same-key snapshot. | +| `chatgpt.messages` | **Confirmed** | `connectors/chatgpt/parsers.ts:432–449`: node ID is stable, but data includes `children_ids` and `on_current_branch`. Adding a reply changes the existing parent’s children; branch selection can change membership. `index.ts:2605` re-emits messages from updated conversations. This is stronger evidence than merely assuming message text edits. | +| `claude-code.messages` | No found | `connectors/claude_code/index.ts:326–347` maps source UUID and JSONL event fields directly. No hydration/run-clock fields in the message snapshot. | +| `claude-code.attachments` | **Confirmed** | `connectors/claude_code/index.ts:385–407` attaches capture status/hash to stable UUID. File attachments use stable path-derived key at `:589`; `:601–618` emit capture status/hash and retain failed capture obligations for another attempt. Successful retry changes the same-key snapshot. | +| `codex.messages` | No found | `connectors/codex/index.ts:911–930`: ID is session + line number and data comes from that log entry; append-tail cursor handling starts at `:1454`. No ordinary same-key mutation found. | +| `codex.function_calls` | **Confirmed on full replay** | `connectors/codex/index.ts:952–961` constructs call-ID record with null output; `:1144–1152` flushes unanswered calls. `:980–993` emits populated output under that call ID when call/output are parsed together. A later full replay of a completed log changes the previously incomplete call snapshot. Normal append-only output fallback at `:995` uses a **different** key, so do not claim every incremental completion overwrites it. | +| `github.events` | No found | `connectors/github/parsers.ts:336–350` stores event ID/type/time/repository/public flag. `index.ts:1574–1583` skips events at/before prior cursor. No connector-created changing field found. | +| `gmail.message_bodies` | **Confirmed** | `connectors/gmail/index.ts:977–1025` emits the body record before testing `fetchFailed`, then records a retry gap. `:739–748` explicitly describes failed-fetch empty records and same-message retry. `parsers.ts:810–826` keeps Gmail message ID while body data changes after successful hydration. | +| `gmail.attachments` | **Confirmed** | `connectors/gmail/index.ts:3190–3204` emits existing attachment identity with null blob/hash and failed status; `:3755–3759` supplies hydrated blob/hash/status. Emission/retry paths at `:879–883` and `:2671–2691` retain the same attachment identity. | +| `google-maps.timeline_points` | Possible | `connectors/google_maps/parsers.ts:178–201` hashes source/kind/segment/time/rounded coordinates, but stores accuracy, altitude, speed, activity and full coordinates outside identity. Re-imported corrected metadata can conflict. | +| `google-maps.timeline_segments` | Possible | `connectors/google_maps/parsers.ts:216–244` hashes substantial segment identity but omits probability and semantic type. Corrected exports can retain key while changing those values. | +| `google-messages.messages` | Possible | `connectors/google_messages/index.ts:388–408` keys by provider message ID but includes current `chat_name`, sender and body. `:797–822` explicitly re-emits when fingerprint changes. Conversation-name changes are a concrete candidate; no source rename replay was performed. | +| `google-takeout.location_history` | Possible | `connectors/google_takeout/parsers.ts:81–89` hashes time/coordinates, while accuracy/activity/velocity/altitude are outside key. `index.ts:148–152` emits records after cursor filtering. | +| `google-takeout.youtube_watch_history` | Possible | `packages/polyfill-connectors/src/youtube-watch-history.ts:62–67` hashes watch time + video URL, but stores video title and channel name/URL. `connectors/google_takeout/index.ts:219` emits the result. Metadata changes in a later export can conflict on replay. | +| `google-takeout.search_history` | Possible | `connectors/google_takeout/parsers.ts:115–123` hashes time/query, with product/header outside identity. `index.ts:281` emits it. No changing-header occurrence established. | +| `google-takeout.photos` | **Confirmed** | `connectors/google_takeout/parsers.ts:179–192` hashes content bytes but stores filename/event time/title/description/location. `index.ts:341–375` adds hydration fields; `:393–404` emits every eligible album copy without same-run content-ID deduplication. Duplicate album copies or retry/re-import hydration can therefore produce changed same-key snapshots. | +| `groupme.direct_chat_messages` | **Confirmed** | `connectors/groupme/index.ts:1062–1085` keeps provider message ID and embeds normalized attachments. `:747–748` adds `normalized.blob_id` only after successful upload. `:2138–2145` fingerprint gate emits the changed message after hydration recovery. | +| `groupme.attachments` | **Confirmed** | `connectors/groupme/index.ts:733–771` keeps attachment ID while hydration status/blob/hash change. `:1797–1800` explicitly withholds parent cursor after hydration failure for retry; `:1821–1822` emits changed fingerprints. | +| `imessage.attachments` | **Confirmed** | `connectors/imessage/index.ts:687–713` uses stable source attachment identity and emits every queried attachment with hydration status/error/hash/blob. Later successful hydration changes the same-key record. | +| `netflix-export.viewing_activity` | Possible | `connectors/netflix_export/parsers.ts:674–684` hashes title + normalized day but stores raw date; `:707–720` full-export key excludes country/raw timestamp. Alternative representations or corrected metadata can conflict on replay. No observed export pair established. | +| `signal.reactions` | No found | `connectors/signal/parsers.ts:205–212` derives ID from precisely the three emitted semantic values: message ID, emoji and sender. `index.ts:777` emits that deterministic record. Removal propagation is separate from changed-data duplicates. | +| `signal.attachments` | **Confirmed** | `connectors/signal/index.ts:1121–1145` keys by local attachment path and emits hydration outcome/hash/blob and linked metadata. Retrying the same path after upload availability/failure changes data. | +| `twitter-archive.tweets` | Possible | `connectors/twitter_archive/parsers.ts:71–80` keys by tweet ID but stores favorite/retweet counts and reply screen name. `index.ts:92` emits after creation-time filtering; later export replay can carry changed counters. | +| `twitter-archive.direct_messages` | No found | `connectors/twitter_archive/parsers.ts:116–122` directly maps message ID, participants, creation time and text. `index.ts:146–150` applies cursor and emits. No connector-created mutation found. | +| `whatsapp.messages` | No found | `connectors/whatsapp/parsers.ts:253–261` content-addresses author/time/content/occurrence; `:923–932` derives attachment flag from content. `index.ts:273–283` emits those same inputs. Edited text normally changes the key. | +| `whatsapp.attachments` | **Confirmed** | `connectors/whatsapp/index.ts:776–809` keys attachment independently of upload result, but includes failed/deferred/hydrated status, error and blob. `:839–841` emits changed fingerprints on subsequent imports. | + +**Inventory distinction:** both scan files list 31 append-only streams, but they are not identical sets: pinned source adds `github.events`; vendor manifests instead contain `spotify.recently_played`. The pinned Spotify implementation only emits `SKIP_RESULT` for `recently_played` at `connectors/spotify/index.ts:935–947`. That does **not** establish behavior of the older shipped vendor implementation. This review audited pinned TypeScript implementations; vendor manifest overlap alone cannot prove packaged JavaScript behaves identically. + + +## Shipped-only and seed findings + +| Declaration | Classification | Evidence | +|---|---|---| +| Shipped `spotify.recently_played` | Possible | `connectors/spotify/index.js:343–365` fetches and emits recently played records. The key at line 353 is track ID + played-at milliseconds; track name, artist names, album name and context type at lines 357–361 are outside identity. Lines 66–73 intentionally replay the one-millisecond cursor boundary. Metadata changes can therefore produce changed same-key records, including a boundary replay, but no changing provider response was observed. No run-clock field is added to data. This is an actual emitter, unlike the pinned revision's deferred stream. | +| RI seed `reddit.posts` | Confirmed by two local process runs | `reference-implementation/connectors/seed/index.ts:39–44` computes fixture dates from the current clock. Lines 394–400 pair fixed post ID with relative `source_created_at`; lines 595–599 emit those fixtures every run. Two fresh local processes with identical START scope emitted the same key with different data; the changed field was `source_created_at`. | +| RI seed `reddit.comments` | Confirmed by two local process runs | The same seed file at lines 447–451 pairs fixed comment ID with relative `source_created_at`; lines 601–605 emit it each run. The same two-process check confirmed equal keys but changed `source_created_at`. This finding concerns synthetic RI fixtures, not the production Reddit connector. | + +The seed reproduction ran `reference-implementation/connectors/seed/index.ts` twice with `START.scope.streams` equal to `posts,comments`, parsed RECORD output, and compared the first emitted record of each stream. Both checks returned `sameKey: true`, `sameData: false`, `changedFields: ["source_created_at"]`. No accounts or network access were used. diff --git a/reference-implementation/scripts/postgres-template-eligibility.ts b/reference-implementation/scripts/postgres-template-eligibility.ts index 092c85363..1aefe29b7 100644 --- a/reference-implementation/scripts/postgres-template-eligibility.ts +++ b/reference-implementation/scripts/postgres-template-eligibility.ts @@ -83,6 +83,8 @@ export const POSTGRES_TEMPLATE_ELIGIBLE_FILES: readonly string[] = [ "test/device-ingest-attempt-context-store.test.ts", "test/device-ingest-conformance.test.ts", "test/error-code-query-not-found.test.ts", + "test/experimental-record-updates-http.test.ts", + "test/experimental-record-updates.test.ts", "test/first-party-owner-credential-reuse.test.ts", "test/forward-evidence-debt-wired-probe.test.ts", "test/grant-lifecycle-expired-reporting.test.ts", diff --git a/reference-implementation/scripts/scan-append-only-streams.ts b/reference-implementation/scripts/scan-append-only-streams.ts new file mode 100644 index 000000000..6071195c3 --- /dev/null +++ b/reference-implementation/scripts/scan-append-only-streams.ts @@ -0,0 +1,106 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import { readdir, readFile } from "node:fs/promises"; +import { basename, resolve } from "node:path"; +import { pathToFileURL } from "node:url"; + +interface ManifestStream { + name: string; + primary_key?: string[]; + schema?: { properties?: Record }; + semantics: string; +} + +interface ConnectorManifest { + connector_key?: string; + streams?: ManifestStream[]; + version?: string; +} + +const RUN_TIME_FIELD = + /^(?:fetched|collected|captured|observed|synced|scraped|exported)_at$/; +const SOURCE_MUTABLE_FIELD = + /(?:^|_)(?:counts?|status|score|likes|followers|title|body|content|updated_at|edited_at)(?:_|$)/; +const EXCLUDED_DIRECTORIES = new Set([".git", "node_modules", "dist", "build"]); + +/** Candidates require implementation evidence; a manifest cannot prove re-emission. */ +export function inspectAppendOnlyManifest( + manifest: ConnectorManifest, + manifestPath: string, +) { + return (manifest.streams ?? []) + .filter((stream) => stream.semantics === "append_only") + .map((stream) => { + const fields = Object.keys(stream.schema?.properties ?? {}); + return { + connector: manifest.connector_key ?? null, + fields, + manifest: manifestPath, + primary_key: stream.primary_key ?? [], + run_time_field_candidates: fields.filter((field) => + RUN_TIME_FIELD.test(field), + ), + source_mutable_field_candidates: fields.filter((field) => + SOURCE_MUTABLE_FIELD.test(field), + ), + stream: stream.name, + version: manifest.version ?? null, + }; + }); +} + +async function manifestPaths(directory: string): Promise { + const entries = await readdir(directory, { withFileTypes: true }); + const paths: string[] = []; + const namedManifestDirectory = ["manifests", "seed-manifests"].includes( + basename(directory), + ); + for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) { + const path = resolve(directory, entry.name); + if (entry.isDirectory() && !EXCLUDED_DIRECTORIES.has(entry.name)) { + paths.push(...(await manifestPaths(path))); + } else if ( + entry.isFile() && + (entry.name === "manifest.json" || + (namedManifestDirectory && entry.name.endsWith(".json"))) + ) { + paths.push(path); + } + } + return paths; +} + +export async function scanAppendOnlyStreams(directory: string) { + const manifests = await manifestPaths(resolve(directory)); + const streams = []; + let streamCount = 0; + for (const path of manifests) { + const manifest = JSON.parse( + await readFile(path, "utf8"), + ) as ConnectorManifest; + streamCount += manifest.streams?.length ?? 0; + streams.push(...inspectAppendOnlyManifest(manifest, path)); + } + return { + append_only_stream_count: streams.length, + limitation: + "Static manifest candidates only. Verify re-emission in connector code or a two-run replay.", + manifest_count: manifests.length, + stream_count: streamCount, + streams, + }; +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(resolve(process.argv[1])).href +) { + const directory = process.argv[2]; + if (!directory) { + throw new Error( + "usage: node scripts/scan-append-only-streams.ts ", + ); + } + console.log(JSON.stringify(await scanAppendOnlyStreams(directory), null, 2)); +} diff --git a/reference-implementation/server/experimental-record-updates.ts b/reference-implementation/server/experimental-record-updates.ts new file mode 100644 index 000000000..224cb0945 --- /dev/null +++ b/reference-implementation/server/experimental-record-updates.ts @@ -0,0 +1,221 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import { createRequire } from "node:module"; +import { + assertRecordIdentity, + normalizePrimaryKey, +} from "./record-expand-helpers.ts"; + +// Prototype profile syntax, deliberately distinct from a RECORD snapshot. +export const EXPERIMENTAL_PARTIAL_UPDATE_TYPE = + "PDPP_EXPERIMENTAL_PARTIAL_UPDATE_V1"; +export interface ExperimentalUpdateInput { + data?: unknown; + key: unknown; + op?: unknown; + remove?: unknown; + set?: unknown; + type?: unknown; +} +export interface ExperimentalStream { + primary_key?: unknown; + schema?: unknown; + semantics?: unknown; +} +interface Validator { + errors?: unknown; + (data: unknown): boolean; +} +const requireFromContract = createRequire( + import.meta.resolve("@pdpp/reference-contract"), +); +const Ajv = requireFromContract("ajv/dist/2020.js") as new ( + options: object, +) => { compile: (schema: object) => Validator }; +const ajv = new Ajv({ + addUsedSchema: false, + allErrors: true, + ownProperties: true, + strict: false, +}); +(requireFromContract("ajv-formats") as (instance: unknown) => void)(ajv); +const validators = new Map(); + +export function experimentalRecordUpdatesEnabled(): boolean { + return process.env.PDPP_EXPERIMENTAL_RECORD_UPDATES === "1"; +} +export function recordUpdateError( + code: string, + message: string, +): Error & { code: string } { + return Object.assign(new Error(message), { code }); +} +function isObject(value: unknown): value is Record { + return value !== null && typeof value === "object" && !Array.isArray(value); +} +function containsNonFiniteNumber(value: unknown): boolean { + if (typeof value === "number") { + return !Number.isFinite(value); + } + if (value !== null && typeof value === "object") { + return Object.values(value).some(containsNonFiniteNumber); + } + return false; +} +export function isPartialInstruction(input: ExperimentalUpdateInput): boolean { + return ( + input.type === EXPERIMENTAL_PARTIAL_UPDATE_TYPE || + (experimentalRecordUpdatesEnabled() && + (input.set !== undefined || input.remove !== undefined)) + ); +} +export function validateExperimentalSnapshot( + stream: ExperimentalStream | null, + key: unknown, + data: unknown, +): void { + if (!isObject(data)) { + throw recordUpdateError( + "invalid_record_schema", + "RECORD data must be an object", + ); + } + if (containsNonFiniteNumber(data)) { + throw recordUpdateError( + "invalid_record_schema", + "RECORD numbers must remain finite after JSON parsing", + ); + } + const primaryKey = normalizePrimaryKey(stream?.primary_key); + const scalarKeyParts = + typeof key === "string" || typeof key === "number" ? [key] : []; + const keyParts = Array.isArray(key) ? key : scalarKeyParts; + if ( + primaryKey.length > 0 && + (keyParts.length !== primaryKey.length || + keyParts.some( + (part) => + part === null || part === undefined || typeof part === "object", + )) + ) { + throw recordUpdateError( + "invalid_record_identity", + "Key must provide every declared primary-key value", + ); + } + for (const field of primaryKey) { + if (!Object.hasOwn(data, field)) { + throw recordUpdateError( + "invalid_record_identity", + `Missing primary-key field '${field}'`, + ); + } + } + assertRecordIdentity(primaryKey, key, data); + if (stream?.schema && typeof stream.schema === "object") { + const cacheKey = JSON.stringify(stream.schema); + let validate = validators.get(cacheKey); + if (!validate) { + validate = ajv.compile(stream.schema); + validators.set(cacheKey, validate); + } + if (!validate(data)) { + throw recordUpdateError( + "invalid_record_schema", + "RECORD data does not satisfy the stream schema", + ); + } + } +} +export function applyPartialInstruction( + input: ExperimentalUpdateInput, + stream: ExperimentalStream | null, + current: unknown, +): Record { + if (!experimentalRecordUpdatesEnabled()) { + throw recordUpdateError( + "invalid_partial_update", + "Experimental partial-update profile is disabled", + ); + } + if ( + input.type !== EXPERIMENTAL_PARTIAL_UPDATE_TYPE || + input.op !== undefined || + input.data !== undefined || + !isObject(input.set) || + !Array.isArray(input.remove) || + input.remove.some( + (field) => typeof field !== "string" || field.length === 0, + ) || + Object.keys(input.set).some( + (field) => + field.length === 0 || (input.remove as unknown[]).includes(field), + ) || + new Set(input.remove).size !== input.remove.length + ) { + throw recordUpdateError( + "invalid_partial_update", + "Instruction requires distinct top-level set fields and remove field names", + ); + } + if (stream?.semantics !== "mutable_state") { + throw recordUpdateError( + "invalid_partial_update", + "Partial updates require a mutable_state stream", + ); + } + if (!isObject(current)) { + throw recordUpdateError( + "invalid_partial_update", + "Partial update requires an existing live record", + ); + } + const result = { ...current, ...input.set }; + for (const field of input.remove as string[]) { + delete result[field]; + } + validateExperimentalSnapshot(stream, input.key, result); + return result; +} +export function equalJsonValues(left: unknown, right: unknown): boolean { + if (left === right) { + return true; + } + if (Array.isArray(left) && Array.isArray(right)) { + return ( + left.length === right.length && + left.every((value, index) => equalJsonValues(value, right[index])) + ); + } + if (isObject(left) && isObject(right)) { + const keys = Object.keys(left); + return ( + keys.length === Object.keys(right).length && + keys.every( + (key) => + Object.hasOwn(right, key) && equalJsonValues(left[key], right[key]), + ) + ); + } + return false; +} +export function enforceAppendOnlyDuplicate( + stream: ExperimentalStream | null, + currentData: unknown, + newData: unknown, +): boolean { + if ( + !experimentalRecordUpdatesEnabled() || + stream?.semantics !== "append_only" + ) { + return false; + } + if (!equalJsonValues(currentData, newData)) { + throw recordUpdateError( + "append_only_record_conflict", + "Append-only record already exists with differing data", + ); + } + return true; +} diff --git a/reference-implementation/server/postgres-records.ts b/reference-implementation/server/postgres-records.ts index f481a628c..1bec24c67 100644 --- a/reference-implementation/server/postgres-records.ts +++ b/reference-implementation/server/postgres-records.ts @@ -12,6 +12,7 @@ */ import { createHash } from "node:crypto"; +import { applyPartialInstruction, enforceAppendOnlyDuplicate, experimentalRecordUpdatesEnabled, isPartialInstruction, validateExperimentalSnapshot } from "./experimental-record-updates.ts"; import { buildLimitClampedWarning, clampRecordsPageLimit, @@ -91,6 +92,7 @@ async function postgresQuery( } interface ManifestStream { + semantics?: string; consent_time_field?: string; cursor_field?: string; name?: string; @@ -248,6 +250,9 @@ interface PgQueryError extends Error { } interface IngestRecord { + type?: unknown; + set?: unknown; + remove?: unknown; data?: JsonObject; emitted_at?: string; key: unknown; @@ -1899,9 +1904,10 @@ export async function postgresIngestRecord( ): Promise { const connectorId = resolveStorageConnectorId(storageTarget); const connectorInstanceId = resolveStorageConnectorInstanceId(storageTarget, connectorId ?? ""); - const { stream, key, data, emitted_at: emittedAt, op = "upsert" } = record; + const { stream, key, emitted_at: emittedAt, op = "upsert" } = record; + let data = record.data; const recordKey = encodeKey(key); - const recordJson = data ? JSON.stringify(data) : null; + let recordJson = data ? JSON.stringify(data) : null; // Validate record identity against the manifest-declared primary_key (covers // non-`id` and compound keys), via the same shared guard the SQLite store @@ -1914,6 +1920,11 @@ export async function postgresIngestRecord( attemptStreamFacts ? manifestStreamFromFacts(attemptStreamFacts) : null ); assertRecordIdentity(normalizePrimaryKey(identityManifestStream?.primary_key), key, data ?? null); + const experimentalStream = experimentalRecordUpdatesEnabled() || isPartialInstruction(record) + ? await getCachedPostgresManifestStream(resolvedConnectorId, stream) : null; + if (experimentalRecordUpdatesEnabled() && op !== "delete" && !isPartialInstruction(record)) { + validateExperimentalSnapshot(experimentalStream, key, data); + } const effectiveEmittedAt = emittedAt || nowIso(); const changeHistoryLimit = getChangeHistoryLimit(); @@ -1923,11 +1934,11 @@ export async function postgresIngestRecord( ? manifestStreamFromFacts(attemptStreamFacts) : await getCachedPostgresManifestStream(resolvedConnectorId, stream); } - const storedCursorValue = op === "delete" ? null : cursorValue(data ?? null, manifestStream); - const storedPrimaryKeyText = op === "delete" ? recordKey : primaryKeyText(data ?? null, recordKey, manifestStream); + let storedCursorValue = op === "delete" ? null : cursorValue(data ?? null, manifestStream); + let storedPrimaryKeyText = op === "delete" ? recordKey : primaryKeyText(data ?? null, recordKey, manifestStream); // SEMANTIC time for the Explore merged-timeline sort (upserts only; a delete // keeps the row's existing semantic_time). Falls back to emitted_at. - const storedSemanticTime = op === "delete" ? null : semanticTimeValue(data ?? null, manifestStream); + let storedSemanticTime = op === "delete" ? null : semanticTimeValue(data ?? null, manifestStream); const outcome = await withPostgresTransaction( async (client) => { @@ -1967,6 +1978,21 @@ export async function postgresIngestRecord( ); const current = currentResult.rows[0] || null; + if (isPartialInstruction(record)) { + data = applyPartialInstruction(record, experimentalStream, current && !current.deleted ? current.record_json : null); + recordJson = JSON.stringify(data); + storedCursorValue = cursorValue(data, manifestStream); + storedPrimaryKeyText = primaryKeyText(data, recordKey, manifestStream); + storedSemanticTime = semanticTimeValue(data, manifestStream); + if (current) { + const equality = await client.query("SELECT $1::jsonb = $2::jsonb AS equal", [JSON.stringify(current.record_json), recordJson]); + current.is_identical = equality.rows[0]?.equal; + } + } + if (op !== "delete" && current && !current.deleted && enforceAppendOnlyDuplicate(experimentalStream, current.record_json, data)) { + return finishDurableOutcome({ kind: "noop" }); + } + if (op === "delete" && (!current || current.deleted)) { return finishDurableOutcome({ kind: "noop" }); } diff --git a/reference-implementation/server/records.ts b/reference-implementation/server/records.ts index 03233d30e..8230aac86 100644 --- a/reference-implementation/server/records.ts +++ b/reference-implementation/server/records.ts @@ -6,6 +6,7 @@ import { parseCoverageDiagnosticsStateSnapshot } from "./polyfill-connectors-run * PDPP Resource Server — record storage and grant-enforced query */ import { getDb } from "./db.ts"; +import { applyPartialInstruction, enforceAppendOnlyDuplicate, equalJsonValues, experimentalRecordUpdatesEnabled, isPartialInstruction, recordUpdateError, validateExperimentalSnapshot, type ExperimentalStream } from "./experimental-record-updates.ts"; import { assertGrantedManifestReadAuthority, assertManifestReadAuthority } from "./manifest-read-authority.ts"; import type { ComputedLexicalFields } from "./search.ts"; import type { ComputedSemanticEntries } from "./search-semantic.ts"; @@ -174,6 +175,9 @@ type RecordStorageTarget = | undefined; type RecordData = Record; interface RecordEnvelope { + type?: unknown; + set?: unknown; + remove?: unknown; data?: unknown; emitted_at?: string | null | undefined; key: unknown; @@ -854,6 +858,9 @@ export class RecordIndexAdmissionError extends Error { * by construction, not a correctness bug. */ const PERMANENT_INGEST_FAILURE_CODES: ReadonlySet = new Set([ + "append_only_record_conflict", + "invalid_record_schema", + "invalid_partial_update", "connector_instance_not_found", "connector_instance_not_writable", "invalid_record_identity", @@ -2016,19 +2023,29 @@ function ingestRecordWithinCoordinator( } function toPostgresRecordInput(record: RecordEnvelope): { + type?: unknown; + set?: unknown; + remove?: unknown; data?: RecordData; emitted_at?: string; key: unknown; op?: "delete" | "upsert"; stream: string; } { + if (isPartialInstruction(record) && (record.data !== undefined || record.op !== undefined)) { + throw recordUpdateError("invalid_partial_update", "Instruction must not contain snapshot data or an op"); + } if (record.data !== undefined && record.data !== null && !isRecordData(record.data)) { + if (experimentalRecordUpdatesEnabled()) throw recordUpdateError("invalid_record_schema", "RECORD data must be an object"); throw new TypeError("Postgres record data must be an object when present"); } if (record.emitted_at !== undefined && record.emitted_at !== null && typeof record.emitted_at !== "string") { throw new TypeError("Postgres record emitted_at must be a string when present"); } return { + ...(record.type !== undefined ? { type: record.type } : {}), + ...(record.set !== undefined ? { set: record.set } : {}), + ...(record.remove !== undefined ? { remove: record.remove } : {}), key: record.key, stream: record.stream, ...(isRecordData(record.data) ? { data: record.data } : {}), @@ -2278,9 +2295,15 @@ async function ingestSqliteRecord( ): Promise { const connectorId = connectorIdForStorageTarget(storageTarget); const connectorInstanceId = resolveStorageConnectorInstanceId(storageTarget, connectorId); - const { stream, key, data, emitted_at, op = "upsert" } = record; + const { stream, key, emitted_at, op = "upsert" } = record; + let data = record.data; const recordKey = encodeKey(key); - const recordJson = data ? JSON.stringify(data) : null; + let recordJson = data ? JSON.stringify(data) : null; + const experimentalStream = experimentalRecordUpdatesEnabled() || isPartialInstruction(record) + ? getExperimentalManifestStream(connectorId, stream) : null; + if (experimentalRecordUpdatesEnabled() && op !== "delete" && !isPartialInstruction(record)) { + validateExperimentalSnapshot(experimentalStream, key, data); + } // Validate record identity against the manifest-declared primary_key. The // record `key` is the ordered tuple of primary-key field values; each @@ -2303,7 +2326,7 @@ async function ingestSqliteRecord( // Only meaningful for upserts (a delete keeps the row's existing // semantic_time); computed unconditionally for a simpler, branch-free bind // below — the delete path does not write it. - const semanticTime = + let semanticTime = op === "delete" ? SEMANTIC_TIME_UNKNOWN : computeIngestSemanticTime(connectorId, stream, data, attemptStreamFacts); const changeHistoryLimit = getChangeHistoryLimit(); @@ -2326,6 +2349,15 @@ async function ingestSqliteRecord( recordKey, ]); + if (isPartialInstruction(record)) { + data = applyPartialInstruction(record, experimentalStream, current && !current.deleted && current.record_json ? JSON.parse(current.record_json) : null); + recordJson = JSON.stringify(data); + semanticTime = computeIngestSemanticTime(connectorId, stream, data, attemptStreamFacts); + } + if (op !== "delete" && current && !current.deleted && enforceAppendOnlyDuplicate(experimentalStream, current.record_json ? JSON.parse(current.record_json) : null, data)) { + return finishDurableOutcome({ kind: "noop" }); + } + if (op === "delete" && (!current || current.deleted)) { return finishDurableOutcome({ kind: "noop" }); } @@ -2349,7 +2381,9 @@ async function ingestSqliteRecord( // write). The downstream changed-write delta math is already correct // for an unchanged payload at a new version: the record-count and // current-bytes deltas are zero, and only one history row is appended. - const wouldBeUnchangedUpsert = op !== "delete" && current && !current.deleted && current.record_json === recordJson; + const wouldBeUnchangedUpsert = op !== "delete" && current && !current.deleted && (experimentalRecordUpdatesEnabled() + ? equalJsonValues(current.record_json ? JSON.parse(current.record_json) : null, data) + : current.record_json === recordJson); // Anchor present → genuine no-op. Anchor missing → fall through to // the changed-write path below to re-anchor the current row. if (wouldBeUnchangedUpsert) { @@ -2786,6 +2820,18 @@ async function maintainRecordIndexesWithinPermit( ): Promise { const connectorId = connectorIdForStorageTarget(storageTarget); const connectorInstanceId = resolveStorageConnectorInstanceId(storageTarget, connectorId); + if (isPartialInstruction(record)) { + const recordKey = encodeKey(record.key); + const current = isPostgresStorageBackend() + ? (await postgresQuery<{ record_json: RecordData; version: number; deleted: boolean }>( + "SELECT record_json, version, deleted FROM records WHERE connector_instance_id = $1 AND stream = $2 AND record_key = $3", + [connectorInstanceId, record.stream, recordKey] + )).rows[0] + : getOne(referenceQueries.recordsIngestGetCurrentRecordState, [connectorInstanceId, record.stream, recordKey]); + if (!current || current.deleted || Number(current.version) !== expectedVersion) return; + const data = typeof current.record_json === "string" ? JSON.parse(current.record_json) : current.record_json; + record = { stream: record.stream, key: record.key, data, op: "upsert" }; + } const { stream, key, data, op = "upsert" } = record; const recordKey = encodeKey(key); const attemptFacts = options.attemptContext?.streams?.[stream] ?? null; @@ -8823,6 +8869,13 @@ export async function backfillSqliteRecordSemanticTimesForManifest( return { streams: streamOutcomes, updated }; } +function getExperimentalManifestStream(connectorId: string, streamName: string): ExperimentalStream | null { + const row = getOne(referenceQueries.authConnectorsGetManifestById, [connectorId]); + if (!row?.manifest) return null; + const manifest = JSON.parse(row.manifest) as { streams?: Array }; + return manifest.streams?.find(candidate => candidate.name === streamName) ?? null; +} + // Returns the manifest-declared primary_key field names for a stream, or null // when the manifest/stream is unavailable. Mirrors getManifestConsentTimeField's // load path so identity validation uses the same manifest source of truth. diff --git a/reference-implementation/test/append-only-manifest-scan.test.ts b/reference-implementation/test/append-only-manifest-scan.test.ts new file mode 100644 index 000000000..c9d171c30 --- /dev/null +++ b/reference-implementation/test/append-only-manifest-scan.test.ts @@ -0,0 +1,137 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { + inspectAppendOnlyManifest, + scanAppendOnlyStreams, +} from "../scripts/scan-append-only-streams.ts"; + +test("manifest scan inventories append-only streams and flags candidates without claiming a violation", () => { + const result = inspectAppendOnlyManifest( + { + connector_key: "example", + streams: [ + { name: "profile", semantics: "mutable_state" }, + { + name: "events", + primary_key: ["id"], + schema: { + properties: { + account_id: {}, + country: {}, + fetched_at: {}, + id: {}, + likes_count: {}, + occurred_at: {}, + }, + }, + semantics: "append_only", + }, + ], + version: "0.1.0", + }, + "connectors/example/manifest.json", + ); + assert.deepEqual(result, [ + { + connector: "example", + fields: [ + "account_id", + "country", + "fetched_at", + "id", + "likes_count", + "occurred_at", + ], + manifest: "connectors/example/manifest.json", + primary_key: ["id"], + run_time_field_candidates: ["fetched_at"], + source_mutable_field_candidates: ["likes_count"], + stream: "events", + version: "0.1.0", + }, + ]); +}); + +test("manifest scan reads source and shipped layouts while excluding generated and dependency trees", async () => { + const directory = await mkdtemp(join(tmpdir(), "append-only-scan-")); + try { + for (const name of [ + "connectors/example", + "manifests", + "node_modules", + "dist", + ]) { + await mkdir(join(directory, name), { recursive: true }); + } + const manifest = { + connector_key: "example", + streams: [ + { name: "events", primary_key: ["id"], semantics: "append_only" }, + ], + }; + await writeFile( + join(directory, "connectors/example/manifest.json"), + JSON.stringify(manifest), + ); + await writeFile( + join(directory, "manifests/shipped.json"), + JSON.stringify(manifest), + ); + await writeFile( + join(directory, "node_modules/manifest.json"), + "invalid JSON", + ); + await writeFile(join(directory, "dist/manifest.json"), "invalid JSON"); + const result = await scanAppendOnlyStreams(directory); + assert.equal(result.manifest_count, 2); + assert.equal(result.stream_count, 2); + assert.equal(result.append_only_stream_count, 2); + assert.deepEqual( + result.streams.map((stream) => stream.stream), + ["events", "events"], + ); + } finally { + await rm(directory, { force: true, recursive: true }); + } +}); + +test("shipped apple-photos append_only emits different metadata for the same content-derived key", async () => { + const registryUrl = import.meta.resolve( + "@pdpp/polyfill-connectors/manifests", + ); + const { readPolyfillManifests } = await import(registryUrl); + const manifest = readPolyfillManifests().find( + (entry: { file: string }) => entry.file === "apple_photos.json", + )?.manifest; + assert.equal( + manifest?.streams.find( + (stream: { name: string }) => stream.name === "photos", + )?.semantics, + "append_only", + ); + const { buildPhotoRecord } = await import( + new URL("../connectors/apple_photos/parsers.js", registryUrl).href + ); + const file = { + mtimeIso: "2026-10-01T00:00:00.000Z", + path: "photo.jpg", + sizeBytes: 10, + }; + const hydration = { + blobRef: null, + contentSha256: "a".repeat(64), + hydrationError: null, + hydrationStatus: "skipped_too_large", + sizeBytes: 10, + }; + const first = buildPhotoRecord(file, "photo-a.jpg", hydration); + const renamed = buildPhotoRecord(file, "photo-b.jpg", hydration); + assert.equal(first.id, renamed.id); + assert.notDeepEqual(first, renamed); +}); diff --git a/reference-implementation/test/experimental-record-updates-http.test.ts b/reference-implementation/test/experimental-record-updates-http.test.ts new file mode 100644 index 000000000..30c8f851a --- /dev/null +++ b/reference-implementation/test/experimental-record-updates-http.test.ts @@ -0,0 +1,282 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import test from "node:test"; +import { closeDb } from "../server/db.ts"; +import { EXPERIMENTAL_PARTIAL_UPDATE_TYPE } from "../server/experimental-record-updates.ts"; +import { startServer } from "../server/index.ts"; +import { closePostgresStorage } from "../server/postgres-storage.ts"; +import { createRequestConnectorInstanceStore } from "../server/request-store-factories.ts"; +import { admitOwnerRunConnection } from "../server/stores/connector-instance-store.ts"; +import { dedicatedPostgresTestUrl } from "./helpers/dedicated-postgres-test-url.ts"; +import { withTemporaryPostgresDatabase } from "./helpers/postgres-temp-database.ts"; + +const postgresUrl = dedicatedPostgresTestUrl( + process.env.PDPP_TEST_POSTGRES_URL, +); +const password = "experimental-record-updates-test-password"; +const subject = "owner_alice"; +const csrfPattern = /name="_csrf" value="([^"]+)"/; +function cookie(response: Response, name: string): string { + const cookieValue = response.headers + .getSetCookie() + .map((value) => value.split(";")[0]) + .find((value) => value?.startsWith(`${name}=`)); + assert.ok(cookieValue); + return cookieValue; +} +async function post( + url: string, + body: unknown, + headers: Record = {}, +) { + const response = await fetch(url, { + body: JSON.stringify(body), + headers: { "Content-Type": "application/json", ...headers }, + method: "POST", + }); + const text = await response.text(); + const parsed = text.startsWith("{") + ? (JSON.parse(text) as Record) + : {}; + return { body: parsed, status: response.status }; +} + +for (const backend of ["sqlite", "postgres"] as const) { + test(`${backend}: HTTP prototype conflicts and malformed instructions have permanent replayable receipts`, { + skip: + backend === "postgres" && !postgresUrl + ? "PDPP_TEST_POSTGRES_URL unset" + : false, + }, async () => { + async function run(databaseUrl?: string) { + process.env.PDPP_EXPERIMENTAL_RECORD_UPDATES = "1"; + const server = await startServer({ + asPort: 0, + ownerAuthPassword: password, + ownerAuthSubjectId: subject, + quiet: true, + rsPort: 0, + ...(databaseUrl + ? { databaseUrl, storageBackend: "postgres" } + : { dbPath: ":memory:" }), + }); + try { + const asUrl = `http://localhost:${server.asPort}`; + const rsUrl = `http://localhost:${server.rsPort}`; + const loginPage = await fetch(`${asUrl}/owner/login`, { + redirect: "manual", + }); + const csrfCookie = cookie(loginPage, "pdpp_owner_csrf"); + const csrf = (await loginPage.text()).match(csrfPattern)?.[1]; + assert.ok(csrf); + const login = await fetch(`${asUrl}/owner/login`, { + body: new URLSearchParams({ _csrf: csrf, password, return_to: "/" }), + headers: { + "Content-Type": "application/x-www-form-urlencoded", + Cookie: csrfCookie, + }, + method: "POST", + redirect: "manual", + }); + const ownerSession = cookie(login, "pdpp_owner_session"); + const device = await post(`${asUrl}/oauth/device_authorization`, { + client_id: "pdpp_cli", + }); + const approved = await post( + `${asUrl}/device/approve`, + { subject_id: subject, user_code: device.body.user_code }, + { Cookie: ownerSession }, + ); + assert.equal(approved.status, 200); + const token = await post(`${asUrl}/oauth/token`, { + client_id: "pdpp_cli", + device_code: device.body.device_code, + grant_type: "urn:ietf:params:oauth:grant-type:device_code", + }); + assert.equal(token.status, 200); + const connectorId = `experimental-http-${backend}`; + const registration = await post( + `${asUrl}/connectors`, + { + connector_id: connectorId, + connector_key: connectorId, + display_name: "Experimental HTTP", + manifest_uri: `https://example.test/${connectorId}`, + protocol_version: "0.1.0", + streams: ["events", "state"].map((name) => ({ + name, + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + optional: { type: ["string", "null"] }, + value: { type: "string" }, + }, + required: ["id", "value"], + type: "object", + }, + selection: { fields: true, resources: true }, + semantics: name === "events" ? "append_only" : "mutable_state", + views: [ + { fields: ["id", "optional"], id: "basic", label: "Basic" }, + ], + })), + version: "1.0.0", + }, + { Cookie: ownerSession }, + ); + assert.ok( + [200, 201].includes(registration.status), + JSON.stringify(registration.body), + ); + const { connectorInstanceId } = await admitOwnerRunConnection({ + connectorId, + connectorInstanceId: null, + connectorInstanceStore: createRequestConnectorInstanceStore(), + ownerSubjectId: subject, + }); + const authorization = `Bearer ${token.body.access_token}`; + async function ingest(stream: string, records: unknown[]) { + const response = await fetch( + `${rsUrl}/v1/ingest/${stream}?connector_id=${connectorId}&connector_instance_id=${connectorInstanceId}`, + { + body: records.map((input) => JSON.stringify(input)).join("\n"), + headers: { + Authorization: authorization, + "Content-Type": "application/x-ndjson", + }, + method: "POST", + }, + ); + assert.equal(response.status, 200); + return (await response.json()) as { + records_accepted: number; + records_rejected: number; + rejections?: Array<{ receipt_id: string; code: string }>; + }; + } + assert.equal( + ( + await ingest("events", [ + { + data: { id: "a", value: "original" }, + key: "a", + type: "RECORD", + }, + ]) + ).records_accepted, + 1, + ); + const rejected = [ + { data: { id: "a", value: "changed" }, key: "a" }, + { + key: "a", + remove: [], + set: [], + type: EXPERIMENTAL_PARTIAL_UPDATE_TYPE, + }, + ]; + const first = await ingest("events", rejected); + assert.equal(first.records_rejected, 2); + assert.deepEqual( + first.rejections?.map((receipt) => receipt.code), + ["append_only_record_conflict", "invalid_partial_update"], + ); + const replay = await ingest("events", rejected); + assert.deepEqual( + replay.rejections, + first.rejections, + "permanent input has stable receipt IDs on retry", + ); + const read = await fetch( + `${rsUrl}/v1/streams/events/records/a?connector_id=${connectorId}&connector_instance_id=${connectorInstanceId}`, + { headers: { Authorization: authorization } }, + ); + assert.equal(read.status, 200); + const record = (await read.json()) as { data: { value: string } }; + assert.equal(record.data.value, "original"); + await ingest("state", [ + { + data: { id: "a", optional: "present", value: "original" }, + key: "a", + }, + ]); + const partial = await ingest("state", [ + { + key: "a", + remove: ["optional"], + set: { value: "updated" }, + type: EXPERIMENTAL_PARTIAL_UPDATE_TYPE, + }, + ]); + assert.equal(partial.records_accepted, 1); + for (const projection of ["fields=id,optional", "view=basic"]) { + const ownerRead = await fetch( + `${rsUrl}/v1/streams/state/records?connector_id=${connectorId}&connector_instance_id=${connectorInstanceId}&${projection}`, + { headers: { Authorization: authorization } }, + ); + assert.equal(ownerRead.status, 200); + const projected = (await ownerRead.json()) as { + data: Array<{ data: Record }>; + }; + assert.deepEqual( + projected.data[0]?.data, + { id: "a" }, + "resolved owner projection omits absent optional and out-of-projection stored values", + ); + } + const malformed = await ingest("state", [ + { + data: null, + key: "a", + remove: [], + set: {}, + type: EXPERIMENTAL_PARTIAL_UPDATE_TYPE, + }, + { + key: "a", + op: "nonsense", + remove: [], + set: {}, + type: EXPERIMENTAL_PARTIAL_UPDATE_TYPE, + }, + ]); + assert.equal(malformed.records_rejected, 2); + assert.deepEqual( + malformed.rejections?.map((receipt) => receipt.code), + ["invalid_partial_update", "invalid_partial_update"], + ); + } finally { + server.schedulerManager?.stop?.(); + server.abortStartupBackfill?.("test shutdown"); + server.asServer.closeAllConnections?.(); + server.rsServer.closeAllConnections?.(); + await Promise.all([ + new Promise((resolve) => + server.asServer.close(() => resolve()), + ), + new Promise((resolve) => + server.rsServer.close(() => resolve()), + ), + ]); + closeDb(); + await closePostgresStorage(); + delete process.env.PDPP_EXPERIMENTAL_RECORD_UPDATES; + } + } + if (backend === "postgres") { + await withTemporaryPostgresDatabase( + { + closeConnections: closePostgresStorage, + connectionString: postgresUrl ?? "", + databaseName: `pdpp_test_record_updates_http_${Date.now().toString(36)}`, + }, + run, + ); + } else { + await run(); + } + }); +} diff --git a/reference-implementation/test/experimental-record-updates.test.ts b/reference-implementation/test/experimental-record-updates.test.ts new file mode 100644 index 000000000..b250c5e7b --- /dev/null +++ b/reference-implementation/test/experimental-record-updates.test.ts @@ -0,0 +1,903 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import assert from "node:assert/strict"; +import test from "node:test"; +import { assertValidRecordEnvelope } from "../runtime/record-message-validator.ts"; +import { closeDb, getDb, initDb } from "../server/db.ts"; +import { EXPERIMENTAL_PARTIAL_UPDATE_TYPE } from "../server/experimental-record-updates.ts"; +import { deriveReferenceFreshness } from "../server/freshness.ts"; +import { + closePostgresStorage, + initPostgresStorage, + postgresQuery, +} from "../server/postgres-storage.ts"; +import { + classifyIngestFailure, + getRecord, + ingestRecord, + maintainRecordIndexes, + queryRecords, + setClientEventEnqueueHook, +} from "../server/records.ts"; +import { dedicatedPostgresTestUrl } from "./helpers/dedicated-postgres-test-url.ts"; +import { withTemporaryPostgresDatabase } from "./helpers/postgres-temp-database.ts"; + +const connectorId = "experimental_record_updates"; +const target = { + connectorId, + connectorInstanceId: "cin_experimental_record_updates", +}; +const schema = { + properties: { + array: { type: "array" }, + created_at: { format: "date-time", type: "string" }, + id: { type: "string" }, + nested: { type: "object" }, + numeric: { type: "integer" }, + optional: { type: ["string", "null"] }, + required: { type: "string" }, + }, + required: ["id", "required"], + type: "object", +}; +const manifest = { + streams: [ + ...["events", "state"].map((name) => ({ + consent_time_field: "created_at", + cursor_field: "created_at", + name, + primary_key: ["id"], + schema, + semantics: name === "events" ? "append_only" : "mutable_state", + })), + ...["alpha", "beta"].map((name) => ({ + name, + primary_key: ["id"], + schema: { + $id: "https://example.test/reused-schema-id", + properties: { id: { type: "string" }, [name]: { type: "string" } }, + required: ["id", name], + type: "object", + }, + semantics: "mutable_state", + })), + { + name: "own-properties", + primary_key: ["id"], + schema: { + properties: { + constructor: {}, + id: { type: "string" }, + } as Record, + required: ["id", "constructor"], + type: "object", + }, + semantics: "mutable_state", + }, + { + name: "finite-numbers", + primary_key: ["id"], + schema: { + properties: { + array: { type: "array" }, + id: { type: "string" }, + loose: { not: { type: "null" } }, + nested: { type: "object" }, + value: { type: "number" }, + }, + required: ["id", "value"], + type: "object", + }, + semantics: "mutable_state", + }, + { + name: "indexed", + primary_key: ["id"], + query: { search: { lexical_fields: ["required", "optional"] } }, + schema, + semantics: "mutable_state", + }, + ], +}; +function setup() { + process.env.PDPP_EXPERIMENTAL_RECORD_UPDATES = "1"; + initDb(":memory:"); + getDb() + .prepare("INSERT INTO connectors(connector_id, manifest) VALUES(?, ?)") + .run(connectorId, JSON.stringify(manifest)); +} +function snapshot( + stream = "events", + data = { id: "a", nested: { a: 1, b: 2 }, required: "value" }, +) { + return { data, emitted_at: "2026-10-01T00:00:00Z", key: "a", stream }; +} +function rows() { + return getDb() + .prepare("SELECT record_json, emitted_at, version FROM records") + .all(); +} + +const postgresUrl = dedicatedPostgresTestUrl( + process.env.PDPP_TEST_POSTGRES_URL, +); +for (const backend of ["sqlite", "postgres"] as const) { + test(`${backend}: experimental #398 snapshots, duplicates and partial instructions`, { + skip: + backend === "postgres" && !postgresUrl + ? "PDPP_TEST_POSTGRES_URL unset" + : false, + }, async (t) => { + async function run(databaseUrl?: string) { + process.env.PDPP_EXPERIMENTAL_RECORD_UPDATES = "1"; + initDb(":memory:"); + if (databaseUrl) { + await initPostgresStorage({ backend: "postgres", databaseUrl }); + await postgresQuery( + "INSERT INTO connectors(connector_id, manifest, created_at) VALUES($1, $2::jsonb, $3)", + [connectorId, JSON.stringify(manifest), "2026-10-01T00:00:00Z"], + ); + } else { + getDb() + .prepare( + "INSERT INTO connectors(connector_id, manifest) VALUES(?, ?)", + ) + .run(connectorId, JSON.stringify(manifest)); + } + const write = ( + record: Parameters[1], + instance = target, + ) => ingestRecord(instance, record, { deferIndexes: true }); + const grant = { + streams: [ + { + fields: ["id", "required", "optional", "nested", "array"], + name: "state", + }, + { + fields: ["id", "required", "optional", "nested", "array"], + name: "events", + }, + ], + }; + const read = async (stream: string, fields?: string[]) => { + const record = await getRecord( + target, + stream, + "a", + fields ? { streams: [{ fields, name: stream }] } : grant, + manifest, + ); + assert.ok(record.data); + return { ...record, data: record.data }; + }; + const instruction = ( + set: unknown, + remove: unknown = [], + stream = "state", + ) => ({ + key: "a", + remove, + set, + stream, + type: EXPERIMENTAL_PARTIAL_UPDATE_TYPE, + }); + const databaseState = async () => { + const tables = ["records", "record_changes", "version_counter"]; + return await Promise.all( + tables.map(async (table) => { + const sql = `SELECT * FROM ${table} ORDER BY stream, ${table === "version_counter" ? "max_version" : "version"}`; + return databaseUrl + ? (await postgresQuery(sql)).rows + : getDb().prepare(sql).all(); + }), + ); + }; + try { + await t.test( + "JSON equality and append_only schema/identity validation before duplicate conflict", + async () => { + let notifications = 0; + setClientEventEnqueueHook(() => { + notifications += 1; + }); + await write({ ...snapshot(), type: "RECORD" }); + const original = await databaseState(); + await write({ + ...snapshot(), + data: JSON.parse( + '{ "nested": {"b":2.0,"a":1e0}, "required":"value", "id":"a" }', + ), + emitted_at: "2026-10-02T00:00:00Z", + }); + assert.deepEqual( + await databaseState(), + original, + "equal JSON retry changes no storage", + ); + await assert.rejects( + write({ + ...snapshot(), + data: { id: "wrong", required: "value" }, + }), + { + code: "invalid_record_identity", + }, + ); + await assert.rejects(write({ ...snapshot(), key: [] }), { + code: "invalid_record_identity", + }); + await assert.rejects( + write({ ...snapshot(), data: { required: "value" } }), + { + code: "invalid_record_identity", + }, + ); + await assert.rejects(write({ ...snapshot(), data: { id: "a" } }), { + code: "invalid_record_schema", + }); + await assert.rejects( + write({ ...snapshot(), data: { id: "a", required: null } }), + { + code: "invalid_record_schema", + }, + ); + await assert.rejects( + write(instruction({ required: "no" }, [], "events")), + { + code: "invalid_partial_update", + }, + ); + const conflict = await write({ + ...snapshot(), + data: { id: "a", required: "different" }, + }).catch((error) => error); + assert.deepEqual(classifyIngestFailure(conflict), { + code: "append_only_record_conflict", + message: conflict.message, + retryable: false, + }); + assert.deepEqual( + await databaseState(), + original, + "rejected writes leave history and versions intact", + ); + assert.equal( + notifications, + 1, + "equal retry and changed conflict emit no client event", + ); + setClientEventEnqueueHook(null); + await write({ + data: { array: [1, 2], id: "array", required: "value" }, + key: "array", + stream: "events", + }); + await assert.rejects( + write({ + data: { array: [2, 1], id: "array", required: "value" }, + key: "array", + stream: "events", + }), + { code: "append_only_record_conflict" }, + ); + }, + ); + await t.test( + "beginning and retries; pruned append_only history anchor stays unchanged", + async () => { + const beginning = await queryRecords( + target, + "events", + grant, + { changes_since: "beginning" }, + manifest, + ); + assert.equal(beginning.data.length, 2); + const cursor = beginning.next_changes_since; + if (cursor) { + const afterRetry = await queryRecords( + target, + "events", + grant, + { changes_since: cursor }, + manifest, + ); + assert.equal(afterRetry.data.length, 0); + } + if (databaseUrl) { + await postgresQuery( + "DELETE FROM record_changes WHERE stream = $1 AND record_key = 'a'", + ["events"], + ); + } else { + getDb() + .prepare( + "DELETE FROM record_changes WHERE stream = ? AND record_key = 'a'", + ) + .run("events"); + } + const pruned = await databaseState(); + assert.equal( + (await write(snapshot())).changed, + false, + "pruned append_only anchor stays no-op", + ); + assert.deepEqual(await databaseState(), pruned); + }, + ); + await t.test( + "observed residual: beginning after anchor pruning expires although current append_only record survives", + async () => { + assert.equal((await read("events")).data.required, "value"); + await assert.rejects( + queryRecords( + target, + "events", + grant, + { changes_since: "beginning" }, + manifest, + ), + { + code: "cursor_expired", + }, + ); + }, + ); + await t.test( + "instance scoping, delete and post-delete reinsert", + async () => { + const otherTarget = { + ...target, + connectorInstanceId: "cin_other_experimental_record_updates", + }; + await write(snapshot(), otherTarget); + const beforeDelete = await queryRecords( + otherTarget, + "events", + grant, + { changes_since: "beginning" }, + manifest, + ); + assert.ok(beforeDelete.next_changes_since); + await write( + { key: "a", op: "delete", stream: "events" }, + otherTarget, + ); + const tombstones = await queryRecords( + otherTarget, + "events", + grant, + { changes_since: beforeDelete.next_changes_since }, + manifest, + ); + assert.equal(tombstones.data.length, 1); + assert.equal(tombstones.data[0]?.deleted, true); + assert.equal(tombstones.data[0]?.id, "a"); + assert.equal( + tombstones.data[0]?.data, + undefined, + "delete is served as a tombstone, not an instruction or snapshot", + ); + await write({ key: "a", op: "delete", stream: "events" }); + await write({ + ...snapshot(), + data: { id: "a", required: "reinserted" }, + }); + assert.equal((await read("events")).data.required, "reinserted"); + }, + ); + await t.test( + "partial set/null/remove preserve unnamed values and projected reads", + async () => { + await write({ + ...snapshot("state"), + data: { + array: [1, 2], + created_at: "2026-10-01T00:00:00Z", + id: "a", + nested: { one: 1 }, + optional: "present", + required: "value", + }, + }); + await write(instruction({ optional: null })); + assert.equal((await read("state")).data.optional, null); + await write(instruction({}, ["optional"])); + const absent = await read("state"); + assert.equal(Object.hasOwn(absent.data, "optional"), false); + assert.deepEqual(absent.data.nested, { one: 1 }); + const projected = await read("state", ["id", "optional"]); + assert.deepEqual( + projected.data, + { id: "a", required: "value" }, + "owner effective projection includes required fields", + ); + const clientRecord = await getRecord( + target, + "state", + "a", + { + streams: [ + { + fields: ["id", "optional"], + instance_ids: [target.connectorInstanceId], + name: "state", + }, + ], + }, + manifest, + ); + assert.deepEqual( + clientRecord.data, + { id: "a" }, + "client projection uses frozen grant fields; omission outside it reveals nothing", + ); + }, + ); + await t.test( + "required fields, nullability, identity and malformed instructions roll back atomically", + async () => { + const beforeInvalid = await databaseState(); + for (const invalid of [ + instruction({}, ["required"]), + instruction({ required: null }), + instruction({ id: "changed" }), + instruction({}, ["id"]), + instruction({ optional: "x" }, ["optional"]), + instruction([], []), + instruction({}, [42]), + { ...instruction({}), data: null }, + { ...instruction({}), op: "upsert" as const }, + ]) { + await assert.rejects(write(invalid)); + } + assert.deepEqual( + await databaseState(), + beforeInvalid, + "invalid resulting snapshots roll back atomically", + ); + }, + ); + await t.test( + "concurrent instructions preserve both writers and array order is significant", + async () => { + await Promise.all([ + write(instruction({ optional: "concurrent" })), + write(instruction({ nested: { two: 2 } })), + ]); + const concurrent = await read("state"); + assert.equal(concurrent.data.optional, "concurrent"); + assert.deepEqual(concurrent.data.nested, { two: 2 }); + assert.equal(concurrent.data.required, "value"); + await write(instruction({ array: [2, 1] })); + assert.deepEqual( + (await read("state")).data.array, + [2, 1], + "array order is meaningful", + ); + }, + ); + await t.test( + "time constraints and changes_since use complete resulting snapshots", + async () => { + const timedGrant = { + streams: [ + { + fields: [ + "id", + "required", + "optional", + "nested", + "array", + "created_at", + ], + name: "state", + time_constraint: { + field: "created_at", + since: "2026-10-02T00:00:00Z", + }, + }, + ], + }; + assert.equal( + (await queryRecords(target, "state", timedGrant, {}, manifest)) + .data.length, + 0, + ); + await write(instruction({ created_at: "2026-10-03T00:00:00Z" })); + const timed = await queryRecords( + target, + "state", + timedGrant, + {}, + manifest, + ); + assert.equal(timed.data.length, 1); + assert.equal( + timed.data[0]?.data?.optional, + "concurrent", + "unobserved named-independent field survives instruction", + ); + const changes = await queryRecords( + target, + "state", + timedGrant, + { changes_since: "beginning" }, + manifest, + ); + assert.equal(changes.data.length, 1); + assert.equal(changes.data[0]?.data?.required, "value"); + assert.deepEqual(changes.data[0]?.data?.nested, { two: 2 }); + assert.equal( + Object.hasOwn(changes.data[0]?.data ?? {}, "type"), + false, + ); + }, + ); + await t.test( + "new collection freshness does not establish reobservation of preserved unnamed fields", + async () => { + const previouslyStored = (await read("state")).data.optional; + const capturedAt = "2026-10-04T00:00:00Z"; + await write({ + ...instruction({ required: "freshly collected" }), + emitted_at: capturedAt, + }); + assert.equal( + (await read("state")).data.optional, + previouslyStored, + "instruction leaves unobserved optional field intact", + ); + const freshness = deriveReferenceFreshness({ + lastAttemptedAt: capturedAt, + lastSuccessfulRunAt: capturedAt, + maximumStalenessSeconds: 3600, + now: "2026-10-04T00:05:00Z", + }); + assert.deepEqual(freshness, { + captured_at: "2026-10-04T00:00:00.000Z", + last_attempted_at: "2026-10-04T00:00:00.000Z", + status: "current", + }); + }, + ); + await t.test( + "snapshot replacement and complete version history", + async () => { + const clientCopyBefore = { ...(await read("state")).data }; + assert.equal(Object.hasOwn(clientCopyBefore, "optional"), true); + await write({ + ...snapshot("state"), + data: { id: "a", required: "replacement" }, + }); + assert.deepEqual( + (await read("state")).data, + { id: "a", required: "replacement" }, + "snapshots replace omitted values", + ); + const replacementCopy = { ...(await read("state")).data }; + assert.equal( + Object.hasOwn(replacementCopy, "optional"), + false, + "a client replacing its copy within the same projection removes absent values", + ); + const history = databaseUrl + ? ( + await postgresQuery( + "SELECT record_json FROM record_changes WHERE stream = 'state' AND NOT deleted", + ) + ).rows + : getDb() + .prepare( + "SELECT record_json FROM record_changes WHERE stream = 'state' AND NOT deleted", + ) + .all(); + for (const row of history as Array<{ record_json: unknown }>) { + const data = + typeof row.record_json === "string" + ? JSON.parse(row.record_json) + : row.record_json; + assert.equal(Object.hasOwn(data, "type"), false); + assert.equal(typeof data.required, "string"); + assert.equal(data.id, "a"); + } + }, + ); + await t.test( + "schema IDs are scoped to schema contents and do not collide across streams", + async () => { + await write({ + data: { alpha: "first schema", id: "a" }, + key: "a", + stream: "alpha", + }); + await write({ + data: { beta: "second schema", id: "a" }, + key: "a", + stream: "beta", + }); + await assert.rejects( + write({ + data: { alpha: "wrong schema", id: "a" }, + key: "a", + stream: "beta", + }), + { + code: "invalid_record_schema", + }, + ); + }, + ); + await t.test( + "removing required constructor cannot be satisfied by inherited properties and is atomic", + async () => { + await write({ + data: { constructor: "own value", id: "a" }, + key: "a", + stream: "own-properties", + }); + const before = await databaseState(); + await assert.rejects( + write(instruction({}, ["constructor"], "own-properties")), + { + code: "invalid_record_schema", + }, + ); + assert.deepEqual(await databaseState(), before); + }, + ); + await t.test( + "derived lexical index receives complete snapshots after partial instructions", + async () => { + const full = { + data: { + id: "a", + optional: "old search text", + required: "retained search text", + }, + key: "a", + stream: "indexed", + }; + const initial = await write(full); + assert.ok(initial.version); + const indexOptions = { + attemptContext: { + streams: { + indexed: { + lexicalFields: ["required", "optional"], + primaryKey: ["id"], + semanticFields: [], + }, + }, + }, + }; + await maintainRecordIndexes( + target, + full, + initial.version, + indexOptions, + ); + const partial = instruction( + { optional: "updated search text" }, + [], + "indexed", + ); + const updated = await write(partial); + assert.ok(updated.version); + await maintainRecordIndexes( + target, + partial, + updated.version, + indexOptions, + ); + const sql = + "SELECT field, text FROM lexical_search_index WHERE connector_instance_id = ? AND stream = 'indexed' ORDER BY field"; + const indexed = databaseUrl + ? ( + await postgresQuery( + sql + .replace("?", "$1") + .replace("field, text", "field, value AS text"), + [target.connectorInstanceId], + ) + ).rows + : getDb().prepare(sql).all(target.connectorInstanceId); + assert.deepEqual( + indexed.map((row) => ({ ...(row as object) })), + [ + { field: "optional", text: "updated search text" }, + { field: "required", text: "retained search text" }, + ], + ); + const removal = instruction({}, ["optional"], "indexed"); + const removed = await write(removal); + assert.ok(removed.version); + await maintainRecordIndexes( + target, + removal, + removed.version, + indexOptions, + ); + const remaining = databaseUrl + ? ( + await postgresQuery( + sql + .replace("?", "$1") + .replace("field, text", "field, value AS text"), + [target.connectorInstanceId], + ) + ).rows + : getDb().prepare(sql).all(target.connectorInstanceId); + assert.deepEqual( + remaining.map((row) => ({ ...(row as object) })), + [{ field: "required", text: "retained search text" }], + "named field disappears from index while unnamed full-snapshot field survives", + ); + }, + ); + await t.test( + "observed precision limit: distinct unsafe JSON integers collapse before append_only equality", + async () => { + const firstJson = + '{"stream":"events","key":"precision","data":{"id":"precision","required":"value","numeric":9007199254740992}}'; + const secondJson = + '{"stream":"events","key":"precision","data":{"id":"precision","required":"value","numeric":9007199254740993}}'; + assert.notEqual(firstJson, secondJson); + const first = JSON.parse(firstJson); + const second = JSON.parse(secondJson); + assert.equal( + first.data.numeric, + second.data.numeric, + "native JSON.parse already discarded the distinct integer value", + ); + await write(first); + const before = await databaseState(); + assert.equal( + (await write(second)).changed, + false, + "prototype compares parsed JSON values and cannot detect this raw producer conflict", + ); + assert.deepEqual(await databaseState(), before); + }, + ); + await t.test( + "overflowing JSON numbers cannot become stored null and reject atomically", + async () => { + await write({ + data: { id: "a", value: 1 }, + key: "a", + stream: "finite-numbers", + }); + const before = await databaseState(); + for (const value of ["1e400", "-1e400"]) { + for (const rawSet of [ + `{"value":${value}}`, + `{"loose":${value}}`, + `{"nested":{"value":${value}}}`, + `{"array":[${value}]}`, + ]) { + const set = JSON.parse(rawSet); + await assert.rejects( + write({ + data: { id: "a", value: 1, ...set }, + key: "a", + stream: "finite-numbers", + }), + { code: "invalid_record_schema" }, + ); + await assert.rejects( + write(instruction(set, [], "finite-numbers")), + { code: "invalid_record_schema" }, + ); + } + } + assert.deepEqual(await databaseState(), before); + }, + ); + await t.test( + "flag off retains RECORD legacy rewrites and rejects prototype instructions", + async () => { + delete process.env.PDPP_EXPERIMENTAL_RECORD_UPDATES; + await write({ + ...snapshot(), + data: { id: "a", required: "legacy rewrite" }, + type: "RECORD", + }); + assert.equal( + (await read("events")).data.required, + "legacy rewrite", + ); + const extendedSnapshot = { + ...snapshot(), + data: { id: "a", required: "legacy snapshot with extensions" }, + type: "RECORD", + set: { required: "must remain an ignored extension" }, + remove: ["required"], + }; + assert.doesNotThrow(() => + assertValidRecordEnvelope(extendedSnapshot), + ); + assert.equal((await write(extendedSnapshot)).accepted, true); + assert.equal( + (await read("events")).data.required, + "legacy snapshot with extensions", + "flag-off RECORD extensions must not be treated as partial instructions", + ); + await assert.rejects(write(instruction({ optional: "disabled" })), { + code: "invalid_partial_update", + }); + }, + ); + } finally { + setClientEventEnqueueHook(null); + closeDb(); + await closePostgresStorage(); + delete process.env.PDPP_EXPERIMENTAL_RECORD_UPDATES; + } + } + if (backend === "postgres") { + await withTemporaryPostgresDatabase( + { + closeConnections: closePostgresStorage, + connectionString: postgresUrl ?? "", + databaseName: `pdpp_test_record_updates_${Date.now().toString(36)}`, + }, + run, + ); + } else { + await run(); + } + }); +} +test("experimental append_only rejects changed duplicate without rewriting", async () => { + setup(); + try { + await ingestRecord(target, snapshot(), { deferIndexes: true }); + const before = rows(); + await assert.rejects( + ingestRecord( + target, + snapshot("events", { + id: "a", + nested: { a: 1, b: 2 }, + required: "changed", + }), + { + deferIndexes: true, + }, + ), + { code: "append_only_record_conflict" }, + ); + assert.deepEqual(rows(), before); + } finally { + closeDb(); + delete process.env.PDPP_EXPERIMENTAL_RECORD_UPDATES; + } +}); +test("experimental SQLite JSON equality ignores nested member order", async () => { + setup(); + try { + await ingestRecord(target, snapshot(), { deferIndexes: true }); + const before = rows(); + const result = await ingestRecord( + target, + { + ...snapshot(), + data: { + id: "a", + nested: JSON.parse('{"b":2,"a":1}'), + required: "value", + }, + emitted_at: "2026-10-02T00:00:00Z", + }, + { deferIndexes: true }, + ); + assert.equal(result.changed, false); + assert.deepEqual(rows(), before); + } finally { + closeDb(); + delete process.env.PDPP_EXPERIMENTAL_RECORD_UPDATES; + } +}); From ac3f742b2556ddd5ca216203e2889b4512921c6a Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Sun, 11 Oct 2026 02:41:53 -0500 Subject: [PATCH 2/3] fix(records): repair sparse reads and pruned sync bootstrap Sparse responses used the grant-wide replacement rule, and beginning sessions depended on history that may already have been pruned. Separate response fields from full-grant change eligibility and retain authorized snapshots for bounded bootstrap pages and subsequent sync baselines. Preserve tombstone envelopes when the final response projection runs. The history test accepted an empty result because every assertion was inside its row loop. Compare every expected version and full snapshot, require the retry cursor, and verify both oracles reject injected faults. Gate SQLite duplicate handling before parsing when the flag is disabled. Refs: PDP-Connect/pdpp#398, PDP-Connect/pdpp#383 Assisted-by: AI Signed-off-by: Tim Nunamaker --- docs/261010-ri-absent-fields-prototype.md | 34 +- .../operations/read-projection.ts | 18 + .../operations/rs-records-detail/index.ts | 20 +- .../operations/rs-records-list/index.ts | 30 +- .../server/experimental-record-bootstrap.ts | 255 +++++ .../server/postgres-records.ts | 74 +- .../server/record-expand-helpers.ts | 18 +- reference-implementation/server/records.ts | 77 +- .../server/routes/rs-read.ts | 3 + .../experimental-record-updates-http.test.ts | 13 +- .../test/experimental-record-updates.test.ts | 912 ++++++++++++++++-- 11 files changed, 1360 insertions(+), 94 deletions(-) create mode 100644 reference-implementation/server/experimental-record-bootstrap.ts diff --git a/docs/261010-ri-absent-fields-prototype.md b/docs/261010-ri-absent-fields-prototype.md index 23977dce0..47386a109 100644 --- a/docs/261010-ri-absent-fields-prototype.md +++ b/docs/261010-ri-absent-fields-prototype.md @@ -2,6 +2,8 @@ Prototype of [PDPP draft #398](https://github.com/PDP-Connect/pdpp/pull/398). Not for merge until the specification is final. Baseline: `4e69b5dff8a52e3547444a6ac57d7d12dd6940b8`. +Normative addenda: local PDPP commit `6bc2f35c7a` (`fix/398-dot-sparse-absence`) distinguishes response fields from incremental eligibility; local commit `e8d4141d88` (`fix/383-sync-bootstrap`) requires a new beginning session after pruning. Those branches are read-only inputs to this prototype. + Behavior changes require `PDPP_EXPERIMENTAL_RECORD_UPDATES=1`; the default is off. The experimental profile instruction is `{type:"PDPP_EXPERIMENTAL_PARTIAL_UPDATE_V1", key, set:{...}, remove:[...]}`. The ingest route supplies the stream. Direct storage calls also supply `stream`. `set` replaces named top-level values; `remove` deletes named top-level fields. It cannot include snapshot `data` or `op`. Core defines no such syntax. ## Requirement evidence @@ -10,13 +12,13 @@ The named subtests below run on both SQLite and actual PostgreSQL in `reference- | Requirement | Verdict | Test evidence and result | | --- | --- | --- | -| Omitted top-level field inside the effective projection has no stored value in that version | Held | `partial set/null/remove preserve unnamed values and projected reads`: removed optional field is absent, not null. | -| Omission outside the projection makes no assertion | Held | Same subtest: a client grant with frozen fields omits the stored required value. HTTP tests exercise owner `fields` and resolved `view` projections. | +| Omitted top-level field inside the effective response projection has no stored value in that version | Held | `partial set/null/remove preserve unnamed values and projected reads`: removed optional field is absent, not null. | +| Omission outside the projection makes no assertion | Held | `sparse resolved-client response preserves cached fields outside its projection`: narrower accepted fields leave cached unrequested values intact and remove an absent requested value. HTTP tests exercise owner `fields` and resolved `view` projections. | | Null is distinct from absence and allowed only by the schema | Held | Same subtest preserves explicit nullable null; `required fields, nullability, identity and malformed instructions roll back atomically` rejects nonnullable null. | | Required fields and record identity remain mandatory | Held | Atomic rollback subtest rejects required/identity removal and identity mutation. `removing required constructor cannot be satisfied by inherited properties and is atomic` verifies own-property validation. | -| Returned snapshots replace a client's copy within the same projection | Held for the wire representation | `snapshot replacement and complete version history`: an omitted formerly present field disappears; a test-side replacement copy also loses it. There is no production client cache in this RI test. | +| Returned snapshots replace a client's copy within the same projection | Held for the wire representation | Sparse resolved-client cache subtest applies replacement only inside the effective response fields, including granted schema-required fields. `snapshot replacement and complete version history` separately verifies exact stored snapshots. The cache is test-side; no production client cache is claimed. | | Partial updates are distinct, name set/remove fields and preserve unnamed fields | Held | `partial set/null/remove preserve unnamed values and projected reads` and `concurrent instructions preserve both writers and array order is significant`. | -| Applying an instruction is atomic and yields a complete schema/identity-valid snapshot | Held | Atomic rollback and concurrency subtests compare current records, history and version counters; schema-ID, inherited-property and `overflowing JSON numbers cannot become stored null and reject atomically` regressions are covered. | +| Applying an instruction is atomic and yields a complete schema/identity-valid snapshot | Held | Exact history and atomic rollback/concurrency subtests compare current records, history and version counters; schema-ID, inherited-property and `overflowing JSON numbers cannot become stored null and reject atomically` regressions are covered. | | Reads, time constraints, history and incremental results consume complete snapshots | Held | `time constraints and changes_since use complete resulting snapshots` and `snapshot replacement and complete version history`; instructions are never served as records. | | A partial instruction cannot modify an existing append-only record | Held | `JSON equality and append_only schema/identity validation before duplicate conflict` rejects it before storage. | | Core specifies no partial-update syntax | Underspecified by design | `flag off retains RECORD legacy rewrites and rejects prototype instructions` verifies the boundary; marker, no-op instructions, missing-record rejection and overlapping set/remove rejection are profile choices. | @@ -25,19 +27,29 @@ The named subtests below run on both SQLite and actual PostgreSQL in `reference- | JSON-value equality ignores object order and whitespace; arrays retain order | Held | JSON equality subtest uses nested reordering and rejects reordered arrays. | | Numbers compare by numeric value regardless of spelling | Underspecified precision | JSON equality subtest covers `2.0`/`1e0`. `observed precision limit: distinct unsafe JSON integers collapse before append_only equality` shows that exact numeric comparison beyond JavaScript binary64 precision is not established. | | Different append-only data reports a producer conflict and should be visible to the owner | Held for the ingest interface | `: HTTP prototype conflicts and malformed instructions have permanent replayable receipts` in `experimental-record-updates-http.test.ts`: permanent conflict code, stable receipts and unchanged owner read. UI presentation of those receipts is unverified. | -| A duplicate creates no incremental result; the existing record appears from beginning | Held with retained history; wrong after pruning | Beginning/retry subtest returns the original record with no duplicate result. `observed residual: beginning after anchor pruning expires although current append_only record survives` instead returns `cursor_expired` on both backends. | -| Explicit deletion remains allowed and produces tombstones | Held | Instance/delete/reinsert subtest asserts a tombstone with no `data` snapshot, then permits reinsertion. Owner deletion uses the existing delete path; this test drives ingest deletion. | +| A duplicate creates no incremental result; the existing record appears from beginning | Held within the bounded prototype session | `beginning after anchor pruning enumerates current append_only state without a duplicate version` replaces the earlier expiry counterexample. The retry subtest requires its saved cursor and explicitly retries before checking zero results. | +| New beginning sessions enumerate authorized h1 state after history pruning | Held within bounded sessions | `bootstrap pages preserve authorized h1 state after writes and history pruning` verifies frozen ordinary edits, exclusion of later deletions, resource/time visibility, no tombstones, terminal-only sync tokens, context binding and normal issued-token expiry. | +| Retained sync pages sort records and tombstones by canonical key and bind direction | Held | `retained sync pages order records and tombstones by canonical key and bind direction` verifies ascending/descending results, mixed tombstones/snapshots, and rejection of a changed page direction. | +| Client incremental eligibility uses the full frozen grant, separately from sparse response fields | Held | `sparse client changes_since eligibility uses the full frozen grant projection`: an unrequested granted-field change returns a sparse delta. | +| Required fields remain included without widening frozen authorization | Underspecified after declaration changes | Sparse client/owner tests include required fields; existing frozen-grant test excludes a required field outside the grant. The local sparse fix does not say which schema controls this case; the prototype preserves frozen authorization. | +| Explicit deletion remains allowed and produces tombstones | Held | `client list preserves tombstone envelopes with full and sparse responses` checks the served envelope; `post-pruning bootstrap suppresses ungranted changes and retains later deletion baselines` verifies rolling sync baselines. Instance/delete/reinsert subtest permits reinsertion. Owner deletion uses the existing delete path; this test drives ingest deletion. | | Collection freshness does not establish reobservation of every returned field | Held as a counterexample | `new collection freshness does not establish reobservation of preserved unnamed fields`: a later partial collection produces current freshness while an unnamed value survives from an earlier write. Freshness derivation is tested directly, not via a collection scheduler. | `derived lexical index receives complete snapshots after partial instructions` additionally verifies retained unnamed index fields and deletion of a named indexed field. `schema IDs are scoped to schema contents and do not collide across streams` covers legal schemas with reused `$id` values. +`retained bootstrap state binds tokens and pages to context and storage lifecycle` verifies grant-context binding, token forgery rejection, five-minute expiry and storage lifecycle invalidation. `retained bootstrap capacity rejects complete captures and counts terminal baselines` covers record/byte limits, aggregate session capacity and intact previously issued state after rejection. + ## Backend findings The baseline SQLite no-op check compares serialized JSON. PostgreSQL compares `jsonb`; reordered object members therefore differ on SQLite but not PostgreSQL. The flag-on prototype uses JSON-value equality for append-only retries on both and for SQLite mutable-state retries. Both reject changed append-only data before allocating a version. Mutable-state instructions apply under the existing writer transaction/lock and are indexed from the resulting complete snapshot with a version guard. Baseline reproductions failed with a missing append-only rejection and a SQLite reordered-object rewrite. Both pass after the patch. Independent review also reproduced a shared AJV registry collision for reused schema IDs and acceptance of removed required `constructor` through inheritance. The prototype isolates schema registration and requires own properties; both regressions now pass on both backends. Default-off tests also preserve legacy RECORD envelopes with ignored `set`/`remove` extensions and reject the explicit experimental marker. -The retained-history requirement is not a blanket pass. The existing RI repairs missing append-only history anchors by making an equal retry a changed write. That violates #398's no-new-version requirement. This prototype prevents that repair; the current record remains readable, but a new beginning session expires. Resolving the feed behavior requires the later change-feed step; changing the draft is not implied by this finding. +The original beginning test correctly exposed an expiry after anchor pruning, but that is now a repaired prototype defect under `e8d4141d88`. A beginning session captures authorized current state at h1 without historical anchors or record writes. Saved pages retain h1 values across later ordinary edits and pruning; a later deletion suppresses that record on subsequent pages, as Section 8 requires. Follow-up sync retains an h0 baseline so unchanged ungranted fields do not trigger a delta and a later deletion can produce a tombstone. The implementation does not allocate record versions or restore history for duplicate retries. + +The original complete-history oracle was wrong: it iterated only returned rows and accepted an empty result. The replacement records each changed-write version and the exact complete snapshot after initial/set-null/remove/replacement writes, then compares count, order and all payload values. Fault tests reject empty, missing, reordered and corrupted history. Separate test-copy faults delete actual history before its query and remove the saved cursor; both now fail on both backends. + +Frozen authorization remains a specification question. The local sparse-response text says schema-required fields remain included, but does not say whether a later declaration may add an ungranted required field to a frozen client grant. The prototype includes required fields only within frozen authorization; it does not disclose a newly required ungranted field. Numeric precision is a second scope limit. `operations/rs-records-ingest/index.ts` parses NDJSON with native `JSON.parse`, so distinct JSON integers `9007199254740992` and `9007199254740993` both become `9007199254740992`. An ordinary `type: "integer"` schema accepts the parsed value; neither backend can recover the original digits after storage serialization. No global safe-integer or I-JSON restriction was found in the installed contract or the local Core draft inspected. The prototype compares parsed JavaScript numeric values, not arbitrary-precision decimals. #398 does not specify a numeric precision bound; the large-integer test records this limit without rewriting the ingest parser. @@ -58,13 +70,13 @@ These inventories overlap; the counts are not additive. Pinned Chase balances in ## Validation -The combined three-file run passes 41 tests with zero failures or skips, including actual SQLite and PostgreSQL the numeric precision observation and overflow rejection. Typecheck and scoped checks pass for the five new TypeScript files without warnings. The existing PostgreSQL template registry includes both new application-behavior test files; its three inventory checks pass. Repository-wide `check` fails on the untouched baseline as well; shared files are not reformatted wholesale. +Follow-up targeted verification: 56 tests passed, zero failed or skipped, using SQLite and the actual PostgreSQL disposable-database oracle. This includes storage, HTTP and connector-scan tests. Flag-off helper/operation/boundary and PostgreSQL registry checks passed 105 tests without skips. Typecheck and checks on the new helper and prototype tests passed. The final full RI flag-off suite completed 1,124 file outputs: 11,269 assertions, 10,875 passed, zero failed, and 394 declared skips; runner exit0, memory-default concurrency4. This run includes the final ordering and later-page deletion repairs. PostgreSQL-dependent full-suite tests are declared skips; the separate targeted run uses the actual PostgreSQL oracle. Repository-wide `check` fails on the untouched baseline; shared files are not reformatted wholesale. -The full RI suite completed with the ambient flag off: 1,124 file outputs, 11,259 assertions, 10,865 passed, zero failed and 394 declared skips; runner exit 0. It used `memory-default` with concurrency four. PostgreSQL-dependent tests in this full run are declared skips; the separate 41-test run used the actual PostgreSQL oracle without skips. The overflow guard was added after the prototype files had completed in the full run; the final focused run covers that flag-on-only change and both flag-off storage cases. Final typecheck and scoped formatting checks pass. +Before fixes, expanded tests against reviewed production failed on sparse required-field projection, full-grant eligibility, beginning after pruning and disabled SQLite parsing. The original history-deletion and missing-cursor faults still passed all 72 tests. The corrected test copies reject both faults on both backends; four failing subtests plus their parents are expected fault-test results. Independent review found and reproduced tombstone-envelope loss and missing post-bootstrap baselines; their regressions pass after repair. These receipts are preserved in the local findings report. -Earlier full attempts were interrupted by console setup, a T3 restart and a stalled integration file. The stalled `pdpp.test.ts` passed all 118 tests in isolation on both the current revision and untouched baseline. The final full run also completed that file. Its success does not establish the earlier stall cause. The new test files needed template-registry entries; those two entries are included and all three inventory checks pass. +Prototype sync state is process-local and expires after five minutes or a storage lifecycle change. Limits are 32 retained sessions, 10,000 records per captured state and 32 MiB aggregate serialized state. Capacity errors reject the whole capture; these are prototype limits, not Core limits or a claim of conformance for arbitrarily large streams. Numeric precision remains bounded by native JSON parsing. Live-account connector replay, cross-process concurrent writers, semantic embedding updates, a production client cache, and delete-then-reinsert between bootstrap pages remain unverified. -Live-account connector replay, cross-process concurrent writers, semantic embedding updates and a production client cache are not verified. Independent review also notes that existing SQLite rows are JSON-parsed before the disabled append-only helper returns; default-off performance overhead is unmeasured. PostgreSQL tests use disposable databases and the existing JSONB record oracle; they do not require the optional vector extension. +Preexisting incremental paths remain outside the new retained-state path: PostgreSQL historical incremental results are unpaginated, and SQLite historical incremental pages issue a sync token before the terminal page. New retained-state sessions use stable pagination and terminal-only sync tokens. Broader #383 token opacity and client tombstone-shape conformance are not claimed: the RI still exposes readable base64 cursor positions/horizons and uses its existing tombstone envelope. Those are subsequent feed-lane work, not a contradiction resolved by the two local spec fixes. The default-off SQLite duplicate path now avoids parsing current data; functional legacy RECORD extension and disabled-profile boundaries remain tested. PostgreSQL tests use disposable databases and the existing JSONB oracle without optional vector extensions. ## Pinned implementation audit diff --git a/reference-implementation/operations/read-projection.ts b/reference-implementation/operations/read-projection.ts index e658c02e0..40c8a929c 100644 --- a/reference-implementation/operations/read-projection.ts +++ b/reference-implementation/operations/read-projection.ts @@ -11,6 +11,24 @@ export function normalizeProjectionFields(value: unknown): string[] | null { return fields.length > 0 ? [...new Set(fields)] : null; } +/** Sparse responses retain required fields without widening a frozen grant. */ +export function resolveResponseProjectionFields( + value: unknown, + requiredFields: readonly string[], + authorizedFields: readonly string[] | null = null +): string[] | null { + const requestedFields = normalizeProjectionFields(value); + let fields = authorizedFields === null ? requestedFields : [...authorizedFields]; + if (requestedFields && authorizedFields !== null) { + fields = requestedFields.filter((field) => authorizedFields.includes(field)); + } + if (fields === null) { + return null; + } + const required = requiredFields.filter((field) => authorizedFields === null || authorizedFields.includes(field)); + return [...new Set([...required, ...fields])]; +} + export function projectRecordEnvelope>( record: T, fields: readonly string[] | null | undefined diff --git a/reference-implementation/operations/rs-records-detail/index.ts b/reference-implementation/operations/rs-records-detail/index.ts index ba1cf247a..0bde77e81 100644 --- a/reference-implementation/operations/rs-records-detail/index.ts +++ b/reference-implementation/operations/rs-records-detail/index.ts @@ -36,7 +36,11 @@ * - response writing. */ -import { normalizeProjectionFields, projectRecordEnvelope } from "../read-projection.ts"; +import { + normalizeProjectionFields, + projectRecordEnvelope, + resolveResponseProjectionFields, +} from "../read-projection.ts"; export interface RecordDetailSourceDescriptor { id: string; @@ -55,7 +59,7 @@ export type RecordDetailActor = export interface RecordDetailManifestStream { name: string; - schema?: { properties?: Record }; + schema?: { properties?: Record; required?: string[] }; [extra: string]: unknown; } @@ -94,6 +98,8 @@ export interface RecordDetailDependencies { * carry blob refs). */ decorateRecord: (record: Record) => Record; + /** Host-controlled prototype capability; omitted hosts retain legacy projection. */ + experimentalRecordUpdatesEnabled?: () => boolean; /** * Resolved grant the host derived for this request. Owner actors get an * owner read-grant; client actors get the token's grant. @@ -214,7 +220,15 @@ export async function executeRecordDetail( const record = projectRecordEnvelope( dependencies.decorateRecord(rawRecord), - normalizeProjectionFields(expandOptions.fields) + dependencies.experimentalRecordUpdatesEnabled?.() + ? resolveResponseProjectionFields( + expandOptions.fields, + manifestStream?.schema?.required ?? [], + input.actor.kind === "client" + ? (grant.streams.find((stream) => stream.name === input.streamName)?.fields ?? null) + : null + ) + : normalizeProjectionFields(expandOptions.fields) ); return { diff --git a/reference-implementation/operations/rs-records-list/index.ts b/reference-implementation/operations/rs-records-list/index.ts index e3c907cb4..c9e06e96b 100644 --- a/reference-implementation/operations/rs-records-list/index.ts +++ b/reference-implementation/operations/rs-records-list/index.ts @@ -34,7 +34,11 @@ * block fields populated from operation inputs and result counts. */ -import { normalizeProjectionFields, projectRecordEnvelope } from "../read-projection.ts"; +import { + normalizeProjectionFields, + projectRecordEnvelope, + resolveResponseProjectionFields, +} from "../read-projection.ts"; export interface RecordsListSourceDescriptor { id: string; @@ -58,7 +62,7 @@ export type RecordsListActor = */ export interface RecordsListManifestStream { name: string; - schema?: { properties?: Record }; + schema?: { properties?: Record; required?: string[] }; views?: Array<{ id: string; fields: string[] }>; [extra: string]: unknown; } @@ -105,6 +109,8 @@ export interface RecordsListDependencies { * carry blob refs). */ decorateRecord: (record: Record) => Record; + /** Host-controlled prototype capability; omitted hosts retain legacy projection. */ + experimentalRecordUpdatesEnabled?: () => boolean; /** * Resolved grant the host derived for this request. For client actors this * is the token's grant; for owner actors the operation will overwrite this @@ -296,10 +302,22 @@ export async function executeRecordsList( const rawResult = await dependencies.queryRecords(input.streamName, grant, input.requestParams, manifest); - const projectionFields = normalizeProjectionFields(input.requestParams.fields); - const decoratedData = rawResult.data.map((record) => - projectRecordEnvelope(dependencies.decorateRecord(record), projectionFields) - ); + const projectionFields = dependencies.experimentalRecordUpdatesEnabled?.() + ? resolveResponseProjectionFields( + input.requestParams.fields, + mStream?.schema?.required ?? [], + input.actor.kind === "client" + ? (grant.streams.find((stream) => stream.name === input.streamName)?.fields ?? null) + : null + ) + : normalizeProjectionFields(input.requestParams.fields); + const decoratedData = rawResult.data.map((record) => { + const decorated = dependencies.decorateRecord(record); + if (dependencies.experimentalRecordUpdatesEnabled?.() === true && decorated.deleted === true) { + return decorated; + } + return projectRecordEnvelope(decorated, projectionFields); + }); const decoratedResult: RecordsListQueryResult = { ...rawResult, data: decoratedData, diff --git a/reference-implementation/server/experimental-record-bootstrap.ts b/reference-implementation/server/experimental-record-bootstrap.ts new file mode 100644 index 000000000..f6251e584 --- /dev/null +++ b/reference-implementation/server/experimental-record-bootstrap.ts @@ -0,0 +1,255 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 +import { randomUUID } from "node:crypto"; +import { equalJsonValues } from "./experimental-record-updates.ts"; +import { currentStorageGeneration } from "./storage-generation.ts"; + +// Prototype only: retain serialized authorized snapshots, independent of history +// pruning. Sessions do not survive a process restart or a storage lifecycle change. +export const BOOTSTRAP_MAX_RECORDS = 10_000; +const MAX_SESSIONS = 32; +const MAX_BYTES = 32 * 1024 * 1024; +const SESSION_TTL_MS = 5 * 60 * 1000; +interface Session { + bytes: number; + changesCursor: string; + context: string; + expiresAt: number; + generation: number; + records: string[]; + baseline: Map; +} +export interface BootstrapSnapshot { + record: T; + eligibility: unknown; +} +interface BootstrapPage extends Record { + object: "list"; + data: T[]; + has_more: boolean; + next_cursor?: string; + next_changes_since?: string; +} +interface BootstrapCursor { + kind?: unknown; + session?: unknown; + bootstrap_session?: unknown; + bootstrap_offset?: unknown; +} +const sessions = new Map(); + +function bootstrapError( + code: string, + message: string, +): Error & { code: string } { + return Object.assign(new Error(message), { code }); +} + +function pruneSessions(): void { + for (const [id, session] of sessions) { + if ( + session.expiresAt <= Date.now() || + session.generation !== currentStorageGeneration() + ) { + sessions.delete(id); + } + } +} + +function page( + id: string, + session: Session, + offset: number, + limit: number, +): BootstrapPage { + const end = Math.min(offset + limit, session.records.length); + const hasMore = end < session.records.length; + return { + object: "list", + data: session.records + .slice(offset, end) + .map((record) => JSON.parse(record) as T), + has_more: hasMore, + ...(hasMore + ? { + next_cursor: Buffer.from( + JSON.stringify({ + kind: "page", + session: "changes", + bootstrap_session: id, + bootstrap_offset: end, + }), + ).toString("base64url"), + } + : { next_changes_since: session.changesCursor }), + }; +} + +function retainedSession(cursor: string, context: string): Session { + pruneSessions(); + let decoded: Record; + try { + decoded = JSON.parse(Buffer.from(cursor, "base64url").toString("utf8")); + } catch { + throw bootstrapError( + "invalid_cursor", + "Malformed retained bootstrap token", + ); + } + const session = + typeof decoded.bootstrap_baseline === "string" + ? sessions.get(decoded.bootstrap_baseline) + : null; + if (!session) + throw bootstrapError( + "cursor_expired", + "Experimental sync baseline expired; restart from beginning", + ); + if (session.context !== context || session.changesCursor !== cursor) { + throw bootstrapError( + "invalid_cursor", + "Experimental sync token does not match this query context", + ); + } + return session; +} + +export function retainedBootstrapRecordIds( + cursor: string, + context: string, +): string[] { + return [...retainedSession(cursor, context).baseline.keys()]; +} + +export function startBootstrapPage({ + context, + records, + changesCursor, + limit, + sinceCursor, + removedRecords = [], + order = "ASC", +}: { + context: string; + records: Iterable>; + changesCursor: string; + limit: number; + sinceCursor?: string; + removedRecords?: Iterable; + order?: "ASC" | "DESC"; +}): BootstrapPage { + pruneSessions(); + const previous = sinceCursor ? retainedSession(sinceCursor, context) : null; + const baseline = new Map(); + const serialized: string[] = []; + let bytes = 0; + for (const snapshot of records) { + const value = JSON.stringify(snapshot); + baseline.set(snapshot.record.id, value); + bytes += Buffer.byteLength(value); + if (baseline.size > BOOTSTRAP_MAX_RECORDS || bytes > MAX_BYTES) { + throw bootstrapError( + "bootstrap_capacity_exceeded", + "Experimental bootstrap snapshot exceeds its record or byte limit", + ); + } + const prior = previous?.baseline.get(snapshot.record.id); + if ( + !prior || + !equalJsonValues(JSON.parse(prior).eligibility, snapshot.eligibility) + ) { + serialized.push(JSON.stringify(snapshot.record)); + } + } + if (previous) { + const removed = new Map( + [...removedRecords].map((record) => [record.id, record]), + ); + for (const id of previous.baseline.keys()) { + if (baseline.has(id)) continue; + const record = removed.get(id); + if (!record) + throw bootstrapError( + "cursor_expired", + "Experimental sync deletion baseline cannot be resolved", + ); + serialized.push(JSON.stringify(record)); + } + } + bytes += serialized.reduce( + (total, value) => total + Buffer.byteLength(value), + 0, + ); + const orderedRecords = serialized.map((record) => ({ + id: (JSON.parse(record) as T).id, + record, + })); + orderedRecords.sort((left, right) => { + const comparison = left.id < right.id ? -1 : left.id > right.id ? 1 : 0; + return order === "DESC" ? -comparison : comparison; + }); + const id = randomUUID(); + const token = JSON.parse( + Buffer.from(changesCursor, "base64url").toString("utf8"), + ); + const session: Session = { + bytes, + context, + changesCursor: Buffer.from( + JSON.stringify({ ...token, bootstrap_baseline: id }), + ).toString("base64url"), + records: orderedRecords.map((entry) => entry.record), + baseline, + expiresAt: Date.now() + SESSION_TTL_MS, + generation: currentStorageGeneration(), + }; + const retainedBytes = [...sessions.values()].reduce( + (total, existing) => total + existing.bytes, + 0, + ); + if (sessions.size >= MAX_SESSIONS || retainedBytes + bytes > MAX_BYTES) { + throw bootstrapError( + "bootstrap_capacity_exceeded", + "Experimental bootstrap session capacity is exhausted", + ); + } + sessions.set(id, session); + return page(id, session, 0, limit); +} + +export function continueBootstrapPage( + cursor: BootstrapCursor, + context: string, + limit: number, +): BootstrapPage { + pruneSessions(); + const id = cursor.bootstrap_session; + const offset = cursor.bootstrap_offset; + if ( + cursor.kind !== "page" || + cursor.session !== "changes" || + typeof id !== "string" || + typeof offset !== "number" || + !Number.isInteger(offset) || + offset < 0 + ) { + throw bootstrapError( + "invalid_cursor", + "Malformed experimental bootstrap cursor", + ); + } + const session = sessions.get(id); + if (!session) { + throw bootstrapError( + "cursor_expired", + "Experimental bootstrap session expired; restart from beginning", + ); + } + if (session.context !== context || offset > session.records.length) { + throw bootstrapError( + "invalid_cursor", + "Experimental bootstrap cursor does not match this query context", + ); + } + return page(id, session, offset, limit); +} diff --git a/reference-implementation/server/postgres-records.ts b/reference-implementation/server/postgres-records.ts index 1bec24c67..34445d1d4 100644 --- a/reference-implementation/server/postgres-records.ts +++ b/reference-implementation/server/postgres-records.ts @@ -13,6 +13,7 @@ import { createHash } from "node:crypto"; import { applyPartialInstruction, enforceAppendOnlyDuplicate, experimentalRecordUpdatesEnabled, isPartialInstruction, validateExperimentalSnapshot } from "./experimental-record-updates.ts"; +import { BOOTSTRAP_MAX_RECORDS, continueBootstrapPage, retainedBootstrapRecordIds, startBootstrapPage } from "./experimental-record-bootstrap.ts"; import { buildLimitClampedWarning, clampRecordsPageLimit, @@ -32,6 +33,7 @@ import { isPostgresStorageBackend, postgresQuery as rawPostgresQuery, withPostgresTransaction, + withPostgresReadOnlyTransaction, } from "./postgres-storage.ts"; import { assertExpansionInstanceAuthorized, @@ -183,6 +185,7 @@ interface PgRow { } interface EffectiveFilter { + eligibilityFields?: string[] | null; fields?: string[] | null; resources?: string[] | null; timeConstraint?: { field: string; since?: string; until?: string } | null; @@ -221,6 +224,7 @@ interface QueryRequestParams extends Record { type ExpansionEntry = ReturnType[number]; interface ResponseRecord extends JsonObject { + id: string; expanded?: JsonObject; // Logical byte length of this record's current `record_json` // (octet_length(record_json)). Only populated on the single-record-detail @@ -386,9 +390,10 @@ function validateWindowKind(value: unknown): void { } } -function rejectListOnlyParamsForChangesFeed(requestParams: QueryRequestParams): void { +function rejectListOnlyParamsForChangesFeed(requestParams: QueryRequestParams, allowOrder = false): void { const unsupported: string[] = []; for (const key of ["sort", "count", "order", "window"]) { + if (key === "order" && allowOrder) continue; if (requestParams[key] !== null && requestParams[key] !== undefined && requestParams[key] !== "") { unsupported.push(key); } @@ -595,7 +600,9 @@ function fieldsFor( // resolution. Filtering and record identity are computed separately, so // current manifest requirements cannot add response fields. Owner grants // omit `instance_ids` and retain the current-manifest projection. - const projectionRequiredFields = Object.hasOwn(streamGrant, "instance_ids") ? [] : requiredFields; + const projectionRequiredFields = Object.hasOwn(streamGrant, "instance_ids") + ? (experimentalRecordUpdatesEnabled() ? requiredFields.filter((field) => !streamGrant.fields || streamGrant.fields.includes(field)) : []) + : requiredFields; for (const required of projectionRequiredFields) { if (!seen.has(required)) { effective.push(required); @@ -1392,7 +1399,8 @@ function resolvePostgresExpansionFields( const childEffective = buildEffectiveFilter( expansion.childGrant, {}, - childRequiredFields + childRequiredFields, + experimentalRecordUpdatesEnabled() ) as unknown as EffectiveFilter; return { childEffective, @@ -2151,7 +2159,7 @@ async function visiblePostgresChange({ if (!(current && currentVisible && passesRequestFilters(current.data, compiledFilters))) { return null; } - const fields = effective.fields ?? null; + const fields = (effective.eligibilityFields !== undefined ? effective.eligibilityFields : effective.fields) ?? null; const previousProjection = previous && previousVisible ? projectFields(previous.data, fields) : null; const currentProjection = projectFields(current.data, fields); if (previousProjection && JSON.stringify(previousProjection) === JSON.stringify(currentProjection)) { @@ -2160,7 +2168,7 @@ async function visiblePostgresChange({ return responseRecord({ fields: null, identity, - row: { ...row, record_json: currentProjection }, + row: { ...row, record_json: projectFields(current.data, effective.fields ?? null) }, stream, }); } @@ -2385,7 +2393,8 @@ export async function postgresQueryRecords( const effective = buildEffectiveFilter( streamGrant, {}, - requiredFieldsFor(manifestStream) + requiredFieldsFor(manifestStream), + experimentalRecordUpdatesEnabled() ) as unknown as EffectiveFilter; effective.fields = fields; const compiledFilters = compileRequestFilters(requestParams.filter, streamGrant, manifestStream ?? {}); @@ -2418,7 +2427,10 @@ export async function postgresQueryRecords( enforceConnectionNarrowing(requestParams, connectorInstanceId); const identity = await resolveRecordIdentityForBinding(connectorInstanceId, connectorId); - rejectExpandWithChangesSince(requestParams); + const bootstrapCursor = experimentalRecordUpdatesEnabled() && requestParams.cursor ? decodeCursor(requestParams.cursor) : null; + const retainedCursor = experimentalRecordUpdatesEnabled() && typeof requestParams.changes_since === "string" ? decodeCursor(requestParams.changes_since) : null; + const isBootstrapPage = experimentalRecordUpdatesEnabled() && bootstrapCursor?.bootstrap_session !== undefined; + rejectExpandWithChangesSince(isBootstrapPage ? { ...requestParams, changes_since: "beginning" } : requestParams); // Resolve and validate expansions up front so misuse rejects before any // SQL runs. SQLite path does the same in records.ts#normalizeExpandRequest. const expansions = normalizeExpandRequest( @@ -2429,6 +2441,48 @@ export async function postgresQueryRecords( order === "asc" ? "ASC" : "DESC" ); + if (experimentalRecordUpdatesEnabled() && (requestParams.changes_since === "beginning" || isBootstrapPage || retainedCursor?.bootstrap_baseline !== undefined)) { + rejectListOnlyParamsForChangesFeed(requestParams, true); + const context = JSON.stringify(["postgres", connectorInstanceId, stream, grant, effective, compiledFilters, manifest, orderDirection]); + const response = isBootstrapPage && bootstrapCursor + ? continueBootstrapPage(bootstrapCursor, context, limit) + : await withPostgresReadOnlyTransaction(async (client) => { + const sinceCursor = retainedCursor?.bootstrap_baseline !== undefined ? String(requestParams.changes_since) : undefined; + const previousIds = sinceCursor ? retainedBootstrapRecordIds(sinceCursor, context) : []; + const horizon = await client.query("SELECT max_version FROM version_counter WHERE connector_instance_id = $1 AND stream = $2", [connectorInstanceId, stream]); + const maxVersion = Number(horizon.rows[0]?.max_version ?? 0); + if (sinceCursor) { + const sinceVersion = retainedCursor?.v; + const minResult = await client.query("SELECT MIN(version)::bigint AS min_version FROM record_changes WHERE connector_instance_id = $1 AND stream = $2", [connectorInstanceId, stream]); + const minVersion = minResult.rows[0]?.min_version == null ? null : Number(minResult.rows[0].min_version); + if (typeof sinceVersion !== "number" || !Number.isInteger(sinceVersion)) throw Object.assign(new Error("Malformed changes_since cursor"), { code: "invalid_cursor" }); + if ((minVersion !== null && sinceVersion < minVersion - 1) || (minVersion === null && sinceVersion < maxVersion)) throw Object.assign(new Error("changes_since history has been pruned; full re-sync required"), { code: "cursor_expired" }); + } + const params: unknown[] = [connectorInstanceId, stream]; + const whereParts = ["connector_instance_id = $1", "stream = $2", "deleted = FALSE"]; + appendGrantVisibilityClauses(whereParts, params, effective); + const where = `WHERE ${whereParts.join(" AND ")}${buildFilterClause(compiledFilters, requestParams.filter, params)}`; + params.push(BOOTSTRAP_MAX_RECORDS + 1); + const rows = await client.query(`SELECT record_key, record_json, emitted_at FROM records ${where} ORDER BY version, record_key LIMIT $${params.length}`, params); + const removed = previousIds.length ? await client.query("SELECT record_key, emitted_at, deleted_at FROM records WHERE connector_instance_id = $1 AND stream = $2 AND record_key = ANY($3::text[])", [connectorInstanceId, stream, previousIds]) : { rows: [] }; + const eligibilityFields = effective.eligibilityFields !== undefined ? effective.eligibilityFields : fields; + return startBootstrapPage({ + context, records: rows.rows.map((row) => ({ record: responseRecord({ row, stream, fields, identity }), eligibility: projectFields(row.record_json, eligibilityFields ?? null) })), limit, order: orderDirection, + changesCursor: encodeCursor({ v: maxVersion }), + removedRecords: removed.rows.map((row) => deletedResponseRecord({ row, stream, identity })), + ...(sinceCursor ? { sinceCursor } : {}), + }); + }); + if (isBootstrapPage) { + const ids = response.data.filter((record) => record.deleted !== true).map((record) => record.id); + const current = ids.length ? await postgresQuery("SELECT record_key, deleted FROM records WHERE connector_instance_id = $1 AND stream = $2 AND record_key = ANY($3::text[])", [connectorInstanceId, stream, ids]) : { rows: [] }; + const liveIds = new Set(current.rows.filter((row) => !row.deleted).map((row) => row.record_key)); + response.data = response.data.filter((record) => record.deleted === true || liveIds.has(record.id)); + } + attachRequestWarningsToResponse(response, requestWarnings); + return response; + } + if (requestParams.changes_since !== null && requestParams.changes_since !== undefined) { return queryPostgresChangesSince({ compiledFilters, @@ -2672,7 +2726,8 @@ export async function postgresGetRecord( const effective = buildEffectiveFilter( streamGrant, {}, - requiredFieldsFor(manifestStream) + requiredFieldsFor(manifestStream), + experimentalRecordUpdatesEnabled() ) as unknown as EffectiveFilter; effective.fields = fields; const { warnings: requestWarnings } = resolveRequestConnectionId(requestParams); @@ -2840,7 +2895,8 @@ export async function postgresGetRecordFieldWindow( const effective = buildEffectiveFilter( streamGrant, {}, - requiredFieldsFor(manifestStream) + requiredFieldsFor(manifestStream), + experimentalRecordUpdatesEnabled() ) as unknown as EffectiveFilter; assertFieldVisibleToGrant(fieldPath, effective.fields); diff --git a/reference-implementation/server/record-expand-helpers.ts b/reference-implementation/server/record-expand-helpers.ts index 54d9f78cb..5d301ed97 100644 --- a/reference-implementation/server/record-expand-helpers.ts +++ b/reference-implementation/server/record-expand-helpers.ts @@ -33,6 +33,7 @@ interface EffectiveFilterGrant { time_constraint?: TimeConstraint | null; } export interface EffectiveFilter { + eligibilityFields?: string[] | null; fields: string[] | null; resources: string[] | null; timeConstraint: TimeConstraint | null; @@ -189,15 +190,21 @@ export function assertNonEmptyJsonField(field: unknown, label: string): asserts export function buildEffectiveFilter( streamGrant: EffectiveFilterGrant, requestParams: RequestParams, - requiredFields: string[] = [] + requiredFields: string[] = [], + experimentalRecordUpdates = false ): EffectiveFilter { const timeConstraint = requireTimeConstraint(streamGrant.time_constraint); - const effective = { + const effective: EffectiveFilter = { fields: streamGrant.fields || null, resources: streamGrant.resources || null, timeConstraint, timeConstraintField: timeConstraint ? timeConstraint.field : null, }; + const isClientGrant = Object.hasOwn(streamGrant, "instance_ids"); + if (experimentalRecordUpdates && isClientGrant) { + // Sync eligibility uses the frozen grant, independently of sparse responses. + effective.eligibilityFields = streamGrant.fields || null; + } if (requestParams.fields && effective.fields !== null) { effective.fields = requestParams.fields.filter((f) => effective.fields?.includes(f) === true); @@ -210,7 +217,12 @@ export function buildEffectiveFilter( // authorization time. A later manifest declaration must not widen that // grant by adding newly-required fields. Owner grants omit instance_ids // and retain the current-manifest behavior used by self-reads. - const manifestRequiredFields = Object.hasOwn(streamGrant, "instance_ids") ? [] : requiredFields; + let manifestRequiredFields = requiredFields; + if (isClientGrant) { + manifestRequiredFields = experimentalRecordUpdates + ? requiredFields.filter((field) => !streamGrant.fields || streamGrant.fields.includes(field)) + : []; + } effective.fields = [...new Set([...manifestRequiredFields, ...effective.fields])]; } diff --git a/reference-implementation/server/records.ts b/reference-implementation/server/records.ts index 8230aac86..e65a48074 100644 --- a/reference-implementation/server/records.ts +++ b/reference-implementation/server/records.ts @@ -7,6 +7,7 @@ import { parseCoverageDiagnosticsStateSnapshot } from "./polyfill-connectors-run */ import { getDb } from "./db.ts"; import { applyPartialInstruction, enforceAppendOnlyDuplicate, equalJsonValues, experimentalRecordUpdatesEnabled, isPartialInstruction, recordUpdateError, validateExperimentalSnapshot, type ExperimentalStream } from "./experimental-record-updates.ts"; +import { continueBootstrapPage, retainedBootstrapRecordIds, startBootstrapPage, type BootstrapSnapshot } from "./experimental-record-bootstrap.ts"; import { assertGrantedManifestReadAuthority, assertManifestReadAuthority } from "./manifest-read-authority.ts"; import type { ComputedLexicalFields } from "./search.ts"; import type { ComputedSemanticEntries } from "./search-semantic.ts"; @@ -304,6 +305,7 @@ interface StreamGrant { time_constraint?: TimeConstraint | null; } interface EffectiveReadScope { + eligibilityFields?: string[] | null; fields: string[] | null; resources: string[] | null; timeConstraint: TimeConstraint | null; @@ -633,6 +635,9 @@ interface LogicalPosition { } type PageOrder = "ASC" | "DESC"; interface PaginationCursor { + kind?: unknown; + bootstrap_session?: unknown; + bootstrap_offset?: unknown; after_version?: number; cursor_value?: unknown; order?: unknown; @@ -650,8 +655,9 @@ function buildEffectiveReadScope( requestParams: Parameters[1], requiredFields: string[] = [] ): EffectiveReadScope { - const effective = buildEffectiveFilter(streamGrant, requestParams, requiredFields); + const effective = buildEffectiveFilter(streamGrant, requestParams, requiredFields, experimentalRecordUpdatesEnabled()); return { + ...(effective.eligibilityFields !== undefined ? { eligibilityFields: effective.eligibilityFields } : {}), fields: Array.isArray(effective.fields) ? effective.fields.filter((field): field is string => typeof field === "string") : null, @@ -2354,7 +2360,7 @@ async function ingestSqliteRecord( recordJson = JSON.stringify(data); semanticTime = computeIngestSemanticTime(connectorId, stream, data, attemptStreamFacts); } - if (op !== "delete" && current && !current.deleted && enforceAppendOnlyDuplicate(experimentalStream, current.record_json ? JSON.parse(current.record_json) : null, data)) { + if (experimentalRecordUpdatesEnabled() && op !== "delete" && current && !current.deleted && enforceAppendOnlyDuplicate(experimentalStream, current.record_json ? JSON.parse(current.record_json) : null, data)) { return finishDurableOutcome({ kind: "noop" }); } @@ -3509,9 +3515,10 @@ function validateWindowKind(value: unknown): void { } } -function rejectListOnlyParamsForChangesFeed(requestParams: RequestParams): void { +function rejectListOnlyParamsForChangesFeed(requestParams: RequestParams, allowOrder = false): void { const unsupported: string[] = []; for (const key of ["sort", "count", "order", "window"]) { + if (key === "order" && allowOrder) continue; if (!isNullish(requestParams[key]) && requestParams[key] !== "") { unsupported.push(key); } @@ -5276,16 +5283,17 @@ function resolveChangeGroupRecord( return null; } + const eligibilityFields = effective.eligibilityFields !== undefined ? effective.eligibilityFields : effective.fields; const previousProjection = - previousVisible && previous?.data ? projectFields(previous.data, effective.fields ?? undefined) : null; - const currentProjection = projectFields(current.data, effective.fields ?? undefined); + previousVisible && previous?.data ? projectFields(previous.data, eligibilityFields ?? undefined) : null; + const currentProjection = projectFields(current.data, eligibilityFields ?? undefined); if (previousProjection && JSON.stringify(previousProjection) === JSON.stringify(currentProjection)) { return null; } const changeRecord: ResponseRecord = { - data: currentProjection, + data: projectFields(current.data, effective.fields ?? undefined), emitted_at: current.emitted_at, id: group.record_key, object: "record", @@ -5499,7 +5507,10 @@ function validateQueryCursors( throw codedError("Malformed cursor", "invalid_cursor"); } if (changesSince !== null || paginationCursor?.session === "changes") { - rejectListOnlyParamsForChangesFeed(requestParams); + const retainedCursor = experimentalRecordUpdatesEnabled() && typeof requestParams.changes_since === "string" + ? decodeCursor(requestParams.changes_since) : null; + const allowOrder = experimentalRecordUpdatesEnabled() && (requestParams.changes_since === "beginning" || paginationCursor?.bootstrap_session !== undefined || retainedCursor?.bootstrap_baseline !== undefined); + rejectListOnlyParamsForChangesFeed(requestParams, allowOrder); } if ((changesSince !== null || paginationCursor?.session === "changes") && expansions.length) { throw invalidQueryError("expand is not supported with changes_since", "invalid_expand"); @@ -5593,6 +5604,58 @@ export async function queryRecords( const ctx = await validateQueryRecordsRequest(storageTarget, stream, grant, requestParams, manifest); const { changesSince, paginationCursor } = ctx; + const retainedCursor = experimentalRecordUpdatesEnabled() && typeof requestParams.changes_since === "string" + ? decodeCursor(requestParams.changes_since) : null; + + if (experimentalRecordUpdatesEnabled() && (requestParams.changes_since === "beginning" || paginationCursor?.bootstrap_session !== undefined || retainedCursor?.bootstrap_baseline !== undefined)) { + const context = JSON.stringify(["sqlite", ctx.connectorInstanceId, stream, grant, ctx.effective, ctx.compiledFilters, manifest, ctx.order]); + const response: ListResponse = paginationCursor?.bootstrap_session !== undefined + ? continueBootstrapPage(paginationCursor, context, ctx.limit) + : writeTransaction(() => { + const sinceCursor = retainedCursor?.bootstrap_baseline !== undefined ? String(requestParams.changes_since) : undefined; + const previousIds = sinceCursor ? retainedBootstrapRecordIds(sinceCursor, context) : []; + const horizon = getOne(referenceQueries.recordsIngestGetVersionCounter, [ctx.connectorInstanceId, stream])?.max_version ?? 0; + if (sinceCursor && changesSince) { + validateChangesSinceVersions(changesSince, null, ctx.connectorInstanceId, stream); + const min = getOne(referenceQueries.recordsSnapshotsGetMinRecordChangeVersion, [ctx.connectorInstanceId, stream])?.min_version ?? null; + if (min === null && changesSince.version < horizon) throw codedError("changes_since history has been pruned; full re-sync required", "cursor_expired"); + } + function* visibleRecords(): Generator> { + // REVIEWED-DYNAMIC: fixed instance/stream read; streamed into a bounded + // experimental snapshot so later pages need no retained history. + for (const row of iterateDynamicSqlAcknowledged<{ record_key: string; record_json: string; emitted_at: string }>( + "SELECT record_key, record_json, emitted_at FROM records WHERE connector_instance_id = ? AND stream = ? AND deleted = 0 ORDER BY version, record_key", + [ctx.connectorInstanceId, stream] + )) { + const data = parseStoredRecordData(row.record_json); + if ((ctx.effective.resources && !ctx.effective.resources.includes(row.record_key)) || !passesTimeConstraint(data, ctx.effective.timeConstraint) || !passesRequestFilters(data, ctx.compiledFilters)) continue; + const record: ResponseRecord = { object: "record", id: row.record_key, stream, emitted_at: row.emitted_at, data: projectFields(data, ctx.effective.fields ?? undefined) }; + decorateRecordWithConnectionIdentity(record, ctx.recordIdentity); + const eligibilityFields = ctx.effective.eligibilityFields !== undefined ? ctx.effective.eligibilityFields : ctx.effective.fields; + yield { record, eligibility: projectFields(data, eligibilityFields ?? undefined) }; + } + } + function* removedRecords(): Generator { + for (const key of previousIds) { + const row = getDb().prepare("SELECT emitted_at, deleted_at FROM records WHERE connector_instance_id = ? AND stream = ? AND record_key = ?").get<{ emitted_at: string; deleted_at: string | null }>(ctx.connectorInstanceId, stream, key); + if (!row) continue; + const record: ResponseRecord = { object: "record", id: key, stream, deleted: true, emitted_at: row.emitted_at, deleted_at: row.deleted_at ?? row.emitted_at }; + decorateRecordWithConnectionIdentity(record, ctx.recordIdentity); + yield record; + } + } + return startBootstrapPage({ context, records: visibleRecords(), changesCursor: encodeChangesSinceCursor(horizon), limit: ctx.limit, order: ctx.order, removedRecords: removedRecords(), ...(sinceCursor ? { sinceCursor } : {}) }); + }); + if (paginationCursor?.bootstrap_session !== undefined) { + response.data = response.data.filter((record) => { + if (record.deleted === true) return true; + const current = getDb().prepare("SELECT deleted FROM records WHERE connector_instance_id = ? AND stream = ? AND record_key = ?").get<{ deleted: boolean | number }>(ctx.connectorInstanceId, stream, record.id); + return !!current && !current.deleted; + }); + } + attachRequestWarningsToResponse(response, ctx.requestWarnings); + return response; + } if (changesSince !== null || paginationCursor?.session === "changes") { return queryRecordsChangesSince({ diff --git a/reference-implementation/server/routes/rs-read.ts b/reference-implementation/server/routes/rs-read.ts index 9a89055f2..a04d0818c 100644 --- a/reference-implementation/server/routes/rs-read.ts +++ b/reference-implementation/server/routes/rs-read.ts @@ -77,6 +77,7 @@ import { type StreamsListDependencies, type StreamsListInput, } from "../../operations/rs-streams-list/index.ts"; +import { experimentalRecordUpdatesEnabled } from "../experimental-record-updates.ts"; import { rejectUnsupportedClientQuery } from "../record-filters.ts"; import type { MiddlewareHandler, RouteArg } from "./_route-contract.ts"; @@ -1777,6 +1778,7 @@ function buildRecordsListDeps( const { manifest, storageBinding, sourceDescriptor } = refs; return { decorateRecord: (record: unknown) => ctx.decorateRecordBlobRefs(record), + experimentalRecordUpdatesEnabled, getGrant: () => tokenInfo.grant || { streams: [] }, getManifest: () => manifest, getSourceDescriptor: () => sourceDescriptor, @@ -1970,6 +1972,7 @@ export function mountRsRecordDetail(app: AppLike, ctx: MountRsReadContext): void const dependencies = { decorateRecord: (record: unknown) => ctx.decorateRecordBlobRefs(record), + experimentalRecordUpdatesEnabled, getGrant: () => tokenInfo.grant || { streams: [] }, getManifest: () => manifest, getRecord: async ( diff --git a/reference-implementation/test/experimental-record-updates-http.test.ts b/reference-implementation/test/experimental-record-updates-http.test.ts index 30c8f851a..ea3f74ab3 100644 --- a/reference-implementation/test/experimental-record-updates-http.test.ts +++ b/reference-implementation/test/experimental-record-updates-http.test.ts @@ -223,10 +223,21 @@ for (const backend of ["sqlite", "postgres"] as const) { }; assert.deepEqual( projected.data[0]?.data, - { id: "a" }, + { id: "a", value: "updated" }, "resolved owner projection omits absent optional and out-of-projection stored values", ); } + const sparseDetail = await fetch( + `${rsUrl}/v1/streams/state/records/a?connector_id=${connectorId}&connector_instance_id=${connectorInstanceId}&fields=id,optional`, + { headers: { Authorization: authorization } }, + ); + assert.equal(sparseDetail.status, 200); + assert.deepEqual( + ((await sparseDetail.json()) as { data: Record }) + .data, + { id: "a", value: "updated" }, + "owner detail projection retains required fields and omits removed fields", + ); const malformed = await ingest("state", [ { data: null, diff --git a/reference-implementation/test/experimental-record-updates.test.ts b/reference-implementation/test/experimental-record-updates.test.ts index b250c5e7b..7e892e513 100644 --- a/reference-implementation/test/experimental-record-updates.test.ts +++ b/reference-implementation/test/experimental-record-updates.test.ts @@ -3,8 +3,15 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { executeRecordsList } from "../operations/rs-records-list/index.ts"; import { assertValidRecordEnvelope } from "../runtime/record-message-validator.ts"; import { closeDb, getDb, initDb } from "../server/db.ts"; +import { + BOOTSTRAP_MAX_RECORDS, + continueBootstrapPage, + retainedBootstrapRecordIds, + startBootstrapPage, +} from "../server/experimental-record-bootstrap.ts"; import { EXPERIMENTAL_PARTIAL_UPDATE_TYPE } from "../server/experimental-record-updates.ts"; import { deriveReferenceFreshness } from "../server/freshness.ts"; import { @@ -20,6 +27,7 @@ import { queryRecords, setClientEventEnqueueHook, } from "../server/records.ts"; +import { bumpStorageGeneration } from "../server/storage-generation.ts"; import { dedicatedPostgresTestUrl } from "./helpers/dedicated-postgres-test-url.ts"; import { withTemporaryPostgresDatabase } from "./helpers/postgres-temp-database.ts"; @@ -43,7 +51,13 @@ const schema = { }; const manifest = { streams: [ - ...["events", "state"].map((name) => ({ + ...[ + "events", + "state", + "bootstrap", + "bootstrap-followup", + "bootstrap-order", + ].map((name) => ({ consent_time_field: "created_at", cursor_field: "created_at", name, @@ -119,6 +133,27 @@ function rows() { .all(); } +interface HistoryRow { + record_json: unknown; + version: number | string; +} +function assertCompleteHistory( + history: HistoryRow[], + expected: Array<{ data: unknown; version: number }>, +): void { + assert.deepEqual( + history.map((row) => ({ + data: + typeof row.record_json === "string" + ? JSON.parse(row.record_json) + : row.record_json, + version: Number(row.version), + })), + expected, + "every changed write must retain its complete snapshot in version order", + ); +} + const postgresUrl = dedicatedPostgresTestUrl( process.env.PDPP_TEST_POSTGRES_URL, ); @@ -296,16 +331,23 @@ for (const backend of ["sqlite", "postgres"] as const) { ); assert.equal(beginning.data.length, 2); const cursor = beginning.next_changes_since; - if (cursor) { - const afterRetry = await queryRecords( - target, - "events", - grant, - { changes_since: cursor }, - manifest, - ); - assert.equal(afterRetry.data.length, 0); - } + assert.ok( + cursor, + "the completed beginning session must issue a retry cursor", + ); + assert.equal( + (await write(snapshot())).changed, + false, + "retry after saving the session cursor", + ); + const afterRetry = await queryRecords( + target, + "events", + grant, + { changes_since: cursor }, + manifest, + ); + assert.equal(afterRetry.data.length, 0); if (databaseUrl) { await postgresQuery( "DELETE FROM record_changes WHERE stream = $1 AND record_key = 'a'", @@ -328,20 +370,30 @@ for (const backend of ["sqlite", "postgres"] as const) { }, ); await t.test( - "observed residual: beginning after anchor pruning expires although current append_only record survives", + "beginning after anchor pruning enumerates current append_only state without a duplicate version", async () => { - assert.equal((await read("events")).data.required, "value"); - await assert.rejects( - queryRecords( - target, - "events", - grant, - { changes_since: "beginning" }, - manifest, - ), - { - code: "cursor_expired", - }, + const before = await databaseState(); + const beginning = await queryRecords( + target, + "events", + grant, + { changes_since: "beginning" }, + manifest, + ); + assert.deepEqual(beginning.data.map((record) => record.id).sort(), [ + "a", + "array", + ]); + assert.equal( + beginning.data.find((record) => record.id === "a")?.data + ?.required, + "value", + ); + assert.ok(beginning.next_changes_since); + assert.deepEqual( + await databaseState(), + before, + "bootstrap is read-only", ); }, ); @@ -564,45 +616,637 @@ for (const backend of ["sqlite", "postgres"] as const) { }, ); await t.test( - "snapshot replacement and complete version history", + "sparse resolved-client response preserves cached fields outside its projection", async () => { - const clientCopyBefore = { ...(await read("state")).data }; - assert.equal(Object.hasOwn(clientCopyBefore, "optional"), true); + const stored = { + id: "cache", + nested: { keep: true }, + optional: "keep", + required: "Ada", + }; + await write({ data: stored, key: "cache", stream: "state" }); + const clientGrant = { + streams: [ + { + fields: ["id", "required", "optional", "nested", "array"], + instance_ids: [target.connectorInstanceId], + name: "state", + resources: ["cache"], + }, + ], + }; + const requested = ["optional", "array"]; + const output = await executeRecordsList( + { + actor: { + kind: "client", + subject_id: "owner", + client_id: "client", + grant_id: "grant", + }, + rawQueryFields: requested.join(","), + requestParams: { fields: requested }, + streamName: "state", + }, + { + decorateRecord: (record) => record, + experimentalRecordUpdatesEnabled: () => true, + getGrant: () => clientGrant, + getManifest: () => manifest, + getSourceDescriptor: () => ({ + id: connectorId, + kind: "connector", + }), + queryRecords: async (stream, _grant, params) => { + const result = await queryRecords( + target, + stream, + clientGrant, + params, + manifest, + ); + return { + ...result, + data: result.data.map((record) => ({ ...record })), + }; + }, + validateRequestFields: () => {}, + }, + ); + assert.deepEqual( + output.result.data[0]?.data, + { id: "cache", optional: "keep", required: "Ada" }, + "required fields survive the final operation projection", + ); + const cache: Record = { + ...stored, + array: ["stale"], + }; + const returned = output.result.data[0]?.data as Record< + string, + unknown + >; + for (const field of ["id", "required", ...requested]) { + if (Object.hasOwn(returned, field)) + cache[field] = returned[field]; + else delete cache[field]; + } + assert.deepEqual( + cache, + stored, + "an absent requested field disappears; unrequested nested value survives", + ); + const full = await queryRecords( + target, + "state", + clientGrant, + {}, + manifest, + ); + assert.deepEqual( + full.data[0]?.data, + stored, + "sparse response does not delete stored values", + ); + }, + ); + await t.test( + "sparse client changes_since eligibility uses the full frozen grant projection", + async () => { + const data = { + id: "eligibility", + optional: "before", + required: "stable", + }; + await write({ data, key: "eligibility", stream: "state" }); + const clientGrant = { + streams: [ + { + fields: ["id", "required", "optional"], + instance_ids: [target.connectorInstanceId], + name: "state", + resources: ["eligibility"], + }, + ], + }; + const fields = ["id", "required"]; + const beginning = await queryRecords( + target, + "state", + clientGrant, + { changes_since: "beginning", fields }, + manifest, + ); + assert.ok(beginning.next_changes_since); + assert.deepEqual(beginning.data[0]?.data, { + id: "eligibility", + required: "stable", + }); await write({ - ...snapshot("state"), - data: { id: "a", required: "replacement" }, + ...instruction({ optional: "after" }), + key: "eligibility", }); - assert.deepEqual( - (await read("state")).data, - { id: "a", required: "replacement" }, - "snapshots replace omitted values", + const delta = await queryRecords( + target, + "state", + clientGrant, + { changes_since: beginning.next_changes_since, fields }, + manifest, ); - const replacementCopy = { ...(await read("state")).data }; assert.equal( - Object.hasOwn(replacementCopy, "optional"), - false, - "a client replacing its copy within the same projection removes absent values", + delta.data.length, + 1, + "unrequested granted-field change still qualifies for sync", ); - const history = databaseUrl - ? ( - await postgresQuery( - "SELECT record_json FROM record_changes WHERE stream = 'state' AND NOT deleted", - ) - ).rows - : getDb() + assert.deepEqual( + delta.data[0]?.data, + beginning.data[0]?.data, + "response is sparse even when eligibility uses more fields", + ); + }, + ); + await t.test( + "bootstrap pages preserve authorized h1 state after writes and history pruning", + async () => { + for (const id of ["a", "b", "c", "deleted", "hidden", "old"]) { + await write({ + data: { + created_at: + id === "old" + ? "2026-09-01T00:00:00Z" + : "2026-10-01T00:00:00Z", + id, + required: "at-h1", + }, + key: id, + stream: "bootstrap", + }); + } + await write({ key: "deleted", op: "delete", stream: "bootstrap" }); + const scope = { + streams: [ + { + fields: ["id", "required", "optional"], + instance_ids: [target.connectorInstanceId], + name: "bootstrap", + resources: ["a", "b", "c", "deleted", "old"], + time_constraint: { + field: "created_at", + since: "2026-10-01T00:00:00Z", + }, + }, + ], + }; + const prune = async () => { + if (databaseUrl) + await postgresQuery( + "DELETE FROM record_changes WHERE stream = 'bootstrap'", + ); + else + getDb() .prepare( - "SELECT record_json FROM record_changes WHERE stream = 'state' AND NOT deleted", + "DELETE FROM record_changes WHERE stream = 'bootstrap'", ) - .all(); - for (const row of history as Array<{ record_json: unknown }>) { - const data = - typeof row.record_json === "string" - ? JSON.parse(row.record_json) - : row.record_json; - assert.equal(Object.hasOwn(data, "type"), false); - assert.equal(typeof data.required, "string"); - assert.equal(data.id, "a"); + .run(); + }; + await prune(); + const before = await databaseState(); + let page = await queryRecords( + target, + "bootstrap", + scope, + { changes_since: "beginning", limit: 1, order: "asc" }, + manifest, + ); + assert.equal(page.has_more, true); + assert.ok(page.next_cursor); + assert.equal( + page.next_changes_since, + undefined, + "sync token is terminal-only", + ); + assert.deepEqual( + await databaseState(), + before, + "capturing h1 is read-only", + ); + const firstCursor = page.next_cursor; + await assert.rejects( + queryRecords( + target, + "bootstrap", + scope, + { cursor: firstCursor, fields: ["id"], limit: 1, order: "asc" }, + manifest, + ), + { code: "invalid_cursor" }, + ); + await write({ + data: { + created_at: "2026-10-01T00:00:00Z", + id: "b", + required: "after-h1", + }, + key: "b", + stream: "bootstrap", + }); + await write({ key: "c", op: "delete", stream: "bootstrap" }); + await write({ + data: { + created_at: "2026-10-01T00:00:00Z", + id: "later", + required: "after-h1", + }, + key: "later", + stream: "bootstrap", + }); + await prune(); + const collected = [...page.data]; + while (page.has_more) { + assert.ok(page.next_cursor); + page = await queryRecords( + target, + "bootstrap", + scope, + { cursor: page.next_cursor, limit: 1, order: "asc" }, + manifest, + ); + collected.push(...page.data); + if (page.has_more) + assert.equal(page.next_changes_since, undefined); } + assert.deepEqual( + collected.map((record) => record.id), + ["a", "b"], + ); + assert.deepEqual( + collected.map((record) => record.data), + ["a", "b"].map((id) => ({ id, required: "at-h1" })), + ); + assert.ok(page.next_changes_since); + await write({ + data: { + created_at: "2026-10-01T00:00:00Z", + id: "b", + required: "one", + }, + key: "b", + stream: "bootstrap", + }); + await write({ + data: { + created_at: "2026-10-01T00:00:00Z", + id: "b", + required: "two", + }, + key: "b", + stream: "bootstrap", + }); + if (databaseUrl) + await postgresQuery( + "DELETE FROM record_changes WHERE stream = 'bootstrap' AND version < (SELECT MAX(version) FROM record_changes WHERE stream = 'bootstrap')", + ); + else + getDb() + .prepare( + "DELETE FROM record_changes WHERE stream = 'bootstrap' AND version < (SELECT MAX(version) FROM record_changes WHERE stream = 'bootstrap')", + ) + .run(); + await assert.rejects( + queryRecords( + target, + "bootstrap", + scope, + { changes_since: page.next_changes_since, order: "asc" }, + manifest, + ), + { code: "cursor_expired" }, + ); + const restarted = await queryRecords( + target, + "bootstrap", + scope, + { changes_since: "beginning", order: "asc" }, + manifest, + ); + assert.deepEqual(restarted.data.map((record) => record.id).sort(), [ + "a", + "b", + ]); + }, + ); + await t.test( + "post-pruning bootstrap suppresses ungranted changes and retains later deletion baselines", + async () => { + const record = { + data: { id: "a", optional: "before", required: "stable" }, + key: "a", + stream: "bootstrap-followup", + }; + await write(record); + if (databaseUrl) + await postgresQuery( + "DELETE FROM record_changes WHERE stream = 'bootstrap-followup'", + ); + else + getDb() + .prepare( + "DELETE FROM record_changes WHERE stream = 'bootstrap-followup'", + ) + .run(); + const clientGrant = { + streams: [ + { + fields: ["id", "required"], + instance_ids: [target.connectorInstanceId], + name: "bootstrap-followup", + }, + ], + }; + const beginning = await queryRecords( + target, + "bootstrap-followup", + clientGrant, + { changes_since: "beginning" }, + manifest, + ); + assert.ok(beginning.next_changes_since); + await write( + instruction({ optional: "after" }, [], "bootstrap-followup"), + ); + const unchanged = await queryRecords( + target, + "bootstrap-followup", + clientGrant, + { changes_since: beginning.next_changes_since }, + manifest, + ); + assert.deepEqual( + unchanged.data, + [], + "ungranted changes must not qualify for incremental sync", + ); + assert.ok(unchanged.next_changes_since); + await write({ + key: "a", + op: "delete", + stream: "bootstrap-followup", + }); + const deletion = await queryRecords( + target, + "bootstrap-followup", + clientGrant, + { changes_since: unchanged.next_changes_since }, + manifest, + ); + assert.equal( + deletion.data.length, + 1, + "rolling h0 baseline must retain the prior authorized record", + ); + assert.equal(deletion.data[0]?.deleted, true); + }, + ); + await t.test( + "retained sync pages order records and tombstones by canonical key and bind direction", + async () => { + const stream = "bootstrap-order"; + for (const id of ["z", "a", "m"]) + await write({ + stream, + key: id, + data: { id, required: "before" }, + }); + const orderGrant = { + streams: [ + { + name: stream, + fields: ["id", "required"], + instance_ids: [target.connectorInstanceId], + }, + ], + }; + const first = await queryRecords( + target, + stream, + orderGrant, + { changes_since: "beginning", limit: 1, order: "asc" }, + manifest, + ); + assert.ok(first.next_cursor); + const ids = first.data.map((record) => record.id); + let page = first; + while (page.next_cursor) { + page = await queryRecords( + target, + stream, + orderGrant, + { cursor: page.next_cursor, limit: 1, order: "asc" }, + manifest, + ); + ids.push(...page.data.map((record) => record.id)); + } + assert.deepEqual( + ids, + ["a", "m", "z"], + "bootstrap order must not reveal write order", + ); + assert.ok(page.next_changes_since); + await assert.rejects( + queryRecords( + target, + stream, + orderGrant, + { cursor: first.next_cursor, limit: 1, order: "desc" }, + manifest, + ), + { code: "invalid_cursor" }, + ); + const descending = await queryRecords( + target, + stream, + orderGrant, + { changes_since: "beginning", order: "desc" }, + manifest, + ); + assert.deepEqual( + descending.data.map((record) => record.id), + ["z", "m", "a"], + ); + await write({ + stream, + key: "z", + data: { id: "z", required: "after" }, + }); + await write({ stream, key: "a", op: "delete" }); + const changes = await queryRecords( + target, + stream, + orderGrant, + { changes_since: page.next_changes_since, order: "asc" }, + manifest, + ); + assert.deepEqual( + changes.data.map((record) => [ + record.id, + record.deleted === true, + ]), + [ + ["a", true], + ["z", false], + ], + "tombstones and snapshots share canonical key order", + ); + }, + ); + await t.test( + "client list preserves tombstone envelopes with full and sparse responses", + async () => { + await write({ + data: { id: "tombstone", required: "before" }, + key: "tombstone", + stream: "state", + }); + const clientGrant = { + streams: [ + { + fields: ["id", "required"], + instance_ids: [target.connectorInstanceId], + name: "state", + resources: ["tombstone"], + }, + ], + }; + const beginning = await queryRecords( + target, + "state", + clientGrant, + { changes_since: "beginning" }, + manifest, + ); + assert.ok(beginning.next_changes_since); + await write({ key: "tombstone", op: "delete", stream: "state" }); + for (const fields of [undefined, ["id"]]) { + const params = { + changes_since: beginning.next_changes_since, + ...(fields ? { fields } : {}), + }; + const output = await executeRecordsList( + { + actor: { + kind: "client", + subject_id: "owner", + client_id: "client", + grant_id: "grant", + }, + requestParams: params, + streamName: "state", + }, + { + decorateRecord: (record) => record, + experimentalRecordUpdatesEnabled: () => true, + getGrant: () => clientGrant, + getManifest: () => manifest, + getSourceDescriptor: () => ({ + id: connectorId, + kind: "connector", + }), + queryRecords: async (stream, _grant, request) => { + const result = await queryRecords( + target, + stream, + clientGrant, + request, + manifest, + ); + return { + ...result, + data: result.data.map((record) => ({ ...record })), + }; + }, + validateRequestFields: () => {}, + }, + ); + assert.equal(output.result.data.length, 1); + assert.equal(output.result.data[0]?.deleted, true); + assert.equal(output.result.data[0]?.object, "record"); + assert.equal(output.result.data[0]?.stream, "state"); + assert.equal(output.result.data[0]?.id, "tombstone"); + assert.equal(output.result.data[0]?.data, undefined); + } + }, + ); + await t.test( + "snapshot replacement and complete version history", + async () => { + const initial = { + array: [1, 2], + id: "history", + nested: { one: 1 }, + optional: "present", + required: "initial", + }; + const nullable = { ...initial, optional: null }; + const removed = { + array: [1, 2], + id: "history", + nested: { one: 1 }, + required: "initial", + }; + const replacement = { id: "history", required: "replacement" }; + const writes = [ + { data: initial, key: "history", stream: "state" }, + { ...instruction({ optional: null }), key: "history" }, + { ...instruction({}, ["optional"]), key: "history" }, + { data: replacement, key: "history", stream: "state" }, + ]; + const expected: Array<{ data: unknown; version: number }> = []; + for (const [index, record] of writes.entries()) { + const outcome = await write(record); + assert.equal(outcome.changed, true); + assert.equal(typeof outcome.version, "number"); + expected.push({ + data: [initial, nullable, removed, replacement][index], + version: outcome.version as number, + }); + } + const historySql = + "SELECT record_json, version FROM record_changes WHERE stream = 'state' AND record_key = 'history' AND NOT deleted ORDER BY version"; + const history = databaseUrl + ? (await postgresQuery(historySql)).rows + : getDb().prepare(historySql).all(); + assertCompleteHistory(history as HistoryRow[], expected); + assert.throws( + () => assertCompleteHistory([], expected), + "the oracle must reject deleted history", + ); + assert.throws( + () => + assertCompleteHistory( + (history as HistoryRow[]).slice(1), + expected, + ), + "the oracle must reject a missing partial snapshot", + ); + assert.throws( + () => + assertCompleteHistory( + [...(history as HistoryRow[])].reverse(), + expected, + ), + "the oracle must reject reordered versions", + ); + assert.throws( + () => + assertCompleteHistory( + (history as HistoryRow[]).map((row) => ({ + ...row, + record_json: replacement, + })), + expected, + ), + "the oracle must reject loss of unnamed historical values", + ); }, ); await t.test( @@ -901,3 +1545,163 @@ test("experimental SQLite JSON equality ignores nested member order", async () = delete process.env.PDPP_EXPERIMENTAL_RECORD_UPDATES; } }); + +test("flag-off SQLite retries do not parse current snapshots for disabled duplicate handling", async (t) => { + setup(); + delete process.env.PDPP_EXPERIMENTAL_RECORD_UPDATES; + try { + const data = { id: "a", required: "unique-disabled-parse-probe" }; + const record = { data, key: "a", stream: "events" }; + await ingestRecord(target, record, { deferIndexes: true }); + const storedJson = getDb() + .prepare("SELECT record_json FROM records WHERE stream = 'events'") + .get()?.record_json; + let calls = 0; + const parse = JSON.parse; + t.mock.method( + JSON, + "parse", + (value: string, reviver?: Parameters[1]) => { + if (value === storedJson) calls += 1; + return parse(value, reviver); + }, + ); + assert.equal( + (await ingestRecord(target, record, { deferIndexes: true })).changed, + false, + ); + assert.equal( + calls, + 0, + "flag-off duplicate handling must retain the legacy string comparison", + ); + } finally { + t.mock.restoreAll(); + closeDb(); + delete process.env.PDPP_EXPERIMENTAL_RECORD_UPDATES; + } +}); + +test("retained bootstrap state binds tokens and pages to context and storage lifecycle", (t) => { + bumpStorageGeneration(); + const token = Buffer.from(JSON.stringify({ v: 2 })).toString("base64url"); + const records = ["a", "b"].map((id) => ({ + record: { id, data: { id } }, + eligibility: { id }, + })); + const result = startBootstrapPage({ + context: "authorized", + records, + changesCursor: token, + limit: 1, + }); + assert.equal(result.has_more, true); + assert.equal(result.next_changes_since, undefined); + assert.ok(result.next_cursor); + const cursor = JSON.parse( + Buffer.from(result.next_cursor, "base64url").toString(), + ); + assert.throws(() => continueBootstrapPage(cursor, "different-grant", 1), { + code: "invalid_cursor", + }); + const terminal = continueBootstrapPage<{ id: string; data: { id: string } }>( + cursor, + "authorized", + 1, + ); + assert.deepEqual(terminal.data, [{ id: "b", data: { id: "b" } }]); + assert.ok(terminal.next_changes_since); + const terminalToken = terminal.next_changes_since; + assert.deepEqual( + retainedBootstrapRecordIds(terminal.next_changes_since, "authorized"), + ["a", "b"], + ); + const forged = JSON.parse( + Buffer.from(terminal.next_changes_since, "base64url").toString(), + ); + forged.v = 3; + assert.throws( + () => + retainedBootstrapRecordIds( + Buffer.from(JSON.stringify(forged)).toString("base64url"), + "authorized", + ), + { code: "invalid_cursor" }, + ); + assert.throws( + () => retainedBootstrapRecordIds(terminalToken, "different-grant"), + { code: "invalid_cursor" }, + ); + const originalNow = Date.now(); + t.mock.method(Date, "now", () => originalNow + 5 * 60 * 1000 + 1); + assert.throws(() => retainedBootstrapRecordIds(terminalToken, "authorized"), { + code: "cursor_expired", + }); + t.mock.restoreAll(); + const restarted = startBootstrapPage({ + context: "authorized", + records, + changesCursor: token, + limit: 10, + }); + assert.ok(restarted.next_changes_since); + const restartedToken = restarted.next_changes_since; + bumpStorageGeneration(); + assert.throws( + () => retainedBootstrapRecordIds(restartedToken, "authorized"), + { code: "cursor_expired" }, + ); +}); + +test("retained bootstrap capacity rejects complete captures and counts terminal baselines", () => { + bumpStorageGeneration(); + const changesCursor = Buffer.from(JSON.stringify({ v: 1 })).toString( + "base64url", + ); + const capture = ( + records: Array<{ + record: { id: string; value?: string }; + eligibility: unknown; + }>, + ) => + startBootstrapPage({ + context: "capacity", + records, + changesCursor, + limit: 100, + }); + assert.throws( + () => + capture( + Array.from({ length: BOOTSTRAP_MAX_RECORDS + 1 }, (_, i) => ({ + record: { id: String(i) }, + eligibility: { id: String(i) }, + })), + ), + { code: "bootstrap_capacity_exceeded" }, + ); + assert.throws( + () => + capture([ + { + record: { id: "large", value: "x".repeat(33 * 1024 * 1024) }, + eligibility: { id: "large" }, + }, + ]), + { code: "bootstrap_capacity_exceeded" }, + ); + let last: ReturnType | undefined; + for (let i = 0; i < 32; i++) + last = capture([ + { record: { id: "retained" }, eligibility: { id: "retained" } }, + ]); + assert.ok(last?.next_changes_since); + assert.throws(() => capture([]), { code: "bootstrap_capacity_exceeded" }); + assert.deepEqual( + retainedBootstrapRecordIds(last.next_changes_since, "capacity"), + ["retained"], + ); + bumpStorageGeneration(); + assert.deepEqual(capture([]).data, []); + bumpStorageGeneration(); +}); From 17e47bb3ce1f35af20cdba9ae875fb74899ada8d Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Sun, 11 Oct 2026 17:14:43 -0500 Subject: [PATCH 3/3] fix(records): suppress retained h1 pages for records deleted after h1 A beginning session serves the h1 snapshot of each record across pages. The page filters only checked that the current row exists and is live. If a record was deleted after h1 and then reinserted with a new payload, the row was live again, so the old page served the pre-deletion payload. Reproduced on SQLite and PostgreSQL: capture h1 with a,b,c at limit=1, delete c, reinsert c, fetch the retained pages -> c at h1 was returned. Reinsertion clears deleted/deleted_at and record_changes may be pruned, so the current row cannot show a past deletion. Add a process-local deletion ledger (experimental-record-bootstrap.ts) with a monotonic epoch. Each session captures the epoch before it reads the snapshot and drops any record deleted after that epoch when it serves a page. Ordinary post-h1 edits are unaffected and still serve h1 values. The ledger is fed from the three delete sites (SQLite ingest, SQLite owner delete, PostgreSQL ingest) and keeps entries for twice the session TTL. It shares the sessions' process-local scope; a durable per-record deletion generation would need a records schema change and was not prototyped. The prototype doc records this limit. Tests: the bootstrap-pages subtest now reinserts c after the delete and requires that the retained pages omit it and a fresh beginning session include the new payload. It fails without the fix on both backends. The history subtest now also reads the current row and requires it to equal the replacement. Assisted-by: AI Signed-off-by: Tim Nunamaker --- docs/261010-ri-absent-fields-prototype.md | 6 +- .../server/experimental-record-bootstrap.ts | 86 ++++++++++++++++++- .../server/postgres-records.ts | 6 +- reference-implementation/server/records.ts | 7 +- .../test/experimental-record-updates.test.ts | 35 ++++++++ 5 files changed, 130 insertions(+), 10 deletions(-) diff --git a/docs/261010-ri-absent-fields-prototype.md b/docs/261010-ri-absent-fields-prototype.md index 47386a109..c174c98bf 100644 --- a/docs/261010-ri-absent-fields-prototype.md +++ b/docs/261010-ri-absent-fields-prototype.md @@ -45,7 +45,7 @@ The baseline SQLite no-op check compares serialized JSON. PostgreSQL compares `j Baseline reproductions failed with a missing append-only rejection and a SQLite reordered-object rewrite. Both pass after the patch. Independent review also reproduced a shared AJV registry collision for reused schema IDs and acceptance of removed required `constructor` through inheritance. The prototype isolates schema registration and requires own properties; both regressions now pass on both backends. Default-off tests also preserve legacy RECORD envelopes with ignored `set`/`remove` extensions and reject the explicit experimental marker. -The original beginning test correctly exposed an expiry after anchor pruning, but that is now a repaired prototype defect under `e8d4141d88`. A beginning session captures authorized current state at h1 without historical anchors or record writes. Saved pages retain h1 values across later ordinary edits and pruning; a later deletion suppresses that record on subsequent pages, as Section 8 requires. Follow-up sync retains an h0 baseline so unchanged ungranted fields do not trigger a delta and a later deletion can produce a tombstone. The implementation does not allocate record versions or restore history for duplicate retries. +The original beginning test correctly exposed an expiry after anchor pruning, but that is now a repaired prototype defect under `e8d4141d88`. A beginning session captures authorized current state at h1 without historical anchors or record writes. Saved pages retain h1 values across later ordinary edits and pruning; a later deletion suppresses that record on subsequent pages, as Section 8 requires, including when the key is reinserted with a new payload. The current row cannot show this (reinsertion clears `deleted`), so a process-local ledger records each deletion with a monotonic epoch, and a session suppresses any record deleted after the epoch it captured. A reinserted record then appears through a new beginning session or sync, not through the old h1 pages. The reproduction (`bootstrap pages preserve authorized h1 state after writes and history pruning`) failed on SQLite and PostgreSQL before this change and passes after it. Follow-up sync retains an h0 baseline so unchanged ungranted fields do not trigger a delta and a later deletion can produce a tombstone. The implementation does not allocate record versions or restore history for duplicate retries. The original complete-history oracle was wrong: it iterated only returned rows and accepted an empty result. The replacement records each changed-write version and the exact complete snapshot after initial/set-null/remove/replacement writes, then compares count, order and all payload values. Fault tests reject empty, missing, reordered and corrupted history. Separate test-copy faults delete actual history before its query and remove the saved cursor; both now fail on both backends. @@ -72,9 +72,11 @@ These inventories overlap; the counts are not additive. Pinned Chase balances in Follow-up targeted verification: 56 tests passed, zero failed or skipped, using SQLite and the actual PostgreSQL disposable-database oracle. This includes storage, HTTP and connector-scan tests. Flag-off helper/operation/boundary and PostgreSQL registry checks passed 105 tests without skips. Typecheck and checks on the new helper and prototype tests passed. The final full RI flag-off suite completed 1,124 file outputs: 11,269 assertions, 10,875 passed, zero failed, and 394 declared skips; runner exit0, memory-default concurrency4. This run includes the final ordering and later-page deletion repairs. PostgreSQL-dependent full-suite tests are declared skips; the separate targeted run uses the actual PostgreSQL oracle. Repository-wide `check` fails on the untouched baseline; shared files are not reformatted wholesale. +After the delete-then-reinsert repair, the two experimental-record test files ran 53 tests on SQLite and the actual PostgreSQL oracle with zero failures or skips. Reverting only the production change failed the bootstrap-pages subtest on both backends (`c was deleted and reinserted after h1, so the h1 snapshot of c must not be served`). The full RI flag-off suite (memory-default, concurrency 4) then exited 0 with 11,272 passing test events and no failure events. + Before fixes, expanded tests against reviewed production failed on sparse required-field projection, full-grant eligibility, beginning after pruning and disabled SQLite parsing. The original history-deletion and missing-cursor faults still passed all 72 tests. The corrected test copies reject both faults on both backends; four failing subtests plus their parents are expected fault-test results. Independent review found and reproduced tombstone-envelope loss and missing post-bootstrap baselines; their regressions pass after repair. These receipts are preserved in the local findings report. -Prototype sync state is process-local and expires after five minutes or a storage lifecycle change. Limits are 32 retained sessions, 10,000 records per captured state and 32 MiB aggregate serialized state. Capacity errors reject the whole capture; these are prototype limits, not Core limits or a claim of conformance for arbitrarily large streams. Numeric precision remains bounded by native JSON parsing. Live-account connector replay, cross-process concurrent writers, semantic embedding updates, a production client cache, and delete-then-reinsert between bootstrap pages remain unverified. +Prototype sync state is process-local and expires after five minutes or a storage lifecycle change. Limits are 32 retained sessions, 10,000 records per captured state and 32 MiB aggregate serialized state. Capacity errors reject the whole capture; these are prototype limits, not Core limits or a claim of conformance for arbitrarily large streams. Numeric precision remains bounded by native JSON parsing. Live-account connector replay, cross-process concurrent writers, semantic embedding updates and a production client cache remain unverified. The deletion ledger shares the sessions' process-local scope: it is not durable, does not see deletions made by another process, and covers the ingest and owner delete paths only. It keeps entries for ten minutes, twice the session lifetime, so its memory grows with the delete rate. A durable per-record deletion generation in `records` is the production alternative and was not prototyped. Delete-then-reinsert between bootstrap pages is covered for one reinsertion on both backends; repeated cycles and reinsertion across a restart are not tested. Preexisting incremental paths remain outside the new retained-state path: PostgreSQL historical incremental results are unpaginated, and SQLite historical incremental pages issue a sync token before the terminal page. New retained-state sessions use stable pagination and terminal-only sync tokens. Broader #383 token opacity and client tombstone-shape conformance are not claimed: the RI still exposes readable base64 cursor positions/horizons and uses its existing tombstone envelope. Those are subsequent feed-lane work, not a contradiction resolved by the two local spec fixes. The default-off SQLite duplicate path now avoids parsing current data; functional legacy RECORD extension and disabled-profile boundaries remain tested. PostgreSQL tests use disposable databases and the existing JSONB oracle without optional vector extensions. diff --git a/reference-implementation/server/experimental-record-bootstrap.ts b/reference-implementation/server/experimental-record-bootstrap.ts index f6251e584..cced16bd4 100644 --- a/reference-implementation/server/experimental-record-bootstrap.ts +++ b/reference-implementation/server/experimental-record-bootstrap.ts @@ -1,7 +1,10 @@ // Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 import { randomUUID } from "node:crypto"; -import { equalJsonValues } from "./experimental-record-updates.ts"; +import { + equalJsonValues, + experimentalRecordUpdatesEnabled, +} from "./experimental-record-updates.ts"; import { currentStorageGeneration } from "./storage-generation.ts"; // Prototype only: retain serialized authorized snapshots, independent of history @@ -16,6 +19,8 @@ interface Session { context: string; expiresAt: number; generation: number; + deletionEpoch: number; + scope: string; records: string[]; baseline: Map; } @@ -38,6 +43,69 @@ interface BootstrapCursor { } const sessions = new Map(); +// A retained page serves the record as captured at h1, so any deletion after h1 +// must suppress it, even if the key is later reinserted with a new payload. +// The current row cannot tell: reinsertion clears deleted and deleted_at, and +// record_changes may be pruned. This process-local ledger mirrors the +// process-local sessions it protects. An entry matters only to sessions +// captured before it. Those expire within SESSION_TTL_MS, so older entries are +// dropped. +const LEDGER_RETENTION_MS = 2 * SESSION_TTL_MS; +let deletionEpoch = 0; +const deletions = new Map< + string, + { at: number; epoch: number; generation: number } +>(); + +function deletionKey(scope: string, id: string): string { + return JSON.stringify([scope, id]); +} + +export function bootstrapScope( + connectorInstanceId: string, + stream: string, +): string { + return JSON.stringify([connectorInstanceId, stream]); +} + +/** Record a deletion. Call from every path that marks a record deleted. */ +export function noteRecordDeletion( + connectorInstanceId: string, + stream: string, + recordKey: string, +): void { + if (!experimentalRecordUpdatesEnabled()) return; + const now = Date.now(); + for (const [key, entry] of deletions) { + if (entry.at > now - LEDGER_RETENTION_MS) break; + deletions.delete(key); + } + const key = deletionKey( + bootstrapScope(connectorInstanceId, stream), + recordKey, + ); + deletions.delete(key); // re-insert so the map stays ordered by time + deletions.set(key, { + at: now, + epoch: ++deletionEpoch, + generation: currentStorageGeneration(), + }); +} + +/** Read before the snapshot query, so deletions that race the capture suppress. */ +export function currentDeletionEpoch(): number { + return deletionEpoch; +} + +function deletedAfterCapture(session: Session, id: string): boolean { + const entry = deletions.get(deletionKey(session.scope, id)); + return ( + entry !== undefined && + entry.generation === session.generation && + entry.epoch > session.deletionEpoch + ); +} + function bootstrapError( code: string, message: string, @@ -56,7 +124,7 @@ function pruneSessions(): void { } } -function page( +function page( id: string, session: Session, offset: number, @@ -68,7 +136,11 @@ function page( object: "list", data: session.records .slice(offset, end) - .map((record) => JSON.parse(record) as T), + .map((record) => JSON.parse(record) as T & { deleted?: unknown }) + .filter( + (record) => + record.deleted === true || !deletedAfterCapture(session, record.id), + ), has_more: hasMore, ...(hasMore ? { @@ -129,8 +201,12 @@ export function startBootstrapPage({ sinceCursor, removedRecords = [], order = "ASC", + scope, + deletionEpoch: capturedDeletionEpoch, }: { context: string; + scope: string; + deletionEpoch: number; records: Iterable>; changesCursor: string; limit: number; @@ -202,6 +278,8 @@ export function startBootstrapPage({ baseline, expiresAt: Date.now() + SESSION_TTL_MS, generation: currentStorageGeneration(), + deletionEpoch: capturedDeletionEpoch, + scope, }; const retainedBytes = [...sessions.values()].reduce( (total, existing) => total + existing.bytes, @@ -217,7 +295,7 @@ export function startBootstrapPage({ return page(id, session, 0, limit); } -export function continueBootstrapPage( +export function continueBootstrapPage( cursor: BootstrapCursor, context: string, limit: number, diff --git a/reference-implementation/server/postgres-records.ts b/reference-implementation/server/postgres-records.ts index 34445d1d4..d99088a57 100644 --- a/reference-implementation/server/postgres-records.ts +++ b/reference-implementation/server/postgres-records.ts @@ -13,7 +13,7 @@ import { createHash } from "node:crypto"; import { applyPartialInstruction, enforceAppendOnlyDuplicate, experimentalRecordUpdatesEnabled, isPartialInstruction, validateExperimentalSnapshot } from "./experimental-record-updates.ts"; -import { BOOTSTRAP_MAX_RECORDS, continueBootstrapPage, retainedBootstrapRecordIds, startBootstrapPage } from "./experimental-record-bootstrap.ts"; +import { BOOTSTRAP_MAX_RECORDS, bootstrapScope, continueBootstrapPage, currentDeletionEpoch, noteRecordDeletion, retainedBootstrapRecordIds, startBootstrapPage } from "./experimental-record-bootstrap.ts"; import { buildLimitClampedWarning, clampRecordsPageLimit, @@ -1743,6 +1743,7 @@ async function writePostgresIngestMutation({ }): Promise { let nextRecordJsonBytes = 0; if (op === "delete") { + noteRecordDeletion(connectorInstanceId, stream, recordKey); await client.query( `UPDATE records SET connector_id = $6, deleted = TRUE, deleted_at = $4, emitted_at = $4, version = $5 @@ -2447,6 +2448,7 @@ export async function postgresQueryRecords( const response = isBootstrapPage && bootstrapCursor ? continueBootstrapPage(bootstrapCursor, context, limit) : await withPostgresReadOnlyTransaction(async (client) => { + const deletionEpoch = currentDeletionEpoch(); const sinceCursor = retainedCursor?.bootstrap_baseline !== undefined ? String(requestParams.changes_since) : undefined; const previousIds = sinceCursor ? retainedBootstrapRecordIds(sinceCursor, context) : []; const horizon = await client.query("SELECT max_version FROM version_counter WHERE connector_instance_id = $1 AND stream = $2", [connectorInstanceId, stream]); @@ -2467,7 +2469,7 @@ export async function postgresQueryRecords( const removed = previousIds.length ? await client.query("SELECT record_key, emitted_at, deleted_at FROM records WHERE connector_instance_id = $1 AND stream = $2 AND record_key = ANY($3::text[])", [connectorInstanceId, stream, previousIds]) : { rows: [] }; const eligibilityFields = effective.eligibilityFields !== undefined ? effective.eligibilityFields : fields; return startBootstrapPage({ - context, records: rows.rows.map((row) => ({ record: responseRecord({ row, stream, fields, identity }), eligibility: projectFields(row.record_json, eligibilityFields ?? null) })), limit, order: orderDirection, + context, scope: bootstrapScope(connectorInstanceId, stream), deletionEpoch, records: rows.rows.map((row) => ({ record: responseRecord({ row, stream, fields, identity }), eligibility: projectFields(row.record_json, eligibilityFields ?? null) })), limit, order: orderDirection, changesCursor: encodeCursor({ v: maxVersion }), removedRecords: removed.rows.map((row) => deletedResponseRecord({ row, stream, identity })), ...(sinceCursor ? { sinceCursor } : {}), diff --git a/reference-implementation/server/records.ts b/reference-implementation/server/records.ts index e65a48074..8ef38de86 100644 --- a/reference-implementation/server/records.ts +++ b/reference-implementation/server/records.ts @@ -7,7 +7,7 @@ import { parseCoverageDiagnosticsStateSnapshot } from "./polyfill-connectors-run */ import { getDb } from "./db.ts"; import { applyPartialInstruction, enforceAppendOnlyDuplicate, equalJsonValues, experimentalRecordUpdatesEnabled, isPartialInstruction, recordUpdateError, validateExperimentalSnapshot, type ExperimentalStream } from "./experimental-record-updates.ts"; -import { continueBootstrapPage, retainedBootstrapRecordIds, startBootstrapPage, type BootstrapSnapshot } from "./experimental-record-bootstrap.ts"; +import { bootstrapScope, continueBootstrapPage, currentDeletionEpoch, noteRecordDeletion, retainedBootstrapRecordIds, startBootstrapPage, type BootstrapSnapshot } from "./experimental-record-bootstrap.ts"; import { assertGrantedManifestReadAuthority, assertManifestReadAuthority } from "./manifest-read-authority.ts"; import type { ComputedLexicalFields } from "./search.ts"; import type { ComputedSemanticEntries } from "./search-semantic.ts"; @@ -1353,6 +1353,7 @@ function writeRecordChange({ stream: string; }): void { if (op === "delete") { + noteRecordDeletion(connectorInstanceId, stream, recordKey); exec(referenceQueries.recordsIngestMarkRecordDeleted, [ effectiveEmittedAt, nextVersion, @@ -5612,6 +5613,7 @@ export async function queryRecords( const response: ListResponse = paginationCursor?.bootstrap_session !== undefined ? continueBootstrapPage(paginationCursor, context, ctx.limit) : writeTransaction(() => { + const deletionEpoch = currentDeletionEpoch(); const sinceCursor = retainedCursor?.bootstrap_baseline !== undefined ? String(requestParams.changes_since) : undefined; const previousIds = sinceCursor ? retainedBootstrapRecordIds(sinceCursor, context) : []; const horizon = getOne(referenceQueries.recordsIngestGetVersionCounter, [ctx.connectorInstanceId, stream])?.max_version ?? 0; @@ -5644,7 +5646,7 @@ export async function queryRecords( yield record; } } - return startBootstrapPage({ context, records: visibleRecords(), changesCursor: encodeChangesSinceCursor(horizon), limit: ctx.limit, order: ctx.order, removedRecords: removedRecords(), ...(sinceCursor ? { sinceCursor } : {}) }); + return startBootstrapPage({ context, scope: bootstrapScope(ctx.connectorInstanceId, stream), deletionEpoch, records: visibleRecords(), changesCursor: encodeChangesSinceCursor(horizon), limit: ctx.limit, order: ctx.order, removedRecords: removedRecords(), ...(sinceCursor ? { sinceCursor } : {}) }); }); if (paginationCursor?.bootstrap_session !== undefined) { response.data = response.data.filter((record) => { @@ -6523,6 +6525,7 @@ async function deleteRecordWithinCoordinator(storageTarget: RecordStorageTarget, maybeDeleteFault("after-version-allocation", { connectorId, connectorInstanceId, nextVersion, recordId, stream }); + noteRecordDeletion(connectorInstanceId, stream, recordId); exec(referenceQueries.recordsIngestMarkRecordDeleted, [now, nextVersion, connectorInstanceId, stream, recordId]); maybeDeleteFault("after-records-mutation", { connectorId, connectorInstanceId, nextVersion, recordId, stream }); diff --git a/reference-implementation/test/experimental-record-updates.test.ts b/reference-implementation/test/experimental-record-updates.test.ts index 7e892e513..cc5c57ba0 100644 --- a/reference-implementation/test/experimental-record-updates.test.ts +++ b/reference-implementation/test/experimental-record-updates.test.ts @@ -851,6 +851,15 @@ for (const backend of ["sqlite", "postgres"] as const) { stream: "bootstrap", }); await write({ key: "c", op: "delete", stream: "bootstrap" }); + await write({ + data: { + created_at: "2026-10-01T00:00:00Z", + id: "c", + required: "reinserted-after-h1", + }, + key: "c", + stream: "bootstrap", + }); await write({ data: { created_at: "2026-10-01T00:00:00Z", @@ -878,6 +887,7 @@ for (const backend of ["sqlite", "postgres"] as const) { assert.deepEqual( collected.map((record) => record.id), ["a", "b"], + "c was deleted and reinserted after h1, so the h1 snapshot of c must not be served", ); assert.deepEqual( collected.map((record) => record.data), @@ -932,7 +942,14 @@ for (const backend of ["sqlite", "postgres"] as const) { assert.deepEqual(restarted.data.map((record) => record.id).sort(), [ "a", "b", + "c", ]); + assert.equal( + restarted.data.find((record) => record.id === "c")?.data + ?.required, + "reinserted-after-h1", + "a fresh beginning session serves the reinserted payload", + ); }, ); await t.test( @@ -1210,6 +1227,18 @@ for (const backend of ["sqlite", "postgres"] as const) { version: outcome.version as number, }); } + const current = await getRecord( + target, + "state", + "history", + grant, + manifest, + ); + assert.deepEqual( + current.data, + replacement, + "RECORD replacement removes omitted fields from current storage", + ); const historySql = "SELECT record_json, version FROM record_changes WHERE stream = 'state' AND record_key = 'history' AND NOT deleted ORDER BY version"; const history = databaseUrl @@ -1591,6 +1620,8 @@ test("retained bootstrap state binds tokens and pages to context and storage lif })); const result = startBootstrapPage({ context: "authorized", + deletionEpoch: 0, + scope: "test", records, changesCursor: token, limit: 1, @@ -1640,6 +1671,8 @@ test("retained bootstrap state binds tokens and pages to context and storage lif t.mock.restoreAll(); const restarted = startBootstrapPage({ context: "authorized", + deletionEpoch: 0, + scope: "test", records, changesCursor: token, limit: 10, @@ -1666,6 +1699,8 @@ test("retained bootstrap capacity rejects complete captures and counts terminal ) => startBootstrapPage({ context: "capacity", + deletionEpoch: 0, + scope: "test", records, changesCursor, limit: 100,