Prototype (not for merge): absent fields and partial updates (#398)

PG-PROFILE-51-REPORT.md

@@ -167,8 +167,8 @@ tail:
 ## Scope counts and comparison baseline
 
 `scripts/run-tests.ts` discovers 1,034 profile files. The template equivalence
-registry is a different measure: 131 eligible plus 30 cold-required files,
-for **161** total. No current tracked source contains a 170-file claim.
+registry is a different measure: 133 eligible plus 30 cold-required files,
+for **163** total. The two #398 prototype files test record behavior, not schema bootstrap, and are template-eligible. No current tracked source contains a 170-file claim.
 
 The sibling report supplies only a memory-default cap-2 baseline (10,252
 assertions, 9,690 passed, 209 failed, 353 skipped; about 10 minutes). It is not

reference-implementation/operations/read-projection.ts

@@ -11,6 +11,24 @@ export function normalizeProjectionFields(value: unknown): string[] | null {
   return fields.length > 0 ? [...new Set(fields)] : null;
 }
 
+/** Sparse responses retain required fields without widening a frozen grant. */
+export function resolveResponseProjectionFields(
+  value: unknown,
+  requiredFields: readonly string[],
+  authorizedFields: readonly string[] | null = null
+): string[] | null {
+  const requestedFields = normalizeProjectionFields(value);
+  let fields = authorizedFields === null ? requestedFields : [...authorizedFields];
+  if (requestedFields && authorizedFields !== null) {
+    fields = requestedFields.filter((field) => authorizedFields.includes(field));
+  }
+  if (fields === null) {
+    return null;
+  }
+  const required = requiredFields.filter((field) => authorizedFields === null || authorizedFields.includes(field));
+  return [...new Set([...required, ...fields])];
+}
+
 export function projectRecordEnvelope<T extends Record<string, unknown>>(
   record: T,
   fields: readonly string[] | null | undefined

reference-implementation/operations/rs-records-detail/index.ts

@@ -36,7 +36,11 @@
  *   - response writing.
  */
 
-import { normalizeProjectionFields, projectRecordEnvelope } from "../read-projection.ts";
+import {
+  normalizeProjectionFields,
+  projectRecordEnvelope,
+  resolveResponseProjectionFields,
+} from "../read-projection.ts";
 
 export interface RecordDetailSourceDescriptor {
   id: string;
@@ -55,7 +59,7 @@ export type RecordDetailActor =
 
 export interface RecordDetailManifestStream {
   name: string;
-  schema?: { properties?: Record<string, unknown> };
+  schema?: { properties?: Record<string, unknown>; required?: string[] };
   [extra: string]: unknown;
 }
 
@@ -94,6 +98,8 @@ export interface RecordDetailDependencies {
    * carry blob refs).
    */
   decorateRecord: (record: Record<string, unknown>) => Record<string, unknown>;
+  /** Host-controlled prototype capability; omitted hosts retain legacy projection. */
+  experimentalRecordUpdatesEnabled?: () => boolean;
   /**
    * Resolved grant the host derived for this request. Owner actors get an
    * owner read-grant; client actors get the token's grant.
@@ -214,7 +220,15 @@ export async function executeRecordDetail(
 
   const record = projectRecordEnvelope(
     dependencies.decorateRecord(rawRecord),
-    normalizeProjectionFields(expandOptions.fields)
+    dependencies.experimentalRecordUpdatesEnabled?.()
+      ? resolveResponseProjectionFields(
+          expandOptions.fields,
+          manifestStream?.schema?.required ?? [],
+          input.actor.kind === "client"
+            ? (grant.streams.find((stream) => stream.name === input.streamName)?.fields ?? null)
+            : null
+        )
+      : normalizeProjectionFields(expandOptions.fields)
   );
 
   return {

reference-implementation/operations/rs-records-list/index.ts

@@ -34,7 +34,11 @@
  *     block fields populated from operation inputs and result counts.
  */
 
-import { normalizeProjectionFields, projectRecordEnvelope } from "../read-projection.ts";
+import {
+  normalizeProjectionFields,
+  projectRecordEnvelope,
+  resolveResponseProjectionFields,
+} from "../read-projection.ts";
 
 export interface RecordsListSourceDescriptor {
   id: string;
@@ -58,7 +62,7 @@ export type RecordsListActor =
  */
 export interface RecordsListManifestStream {
   name: string;
-  schema?: { properties?: Record<string, unknown> };
+  schema?: { properties?: Record<string, unknown>; required?: string[] };
   views?: Array<{ id: string; fields: string[] }>;
   [extra: string]: unknown;
 }
@@ -105,6 +109,8 @@ export interface RecordsListDependencies {
    * carry blob refs).
    */
   decorateRecord: (record: Record<string, unknown>) => Record<string, unknown>;
+  /** Host-controlled prototype capability; omitted hosts retain legacy projection. */
+  experimentalRecordUpdatesEnabled?: () => boolean;
   /**
    * Resolved grant the host derived for this request. For client actors this
    * is the token's grant; for owner actors the operation will overwrite this
@@ -296,10 +302,22 @@ export async function executeRecordsList(
 
   const rawResult = await dependencies.queryRecords(input.streamName, grant, input.requestParams, manifest);
 
-  const projectionFields = normalizeProjectionFields(input.requestParams.fields);
-  const decoratedData = rawResult.data.map((record) =>
-    projectRecordEnvelope(dependencies.decorateRecord(record), projectionFields)
-  );
+  const projectionFields = dependencies.experimentalRecordUpdatesEnabled?.()
+    ? resolveResponseProjectionFields(
+        input.requestParams.fields,
+        mStream?.schema?.required ?? [],
+        input.actor.kind === "client"
+          ? (grant.streams.find((stream) => stream.name === input.streamName)?.fields ?? null)
+          : null
+      )
+    : normalizeProjectionFields(input.requestParams.fields);
+  const decoratedData = rawResult.data.map((record) => {
+    const decorated = dependencies.decorateRecord(record);
+    if (dependencies.experimentalRecordUpdatesEnabled?.() === true && decorated.deleted === true) {
+      return decorated;
+    }
+    return projectRecordEnvelope(decorated, projectionFields);
+  });
   const decoratedResult: RecordsListQueryResult = {
     ...rawResult,
     data: decoratedData,

reference-implementation/scripts/postgres-template-eligibility.ts

@@ -83,6 +83,8 @@ export const POSTGRES_TEMPLATE_ELIGIBLE_FILES: readonly string[] = [
   "test/device-ingest-attempt-context-store.test.ts",
   "test/device-ingest-conformance.test.ts",
   "test/error-code-query-not-found.test.ts",
+  "test/experimental-record-updates-http.test.ts",
+  "test/experimental-record-updates.test.ts",
   "test/first-party-owner-credential-reuse.test.ts",
   "test/forward-evidence-debt-wired-probe.test.ts",
   "test/grant-lifecycle-expired-reporting.test.ts",

reference-implementation/scripts/scan-append-only-streams.ts

@@ -0,0 +1,106 @@
+// Copyright The PDP-Connect Contributors
+// SPDX-License-Identifier: Apache-2.0
+
+import { readdir, readFile } from "node:fs/promises";
+import { basename, resolve } from "node:path";
+import { pathToFileURL } from "node:url";
+
+interface ManifestStream {
+	name: string;
+	primary_key?: string[];
+	schema?: { properties?: Record<string, unknown> };
+	semantics: string;
+}
+
+interface ConnectorManifest {
+	connector_key?: string;
+	streams?: ManifestStream[];
+	version?: string;
+}
+
+const RUN_TIME_FIELD =
+	/^(?:fetched|collected|captured|observed|synced|scraped|exported)_at$/;
+const SOURCE_MUTABLE_FIELD =
+	/(?:^|_)(?:counts?|status|score|likes|followers|title|body|content|updated_at|edited_at)(?:_|$)/;
+const EXCLUDED_DIRECTORIES = new Set([".git", "node_modules", "dist", "build"]);
+
+/** Candidates require implementation evidence; a manifest cannot prove re-emission. */
+export function inspectAppendOnlyManifest(
+	manifest: ConnectorManifest,
+	manifestPath: string,
+) {
+	return (manifest.streams ?? [])
+		.filter((stream) => stream.semantics === "append_only")
+		.map((stream) => {
+			const fields = Object.keys(stream.schema?.properties ?? {});
+			return {
+				connector: manifest.connector_key ?? null,
+				fields,
+				manifest: manifestPath,
+				primary_key: stream.primary_key ?? [],
+				run_time_field_candidates: fields.filter((field) =>
+					RUN_TIME_FIELD.test(field),
+				),
+				source_mutable_field_candidates: fields.filter((field) =>
+					SOURCE_MUTABLE_FIELD.test(field),
+				),
+				stream: stream.name,
+				version: manifest.version ?? null,
+			};
+		});
+}
+
+async function manifestPaths(directory: string): Promise<string[]> {
+	const entries = await readdir(directory, { withFileTypes: true });
+	const paths: string[] = [];
+	const namedManifestDirectory = ["manifests", "seed-manifests"].includes(
+		basename(directory),
+	);
+	for (const entry of entries.sort((a, b) => a.name.localeCompare(b.name))) {
+		const path = resolve(directory, entry.name);
+		if (entry.isDirectory() && !EXCLUDED_DIRECTORIES.has(entry.name)) {
+			paths.push(...(await manifestPaths(path)));
+		} else if (
+			entry.isFile() &&
+			(entry.name === "manifest.json" ||
+				(namedManifestDirectory && entry.name.endsWith(".json")))
+		) {
+			paths.push(path);
+		}
+	}
+	return paths;
+}
+
+export async function scanAppendOnlyStreams(directory: string) {
+	const manifests = await manifestPaths(resolve(directory));
+	const streams = [];
+	let streamCount = 0;
+	for (const path of manifests) {
+		const manifest = JSON.parse(
+			await readFile(path, "utf8"),
+		) as ConnectorManifest;
+		streamCount += manifest.streams?.length ?? 0;
+		streams.push(...inspectAppendOnlyManifest(manifest, path));
+	}
+	return {
+		append_only_stream_count: streams.length,
+		limitation:
+			"Static manifest candidates only. Verify re-emission in connector code or a two-run replay.",
+		manifest_count: manifests.length,
+		stream_count: streamCount,
+		streams,
+	};
+}
+
+if (
+	process.argv[1] &&
+	import.meta.url === pathToFileURL(resolve(process.argv[1])).href
+) {
+	const directory = process.argv[2];
+	if (!directory) {
+		throw new Error(
+			"usage: node scripts/scan-append-only-streams.ts <connector-directory>",
+		);
+	}
+	console.log(JSON.stringify(await scanAppendOnlyStreams(directory), null, 2));
+}