From c05245fd5dd3d000369b7b47fc282b66941cba48 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Thu, 5 Nov 2026 15:46:08 -0600 Subject: [PATCH 1/5] feat(runtime): declare Collection Profile support, binding grammar and source identity A 0.2.0 manifest declares what it needs (bindings, capabilities, human interaction kinds) and what source it speaks for (source declaration). The runtime had no single place that decided whether it could honor that declaration, so a connector could be admitted and then fail late. The runtime now: - keeps one support declaration, `REFERENCE_COLLECTION_PROFILE_SUPPORT`, used by the registration gate and the run path, and serves it to the owner session at `GET /_ref/collection-profile` before any run; - parses `runtime_requirements.bindings` with the pinned grammar and schema (browser, desktop_session, filesystem, network and namespaced names). A required binding it cannot supply is refused before spawn, and START.bindings lists only what it supplies; - refuses a required capability it does not implement (BLOB is no longer tolerated), and stops advertising DETAIL_COVERAGE_KEY_SETS in START; - requires an INTERACTION.kind to be declared in `capabilities.human_interaction` instead of an interactive binding; - refuses a run before spawn with `failure_reason: protocol_capability_unsupported` when the declaration or a required capability cannot be honored; - keeps a registered connector's source identity once retained, and requires a first registration of an unmapped key to carry an explicit source_declaration. The installer verifies the source-declaration sidecar digest. Historical first-party mappings are pinned and checked by the registry generator. Existing tests now declare their fixture source and human interaction kinds explicitly. Assisted-by: AI Signed-off-by: Tim Nunamaker --- PG-PROFILE-51-REPORT.md | 2 +- .../as-polyfill-connector-register/index.ts | 18 +- .../runtime/binding-declarations.ts | 425 +++++++ .../runtime/classify-runtime-failure.ts | 3 +- .../runtime/collection-profile-support.ts | 21 +- .../runtime/connector-binding-grammar.d.mts | 22 + .../runtime/connector-binding-grammar.mjs | 534 ++++++++ .../runtime/connector-manifest.schema.json | 381 ++++++ reference-implementation/runtime/index.ts | 98 +- .../scripts/generate-connector-registry.ts | 29 +- .../legacy-source-binding-validation.ts | 40 + .../scripts/postgres-template-eligibility.ts | 5 + reference-implementation/server/auth.ts | 172 ++- .../connector-install/activation-authority.ts | 75 +- .../server/connector-install/index.ts | 142 ++- .../connector-install/local-source.test.ts | 145 ++- .../server/connector-install/local-source.ts | 66 +- .../server/connector-legacy-source-map.ts | 23 + .../server/connector-manifest-validation.ts | 126 +- .../generated/connector-registry.generated.ts | 3 + reference-implementation/server/index.ts | 9 + reference-implementation/server/transport.ts | 10 +- ...approval-review-artifact-rendering.test.ts | 2 +- .../test/approval-review-seam.test.ts | 10 +- .../test/attention-writer.test.ts | 2 +- ...oll-eligible-schedules-integration.test.ts | 147 ++- .../browser-enrollment-shell-route.test.ts | 5 +- ...dency-profile-conformance-postgres.test.ts | 2 +- ...int-dependency-profile-conformance.test.ts | 79 +- reference-implementation/test/cli.test.ts | 3 +- .../test/collection-profile-020.test.ts | 32 +- ...ction-profile-binding-declarations.test.ts | 399 ++++++ .../collection-profile-owner-journey.test.ts | 20 +- .../test/collection-profile.test.ts | 215 ++-- .../test/composed-origin.test.ts | 5 +- ...ector-install-registry-integration.test.ts | 216 ++++ ...nnector-install-source-declaration.test.ts | 577 +++++++++ .../connector-install-sql-persistence.test.ts | 284 +++++ .../test/connector-install.test.ts | 55 +- .../connector-manifest-validation.test.ts | 9 +- ...ector-registry-manifest-derivation.test.ts | 100 ++ ...connector-source-identity-boundary.test.ts | 1079 +++++++++++++++++ .../test/control-actions.test.ts | 6 +- ...detail-coverage-shortfall-severity.test.ts | 160 +-- .../test/event-spine.test.ts | 57 +- ...kpoint-dependency-multi-parent-scenario.ts | 337 ++--- .../connector-state-scheduler-conformance.ts | 4 +- .../helpers/fixture-source-declaration.ts | 9 +- .../test/helpers/legacy-upgrade.ts | 28 +- ...-store-connector-state-scheduler-driver.ts | 15 +- ...zero-connector-knowledge-data-load-scan.ts | 5 +- ...sqlite-connector-state-scheduler-driver.ts | 15 +- ...introspection-manifest-fail-closed.test.ts | 2 + .../test/legacy-upgrade-preservation.test.ts | 9 +- .../manifest-runtime-requirements.test.ts | 88 +- ...anifest-validation-runtime-refresh.test.ts | 39 +- .../test/owner-connection-intent.test.ts | 25 +- .../test/owner-connector-templates.test.ts | 3 +- ...ro-connector-knowledge-conformance.test.ts | 31 + .../test/run-interaction-control.test.ts | 21 +- .../run-interaction-stream-routes.test.ts | 24 +- .../runtime-requirements-validation.test.ts | 22 +- ...me-stream-collection-failed-commit.test.ts | 84 +- .../scheduler-static-secret-injection.test.ts | 118 +- .../test/scheduler.test.ts | 543 +++++---- 65 files changed, 6114 insertions(+), 1121 deletions(-) create mode 100644 reference-implementation/runtime/binding-declarations.ts create mode 100644 reference-implementation/runtime/connector-binding-grammar.d.mts create mode 100644 reference-implementation/runtime/connector-binding-grammar.mjs create mode 100644 reference-implementation/runtime/connector-manifest.schema.json create mode 100644 reference-implementation/scripts/legacy-source-binding-validation.ts create mode 100644 reference-implementation/server/connector-legacy-source-map.ts create mode 100644 reference-implementation/test/collection-profile-binding-declarations.test.ts create mode 100644 reference-implementation/test/connector-install-registry-integration.test.ts create mode 100644 reference-implementation/test/connector-install-source-declaration.test.ts create mode 100644 reference-implementation/test/connector-install-sql-persistence.test.ts create mode 100644 reference-implementation/test/connector-source-identity-boundary.test.ts diff --git a/PG-PROFILE-51-REPORT.md b/PG-PROFILE-51-REPORT.md index 1c29ee3c9..82acc66a1 100644 --- a/PG-PROFILE-51-REPORT.md +++ b/PG-PROFILE-51-REPORT.md @@ -168,7 +168,7 @@ tail: The completed profile receipt recorded 1,034 files; that historical receipt is unchanged. The template equivalence registry is a different measure: -159 eligible plus 37 cold-required files, for **196** total. The +162 eligible plus 37 cold-required files, for **199** total. The handle-generation, issuance-cutover, opaque-cursor and search-index repair tests remain registered. These counts are recomputed from the merged registry arrays. diff --git a/reference-implementation/operations/as-polyfill-connector-register/index.ts b/reference-implementation/operations/as-polyfill-connector-register/index.ts index 8ae884a99..dd4b547bc 100644 --- a/reference-implementation/operations/as-polyfill-connector-register/index.ts +++ b/reference-implementation/operations/as-polyfill-connector-register/index.ts @@ -33,8 +33,14 @@ export interface AsPolyfillConnectorRegisterDependencies { * whether or not that version is still current; null when none was recorded. */ getAcceptedDeclarationDigest: (connectorId: string, version: string) => Promise | string | null; - /** Produces the exact representation `registerConnector` persists, and its storage key. */ - normalizeManifestForStorage: (manifest: Record) => { + /** + * Produces the exact representation `registerConnector` persists, and its storage key. `context` carries + * the declaration already retained for the connector, when there is one. + */ + normalizeManifestForStorage: ( + manifest: Record, + context: { retainedSourceDeclaration?: unknown } + ) => { connectorId: string; storedManifest: Record; }; @@ -100,7 +106,13 @@ export async function executeAsPolyfillConnectorRegister( // equivocation and the previously accepted content is retained. Same // content is an idempotent re-registration; a new version is a new key. deps.validateManifest(manifest as Record); - const { connectorId, storedManifest } = deps.normalizeManifestForStorage(manifest as Record); + // The candidate is projected under the identity the stored row already holds, exactly as registerConnector + // does. Without it, a manifest that names no source would be refused on the fresh path. + const retained = await acceptedManifestFor(deps, connectorKey); + const { connectorId, storedManifest } = deps.normalizeManifestForStorage( + manifest as Record, + retained?.source_declaration === undefined ? {} : { retainedSourceDeclaration: retained.source_declaration } + ); const version = storedManifest.version; const incomingDigest = deps.declarationDigest(storedManifest); let acceptedDigest: string | null = null; diff --git a/reference-implementation/runtime/binding-declarations.ts b/reference-implementation/runtime/binding-declarations.ts new file mode 100644 index 000000000..5c860337d --- /dev/null +++ b/reference-implementation/runtime/binding-declarations.ts @@ -0,0 +1,425 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +// Collection Profile `runtime_requirements.bindings`: one shared validator for +// the registration gate, the stored-manifest read and the direct `runConnector` +// path, plus the pre-spawn placement plan that consumes its result. +// +// Normative sources (data-connectors a87edcdc099284e8582d24a60c9630474bd2af8b): +// docs/spec/collection-profile.md section 3.3 (registry, instance keys, +// interface, duplicate keys), 3.3.2-3.3.7 (constraint grammars, features, +// rationale), 3.3.8 (features), 3.3.9 (lifecycle outcome), and 3.4 +// (capabilities.human_interaction). The wire grammar is the canonical +// `connector-manifest.schema.json` and `connector-binding-grammar.mjs` beside +// this file, byte copies of the pinned data-connectors files. Nothing here +// re-derives their host, endpoint, IDNA or IPv6 rules. +// +// Scope. This module decides two things: whether a declaration is well formed, +// and whether this runtime can supply each required contract at all. It does +// not decide authority. Passing validation is never a grant, never enforcement, +// and never a claim that a channel is `enforced` (section 3.3.10). The +// effective-authority record, per-channel coverage and the weaker-coverage +// policy source are not decided here. + +import { Ajv2020, type ValidateFunction } from "ajv/dist/2020.js"; + +import { + canonicalizeConnectHost, + canonicalizeNetworkHost, + canonicalizeWebHost, + findDuplicateJsonKeys, + GrammarError, + normalizeFilesystemInputs, +} from "./connector-binding-grammar.mjs"; +import manifestSchema from "./connector-manifest.schema.json" with { type: "json" }; + +// --------------------------------------------------------------------------- +// Errors +// --------------------------------------------------------------------------- + +/** A declaration or manifest text that the canonical rules reject. */ +export class BindingDeclarationError extends Error { + readonly code: "invalid_binding_declaration" | "duplicate_json_key"; + + constructor(message: string, code: BindingDeclarationError["code"] = "invalid_binding_declaration") { + super(message); + this.name = "BindingDeclarationError"; + this.code = code; + } +} + +/** A required binding this runtime cannot supply. Refused before spawn. */ +export class BindingPlacementError extends Error { + readonly code = "unsupported_required_binding" as const; + readonly bindingKeys: readonly string[]; + + constructor(failures: readonly { key: string; reason: string }[]) { + const detail = failures.map((failure) => `${failure.key} (${failure.reason})`).join("; "); + super(`Runtime cannot satisfy required binding: ${failures.map((failure) => failure.key).join(", ")}. ${detail}`); + this.name = "BindingPlacementError"; + this.bindingKeys = failures.map((failure) => failure.key); + } +} + +// --------------------------------------------------------------------------- +// Canonical registry and grammar, read from the vendored schema +// --------------------------------------------------------------------------- + +/** Registry binding names (section 3.3). Read from the canonical schema. */ +export const REGISTRY_BINDING_NAMES: ReadonlySet = new Set(manifestSchema.$defs.registryBinding.enum); + +/** Namespaced extension binding name `/` (section 3.3). */ +const NAMESPACED_BINDING_NAME = new RegExp(namespacedBindingPattern()); + +function namespacedBindingPattern(): string { + const branch = manifestSchema.$defs.bindingName.anyOf.find((entry) => "pattern" in entry) as + | { pattern: string } + | undefined; + if (branch === undefined) { + throw new Error("connector-manifest.schema.json: bindingName has no namespaced pattern"); + } + return branch.pattern; +} + +// --------------------------------------------------------------------------- +// Raw JSON text +// --------------------------------------------------------------------------- + +/** + * Parses manifest JSON text, rejecting any object key that repeats anywhere in + * the text, including nested keys and escaped spellings of the same key. The + * duplicate check runs on the text as written, before `JSON.parse` collapses + * repeats to the last value. Re-serializing an already-parsed object cannot + * recover this, so callers must pass the original text when they have it. + */ +export function parseManifestJsonText(text: string): unknown { + let duplicates: Array<{ path: string; key: string }>; + try { + duplicates = findDuplicateJsonKeys(text); + } catch (error) { + if (error instanceof GrammarError) { + throw new SyntaxError(`manifest JSON is malformed: ${error.message}`, { cause: error }); + } + throw error; + } + const [first] = duplicates; + if (first !== undefined) { + throw new BindingDeclarationError( + `manifest JSON repeats object key "${first.key}" at ${first.path}; a repeated key is rejected, not last-wins`, + "duplicate_json_key" + ); + } + return JSON.parse(text); +} + +// --------------------------------------------------------------------------- +// Shape validation (canonical JSON Schema) and canonical grammar checks +// --------------------------------------------------------------------------- + +export interface BindingInstance { + /** Deep copy of the declaration. Binding-specific fields are preserved. */ + readonly declaration: Readonly>; + /** Stated interface version, exactly as declared. Absent means unspecified. */ + readonly interface: string | undefined; + /** Instance key exactly as declared. */ + readonly key: string; + /** Resolved kind: the key for shorthand, `kind` for a free-form key. */ + readonly kind: string; + readonly required: boolean; +} + +let schemaValidator: ValidateFunction | null = null; + +function bindingsValidator(): ValidateFunction { + if (schemaValidator === null) { + schemaValidator = new Ajv2020({ allErrors: false, strict: true }).compile(manifestSchema); + } + return schemaValidator; +} + +const RUNTIME_REQUIREMENTS_BINDINGS_PREFIX = /^\/runtime_requirements\/bindings/; + +function describeSchemaError(error: { instancePath: string; message?: string }): string { + const relative = error.instancePath.replace(RUNTIME_REQUIREMENTS_BINDINGS_PREFIX, "").replaceAll("/", "."); + return `runtime_requirements.bindings${relative} ${error.message ?? "is invalid"}`; +} + +const REGEX_SPECIAL_CHARACTERS = /[.*+?^${}()|[\]\\]/g; + +function escapeRegExp(value: string): string { + return value.replace(REGEX_SPECIAL_CHARACTERS, "\\$&"); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function fail(message: string): never { + throw new BindingDeclarationError(message); +} + +/** Canonical host check: the stated form must equal its own canonicalization. */ +function assertCanonicalHost(label: string, raw: unknown, canonicalize: (value: string) => string): void { + if (typeof raw !== "string") { + return; + } + let canonical: string; + try { + canonical = canonicalize(raw); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + fail(`${label} "${raw}" is not a valid canonical form: ${reason}`); + } + if (canonical !== raw) { + fail(`${label} "${raw}" must be stated in canonical form "${canonical}"`); + } +} + +function assertNonEmptyRationaleWhenExceptional( + key: string, + declaration: Record, + exceptional: boolean +): void { + const { rationale } = declaration; + if (rationale !== undefined && (typeof rationale !== "string" || rationale.trim().length === 0)) { + fail(`runtime_requirements.bindings.${key}.rationale must be a non-empty string when declared`); + } + if (exceptional && (typeof rationale !== "string" || rationale.trim().length === 0)) { + fail( + `runtime_requirements.bindings.${key} declares an exceptional grant and must carry a non-empty rationale (section 3.3.13)` + ); + } +} + +/** The interface version must be `@` with no leading zero. */ +function assertInterfaceVersion(key: string, kind: string, declaration: Record): void { + const iface = declaration.interface; + if (iface === undefined) { + return; + } + const expected = new RegExp(`^${escapeRegExp(kind)}@[1-9][0-9]*$`); + if (typeof iface !== "string" || !expected.test(iface)) { + fail(`runtime_requirements.bindings.${key}.interface must be "${kind}@" with no leading zero`); + } +} + +/** Both declared filesystem forms must agree (section 3.3.5), and env_var names must be unique. */ +function assertFilesystemInputs(key: string, declaration: Record): void { + let inputs: unknown[] | undefined; + try { + inputs = normalizeFilesystemInputs(declaration as { inputs?: unknown[]; constraints?: { inputs?: unknown[] } }); + } catch (error) { + fail(`runtime_requirements.bindings.${key}: ${error instanceof Error ? error.message : String(error)}`); + } + const envVars = new Set(); + for (const input of inputs ?? []) { + const envVar = isRecord(input) ? input.env_var : undefined; + if (typeof envVar === "string") { + if (envVars.has(envVar)) { + fail(`runtime_requirements.bindings.${key} repeats filesystem input env_var ${envVar}`); + } + envVars.add(envVar); + } + } +} + +/** + * Canonical-form checks for browser and network host constraints. Returns true + * when the declaration is an exceptional grant (section 3.3.13). + */ +function assertHostConstraints(kind: string, constraints: Record): boolean { + if (kind === "browser") { + for (const [index, host] of ((constraints.navigate as unknown[] | undefined) ?? []).entries()) { + assertCanonicalHost(`browser.constraints.navigate[${index}]`, host, canonicalizeWebHost); + } + for (const [index, host] of ((constraints.connect as unknown[] | undefined) ?? []).entries()) { + assertCanonicalHost(`browser.constraints.connect[${index}]`, host, canonicalizeConnectHost); + } + return false; + } + if (kind !== "network") { + return false; + } + let exceptional = false; + const hosts = (constraints.hosts as unknown[] | undefined) ?? []; + for (const [index, entry] of hosts.entries()) { + if (isRecord(entry)) { + if (entry.allow_private === true) { + exceptional = true; + } + continue; + } + assertCanonicalHost(`network.constraints.hosts[${index}]`, entry, canonicalizeNetworkHost); + } + return exceptional; +} + +/** Constraint, interface and rationale checks the JSON Schema cannot express. */ +function assertInstanceContract(key: string, kind: string, declaration: Record): void { + assertInterfaceVersion(key, kind, declaration); + const constraints = isRecord(declaration.constraints) ? declaration.constraints : {}; + if (kind === "filesystem") { + assertFilesystemInputs(key, declaration); + } + let exceptional = assertHostConstraints(kind, constraints); + if (kind === "desktop_session" && Array.isArray(constraints.items) && constraints.items.length > 0) { + // Access to a named sensitive resource is an exceptional grant (section 3.3.13). + exceptional = true; + } + assertNonEmptyRationaleWhenExceptional(key, declaration, exceptional); +} + +/** + * Validates `runtime_requirements.bindings` and resolves each instance. + * + * Absent (`undefined`) bindings yield an empty list. `null` is rejected: the + * canonical schema types the map as an object. The result keeps declaration + * order, one entry per instance key, and never mutates its input. + */ +export function normalizeBindingDeclarations(bindings: unknown): readonly BindingInstance[] { + if (bindings === undefined) { + return []; + } + const validate = bindingsValidator(); + if (!validate({ runtime_requirements: { bindings } })) { + const first = validate.errors?.[0]; + fail(first ? describeSchemaError(first) : "runtime_requirements.bindings is invalid"); + } + if (!isRecord(bindings)) { + fail("runtime_requirements.bindings must be an object"); + } + + return Object.entries(bindings).map(([key, value]) => { + if (!isRecord(value)) { + fail(`runtime_requirements.bindings.${key} must be an object`); + } + const shorthand = REGISTRY_BINDING_NAMES.has(key) || NAMESPACED_BINDING_NAME.test(key); + // The schema has already rejected a shorthand key that carries `kind`, and + // a free-form key without one. Here the kind is resolved from the key. + const kind = shorthand ? key : (value.kind as string); + assertInstanceContract(key, kind, value); + return Object.freeze({ + declaration: Object.freeze(structuredClone(value)), + interface: typeof value.interface === "string" ? value.interface : undefined, + key, + kind, + required: value.required as boolean, + }); + }); +} + +// --------------------------------------------------------------------------- +// Pre-spawn placement +// --------------------------------------------------------------------------- + +/** + * What this runtime supplies. It publishes no feature and enforces no + * constraint, so any declared feature or constraint is an unmet contract. + * `kinds` matches the static START.bindings advertisement in runtime/index.ts. + */ +export interface BindingSupport { + readonly constraintKinds: ReadonlySet; + readonly features: ReadonlySet; + readonly interfaceVersions: ReadonlyMap>; + readonly kinds: ReadonlySet; +} + +export const RUNTIME_BINDING_SUPPORT: BindingSupport = { + constraintKinds: new Set(), + features: new Set(), + interfaceVersions: new Map([ + ["browser", new Set([1])], + ["filesystem", new Set([1])], + ["network", new Set([1])], + ]), + kinds: new Set(["browser", "filesystem", "network"]), +}; + +/** Why an instance cannot be supplied, or `null` when the runtime supplies its contract. */ +export function unsupportedReason(instance: BindingInstance, support: BindingSupport): string | null { + if (!support.kinds.has(instance.kind)) { + return `kind ${instance.kind} is not implemented by this runtime`; + } + if (instance.interface !== undefined) { + const version = Number(instance.interface.slice(instance.interface.indexOf("@") + 1)); + if (!support.interfaceVersions.get(instance.kind)?.has(version)) { + return `interface ${instance.interface} is not implemented by this runtime`; + } + } + const { features } = instance.declaration; + if (Array.isArray(features) && features.length > 0) { + const unmet = features.filter((feature) => !support.features.has(String(feature))); + if (unmet.length > 0) { + return `features ${unmet.join(", ")} are not provided by this runtime`; + } + } + const { constraints } = instance.declaration; + const declaresConstraints = isRecord(constraints) && Object.keys(constraints).length > 0; + const declaresInputs = Array.isArray(instance.declaration.inputs) && instance.declaration.inputs.length > 0; + if ((declaresConstraints || declaresInputs) && !support.constraintKinds.has(instance.kind)) { + return "constraints are declared, and this runtime does not enforce them"; + } + if (instance.key !== instance.kind) { + return "named binding instances need per-instance handles, which this runtime does not provide"; + } + return null; +} + +export interface BindingPlacement { + /** Optional instances this runtime omits, with the reason. */ + readonly omitted: readonly { instance: BindingInstance; reason: string }[]; + /** Instances whose contract this runtime supplies. */ + readonly placed: readonly BindingInstance[]; +} + +/** + * Plans placement before spawn. A required instance this runtime cannot supply + * throws `BindingPlacementError`, and no connector code runs. An optional one + * may be omitted (section 3.3), and the omission must show in START. + */ +export function planBindingPlacement( + instances: readonly BindingInstance[], + support: BindingSupport = RUNTIME_BINDING_SUPPORT +): BindingPlacement { + const placed: BindingInstance[] = []; + const omitted: { instance: BindingInstance; reason: string }[] = []; + const failures: { key: string; reason: string }[] = []; + for (const instance of instances) { + const reason = unsupportedReason(instance, support); + if (reason === null) { + placed.push(instance); + } else if (instance.required) { + failures.push({ key: instance.key, reason }); + } else { + omitted.push({ instance, reason }); + } + } + if (failures.length > 0) { + throw new BindingPlacementError(failures); + } + return { omitted, placed }; +} + +/** + * START.bindings for this run keeps the keys of resolved instances. An omitted + * instance cannot borrow another instance's availability through their shared + * kind. Undeclared kinds keep the existing static availability advertisement; + * that advertisement is not a grant or a claim of channel enforcement. + */ +export function startBindingsFor( + instances: readonly BindingInstance[], + plan: BindingPlacement, + support: BindingSupport = RUNTIME_BINDING_SUPPORT +): Record> { + const declaredKinds = new Set(instances.map((instance) => instance.kind)); + const advertised: Record> = {}; + for (const kind of [...support.kinds].sort()) { + if (!declaredKinds.has(kind)) { + advertised[kind] = {}; + } + } + for (const instance of plan.placed) { + advertised[instance.key] = {}; + } + return advertised; +} diff --git a/reference-implementation/runtime/classify-runtime-failure.ts b/reference-implementation/runtime/classify-runtime-failure.ts index 9a61778dc..4dd615b03 100644 --- a/reference-implementation/runtime/classify-runtime-failure.ts +++ b/reference-implementation/runtime/classify-runtime-failure.ts @@ -31,7 +31,8 @@ export function classifyRuntimeFailure(err: unknown): FailureReason { } if ( message === "Connector emitted INTERACTION while already waiting" || - message === "Connector emitted INTERACTION but START.bindings omitted interactive" || + message === "Connector emitted INTERACTION but no interaction handler is configured" || + message.startsWith("Connector emitted INTERACTION.kind not declared in capabilities.human_interaction:") || message.includes(" while waiting for INTERACTION_RESPONSE") || message.startsWith("Connector emitted invalid INTERACTION.") || message.startsWith("Connector emitted invalid STATE.") || diff --git a/reference-implementation/runtime/collection-profile-support.ts b/reference-implementation/runtime/collection-profile-support.ts index a00996ed4..a567194ab 100644 --- a/reference-implementation/runtime/collection-profile-support.ts +++ b/reference-implementation/runtime/collection-profile-support.ts @@ -1,8 +1,9 @@ // Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 -// Admission of a manifest's Collection Profile declaration on the run path -// (runtime/index.ts). +// Admission of a manifest's Collection Profile declaration, shared by the +// registration gate (server/connector-manifest-validation.ts) and the run +// path (runtime/index.ts) so both apply one rule set. // // Normative source: data-connectors docs spec sections 3.4 (protocol_version // and capabilities) and 3.8 (diagnostic_descriptors). A manifest that declares @@ -12,9 +13,25 @@ import { COLLECTION_PROFILE_VERSIONS, collectionProfileDeclarationError, OBSERVATION_CAPABILITY, + STREAM_EVIDENCE_CAPABILITY, } from "@pdpp/connector-protocol/connector-runtime-protocol"; import { type DiagnosticDescriptors, ObservationContractError, validateDiagnosticDescriptors } from "./observations.ts"; +/** Implemented wire features, with outstanding full-profile obligations disclosed. */ +export const REFERENCE_COLLECTION_PROFILE_SUPPORT = Object.freeze({ + compatible_protocol_versions: Object.freeze([...COLLECTION_PROFILE_VERSIONS]), + conformance: Object.freeze({ outstanding: Object.freeze(["binding_authority"]), status: "partial" }), + protocol_capabilities: Object.freeze([OBSERVATION_CAPABILITY, STREAM_EVIDENCE_CAPABILITY]), + protocol_version: "0.2.0", +}); + +/** + * `failure_reason` of a run refused before spawn because the manifest's + * Collection Profile declaration or required capabilities cannot be honored + * (Section 3.4: the runtime must not start that connector). + */ +export const PROTOCOL_CAPABILITY_UNSUPPORTED_FAILURE_REASON = "protocol_capability_unsupported"; + export interface CollectionProfileDeclaration { readonly descriptors: DiagnosticDescriptors; readonly observationDeclared: boolean; diff --git a/reference-implementation/runtime/connector-binding-grammar.d.mts b/reference-implementation/runtime/connector-binding-grammar.d.mts new file mode 100644 index 000000000..19da639d4 --- /dev/null +++ b/reference-implementation/runtime/connector-binding-grammar.d.mts @@ -0,0 +1,22 @@ +// Type declarations for the vendored `connector-binding-grammar.mjs`, a byte +// copy of data-connectors schemas/connector-binding-grammar.mjs at commit +// a87edcdc099284e8582d24a60c9630474bd2af8b. Declarations mirror the exports as +// they are written in that module; they add no behaviour. + +export class GrammarError extends Error {} + +export function canonicalizeWebHost(raw: string, options?: { schemes?: Set }): string; +export function isCanonicalWebHost(raw: string, options?: { schemes?: Set }): boolean; +export function hostMatchesWildcardApex(apex: string, candidateHost: string): boolean; +export function canonicalizeConnectHost(raw: string): string; +export function isCanonicalConnectHost(raw: string): boolean; +export function canonicalizeEndpoint(raw: string): string; +export function isCanonicalEndpoint(raw: string): boolean; +export function canonicalizeNetworkHost(raw: string): string; +export function isCanonicalNetworkHost(raw: string): boolean; +export function deepEqualJson(a: unknown, b: unknown): boolean; +export function normalizeFilesystemInputs(instance: { + inputs?: unknown[]; + constraints?: { inputs?: unknown[] }; +}): unknown[] | undefined; +export function findDuplicateJsonKeys(jsonText: string): Array<{ path: string; key: string }>; diff --git a/reference-implementation/runtime/connector-binding-grammar.mjs b/reference-implementation/runtime/connector-binding-grammar.mjs new file mode 100644 index 000000000..c6c7dbf09 --- /dev/null +++ b/reference-implementation/runtime/connector-binding-grammar.mjs @@ -0,0 +1,534 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Typed parsers and canonicalizers for the binding constraint grammars + * defined in docs/spec/collection-profile.md Section 3.3. JSON Schema can + * check shape (strings, required members, enums) but cannot perform IDNA + * canonicalization, IPv6 normalization, or "is this string already in its + * one canonical form" equivalence checks. This module does that work, and + * schemas/connector-manifest.schema.json's string patterns are deliberately + * written to accept only the output these functions already produce. + * + * Node's `url.domainToASCII` was evaluated for the IDNA/UTS46 step. This + * module uses the WHATWG `URL` constructor instead, because the same + * host-parsing algorithm that backs `domainToASCII` also normalizes IPv4 + * and IPv6 literals and drops a scheme's default port, so one call covers + * every canonicalization rule in the web-origin grammar instead of hand + * -rolling IPv6 compression or IPv4 interpretation separately. Both paths + * share the same underlying WHATWG URL Standard host parser, so the result + * is the one `domainToASCII` would give for the host component alone. + * + * Grammar (docs/spec/collection-profile.md Section 3.3.2): + * `[scheme://]host[:port]`, exact or one leading `*.` wildcard label, no + * path, query, or fragment. Section 3.3.3 (browser) and 3.3.4 (network) layer + * their own scheme sets and endpoint rules on top of this shared grammar. + */ + +import { URL } from "node:url"; +import net from "node:net"; + +export class GrammarError extends Error { + constructor(message) { + super(message); + this.name = "GrammarError"; + } +} + +// Schemes `browser.navigate` accepts: a navigation always loads a document. +const NAVIGATE_SCHEMES = new Set(["http", "https"]); + +// Schemes `browser.connect` accepts: a page can open a WebSocket from a +// document loaded over http(s), so `connect` is scheme-aware in a way +// `navigate` is not (Section 3.3.3, live capture evidence from chatgpt: +// wss://ws.chatgpt.com, wss://codex-cloud-backend.chatgpt.com). +const CONNECT_SCHEMES = new Set(["http", "https", "ws", "wss"]); + +// network.hosts without an explicit scheme, and any bare http(s) entry, +// reuses the same grammar as `navigate` (Section 3.3.4). +const WEB_SCHEMES = NAVIGATE_SCHEMES; + +// Well-known default ports, used to drop a redundant explicit port from the +// canonical form. Only schemes with one conventional default port are +// listed; an endpoint whose scheme is not here must always state its port +// (Section 3.3.4). ws/wss share http/https's ports because a WebSocket +// upgrade begins as an http(s) request. +// Object.create(null) has no prototype chain, so a scheme named +// "constructor", "__proto__", "toString", or "hasOwnProperty" cannot read an +// inherited value through `DEFAULT_PORTS[scheme]` and be mistaken for a +// scheme with a real default port. A plain object literal is vulnerable to +// exactly this (schemas/connector-binding-grammar.test.mjs tests it). +const DEFAULT_PORTS = Object.assign(Object.create(null), { + http: 80, + https: 443, + ws: 80, + wss: 443, + imap: 143, + imaps: 993, + pop3: 110, + pop3s: 995, + smtp: 25, + smtps: 465, + submission: 587, + ldap: 389, + ldaps: 636, + ftp: 21, + ftps: 990, +}); + +function assertNoWhitespaceOrControl(raw) { + if (typeof raw !== "string" || raw.length === 0) { + throw new GrammarError("must be a non-empty string"); + } + for (let i = 0; i < raw.length; i += 1) { + const code = raw.charCodeAt(i); + if (code <= 0x20 || code === 0x7f) { + throw new GrammarError("must not contain whitespace or control characters"); + } + } +} + +function splitScheme(raw) { + const match = raw.match(/^([A-Za-z][A-Za-z0-9+.-]*):\/\//); + if (!match) return { scheme: null, rest: raw }; + return { scheme: match[1].toLowerCase(), rest: raw.slice(match[0].length) }; +} + +function splitWildcard(rest) { + if (rest.startsWith("*.")) { + const apex = rest.slice(2); + if (apex.includes("*")) { + throw new GrammarError("a wildcard must be exactly one leading '*.' label"); + } + return { wildcard: true, rest: apex }; + } + if (rest.includes("*")) { + throw new GrammarError("a wildcard must be exactly one leading '*.' label"); + } + return { wildcard: false, rest }; +} + +function rejectPathQueryFragment(rest) { + if (/[/?#]/.test(rest)) { + throw new GrammarError("must not contain a path, query, or fragment"); + } +} + +function rejectUserinfo(rest) { + if (rest.includes("@")) { + throw new GrammarError("must not contain userinfo; this grammar has no '@' component"); + } +} + +/** + * Splits `rest` (no scheme, no wildcard, no path) into a host token and an + * optional port token, respecting bracketed IPv6 literals. + */ +function splitHostPort(rest) { + if (rest.startsWith("[")) { + const end = rest.indexOf("]"); + if (end === -1) throw new GrammarError("unterminated IPv6 literal"); + const host = rest.slice(0, end + 1); + const tail = rest.slice(end + 1); + if (tail.length === 0) return { host, port: null, isIPv6Literal: true }; + if (!tail.startsWith(":")) throw new GrammarError("unexpected characters after an IPv6 literal"); + return { host, port: tail.slice(1), isIPv6Literal: true }; + } + const colonIdx = rest.lastIndexOf(":"); + if (colonIdx === -1) return { host: rest, port: null, isIPv6Literal: false }; + return { host: rest.slice(0, colonIdx), port: rest.slice(colonIdx + 1), isIPv6Literal: false }; +} + +function validatePort(port) { + if (!/^[0-9]+$/.test(port)) throw new GrammarError("port must be digits only"); + const value = Number(port); + if (!Number.isInteger(value) || value < 1 || value > 65535 || String(value) !== port) { + throw new GrammarError("port must be a canonical integer from 1 to 65535"); + } + return value; +} + +function stripTrailingDot(host) { + if (host.startsWith("[") || !host.endsWith(".")) return host; + const stripped = host.slice(0, -1); + if (stripped.length === 0 || stripped.endsWith(".")) { + throw new GrammarError("host has more than one trailing dot, or is empty after stripping it"); + } + return stripped; +} + +/** + * Validates DNS label syntax (LDH rule) for a non-IP hostname. The WHATWG + * URL host parser accepts some strings this grammar must reject, such as an + * empty label from a doubled dot (`a..b.example`). + */ +function assertValidDnsLabels(host) { + if (host.length > 253) throw new GrammarError("host exceeds 253 characters"); + for (const label of host.split(".")) { + if (!/^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/.test(label)) { + throw new GrammarError(`invalid host label '${label}'`); + } + if (label.length > 63) throw new GrammarError(`host label '${label}' exceeds 63 octets`); + } +} + +/** + * Canonicalizes the host[:port] portion shared by the web and endpoint + * grammars. `scheme` is the already-lowercased scheme, or null when the web + * grammar's scheme is omitted. Returns `{ host, isIP, port }`, where `port` + * is the canonical port string or `""` when the port is implied. + */ +function canonicalizeHostAndPort(rest, { scheme, allowWildcard }) { + const { wildcard, rest: afterWildcard } = allowWildcard + ? splitWildcard(rest) + : { wildcard: false, rest: (() => { if (rest.includes("*")) throw new GrammarError("a wildcard is not allowed here"); return rest; })() }; + rejectPathQueryFragment(afterWildcard); + rejectUserinfo(afterWildcard); + const { host: hostToken, port: portToken, isIPv6Literal } = splitHostPort(afterWildcard); + if (hostToken.length === 0) throw new GrammarError("host must not be empty"); + if (wildcard && isIPv6Literal) throw new GrammarError("a wildcard cannot apply to an IP literal"); + + const portValue = portToken === null ? null : validatePort(portToken); + const hostForUrl = stripTrailingDot(hostToken); + + // Always probe through a "special" WHATWG scheme (any of http, https, ws, + // wss, ftp, file) so the URL constructor runs its full host algorithm + // (IDNA, IPv4, IPv6). For a non-special scheme such as "imaps", the same + // constructor treats the host as opaque and skips all of that -- compared + // in schemas/connector-binding-grammar.test.mjs. The real scheme is used + // below only to decide default-port dropping, never for this probe. + let probe; + try { + probe = new URL(`https://${hostForUrl}/`); + } catch { + throw new GrammarError(`invalid host '${hostToken}'`); + } + const canonicalHost = probe.hostname; + const isIP = net.isIP(canonicalHost.replace(/^\[|\]$/g, "")) !== 0; + + if (!isIPv6Literal) { + const bareInput = hostForUrl.toLowerCase(); + if (net.isIPv4(canonicalHost) && bareInput !== canonicalHost) { + throw new GrammarError(`'${hostToken}' is an IPv4-looking label; use the exact dotted-quad address`); + } + } + if (!isIP) { + assertValidDnsLabels(canonicalHost); + } + if (wildcard) { + if (isIP) throw new GrammarError("a wildcard cannot apply to an IP literal"); + if (canonicalHost.split(".").length < 2) { + throw new GrammarError("a wildcard apex must have at least one more label"); + } + } + + let canonicalPort = ""; + if (scheme !== null) { + const schemeDefault = DEFAULT_PORTS[scheme]; + if (portValue !== null && portValue !== schemeDefault) canonicalPort = String(portValue); + } else if (portValue !== null) { + canonicalPort = String(portValue); + } + + return { host: canonicalHost, isIP, wildcard, port: canonicalPort }; +} + +/** + * Canonicalizes a web-origin or web-host constraint: `browser.navigate`, an + * HTTP(S) entry in `network`'s `hosts` list (Section 3.3.4), or (with + * `schemes: CONNECT_SCHEMES`) `browser.connect` (Section 3.3.3), which also + * accepts `ws`/`wss` because a page can open a WebSocket from a document + * loaded over http(s). + * + * Matching semantics (stated here because the grammar alone does not fix + * them): an omitted scheme matches every scheme `schemes` allows. An omitted + * port matches the matching scheme's own default port (80 for http/ws, 443 + * for https/wss) -- it does NOT mean "any port", unlike a CSP host-source. + * An explicit port matches exactly that port for the matching scheme(s). + * + * @param {string} raw + * @param {{ schemes?: Set }} [options] `schemes` defaults to `{http, https}`. + * @returns {string} the canonical form + * @throws {GrammarError} + */ +export function canonicalizeWebHost(raw, { schemes = NAVIGATE_SCHEMES } = {}) { + assertNoWhitespaceOrControl(raw); + const { scheme, rest } = splitScheme(raw); + if (scheme !== null && !schemes.has(scheme)) { + const allowed = [...schemes].join("', '"); + throw new GrammarError(`scheme '${scheme}' is not one of '${allowed}'; use canonicalizeEndpoint for a non-HTTP, non-WebSocket scheme`); + } + const { host, wildcard, port } = canonicalizeHostAndPort(rest, { scheme, allowWildcard: true }); + const prefix = scheme === null ? "" : `${scheme}://`; + const wildcardPrefix = wildcard ? "*." : ""; + const portSuffix = port === "" ? "" : `:${port}`; + return `${prefix}${wildcardPrefix}${host}${portSuffix}`; +} + +/** + * Returns whether `raw` is already in `canonicalizeWebHost`'s canonical + * form (idempotent fixed point), without throwing on invalid input. + */ +export function isCanonicalWebHost(raw, options) { + try { + return canonicalizeWebHost(raw, options) === raw; + } catch { + return false; + } +} + +/** + * Whether `candidateHost` (already canonical: lowercase, IDNA ASCII, no + * trailing dot) is matched by the wildcard apex `apex` (the canonical host + * text after a leading `*.`, for example `"chase.com"` from + * `"*.chase.com"`). Section 3.3.2's rule follows the CSP host-source + * convention: a wildcard matches one or more leftmost labels prepended to + * the apex, at any depth, and never the bare apex itself, which an author + * lists as its own separate entry when it also needs to be reachable. A + * single leading `*.` label is this grammar's SYNTAX (Section 3.3.2); it + * does not bound the MATCHING depth, which real captures need: the reddit + * capture loaded `b.thumbs.redditmedia.com`, two labels under + * `redditmedia.com`, which `*.redditmedia.com` must cover. + * + * Examples for apex `"chase.com"`: matches `"secure.chase.com"` and + * `"a.b.chase.com"`; does not match `"chase.com"` itself; does not match + * `"evilchase.com"` (a label boundary is required, not just a string + * suffix). + * + * @param {string} apex + * @param {string} candidateHost + * @returns {boolean} + */ +export function hostMatchesWildcardApex(apex, candidateHost) { + const suffix = `.${apex}`; + if (!candidateHost.endsWith(suffix)) return false; + const prefix = candidateHost.slice(0, candidateHost.length - suffix.length); + return prefix.length > 0; +} + +/** + * Canonicalizes one entry of `browser.connect` (Section 3.3.3): the same + * grammar as `canonicalizeWebHost`, with `ws` and `wss` also accepted. + * + * @param {string} raw + * @returns {string} + * @throws {GrammarError} + */ +export function canonicalizeConnectHost(raw) { + return canonicalizeWebHost(raw, { schemes: CONNECT_SCHEMES }); +} + +export function isCanonicalConnectHost(raw) { + return isCanonicalWebHost(raw, { schemes: CONNECT_SCHEMES }); +} + +/** + * Canonicalizes a scheme-aware non-HTTP network endpoint (Section 3.3.4), + * for example an IMAP mailbox host. Unlike the web grammar: the scheme is + * REQUIRED (there is no implied protocol to fall back to), no wildcard is + * allowed (an endpoint names one account's exact host), and an omitted port + * is accepted only when the scheme has a listed default port. + * + * @param {string} raw + * @returns {string} the canonical form + * @throws {GrammarError} + */ +export function canonicalizeEndpoint(raw) { + assertNoWhitespaceOrControl(raw); + const { scheme, rest } = splitScheme(raw); + if (scheme === null) { + throw new GrammarError("a non-HTTP endpoint must state its scheme explicitly"); + } + if (CONNECT_SCHEMES.has(scheme)) { + throw new GrammarError(`scheme '${scheme}' is an HTTP or WebSocket scheme; use canonicalizeWebHost or canonicalizeConnectHost`); + } + const { host, port } = canonicalizeHostAndPort(rest, { scheme, allowWildcard: false }); + if (port === "" && DEFAULT_PORTS[scheme] === undefined) { + throw new GrammarError(`scheme '${scheme}' has no known default port; state the port explicitly`); + } + const portSuffix = port === "" ? "" : `:${port}`; + return `${scheme}://${host}${portSuffix}`; +} + +export function isCanonicalEndpoint(raw) { + try { + return canonicalizeEndpoint(raw) === raw; + } catch { + return false; + } +} + +/** + * Canonicalizes one entry of a `network` binding's `hosts` list (Section + * 3.3.4), dispatching to the web or endpoint grammar by scheme. A bare host + * with no scheme, or an `http`/`https` scheme, uses the web grammar + * (wildcard allowed). Any other explicit scheme uses the endpoint grammar. + * + * @param {string} raw + * @returns {string} + * @throws {GrammarError} + */ +export function canonicalizeNetworkHost(raw) { + assertNoWhitespaceOrControl(raw); + const { scheme } = splitScheme(raw); + if (scheme === null || WEB_SCHEMES.has(scheme)) return canonicalizeWebHost(raw); + return canonicalizeEndpoint(raw); +} + +export function isCanonicalNetworkHost(raw) { + try { + return canonicalizeNetworkHost(raw) === raw; + } catch { + return false; + } +} + +/** + * Resolves the `filesystem.inputs` alias (Section 3.3.5): a filesystem-kind + * binding instance can declare its inputs at the legacy top-level `inputs` + * member, at `constraints.inputs`, at both, or at neither. When both are + * present they MUST describe the same inputs; this normalizes them to one + * array (order-independent, keyed by `env_var`) and throws when they + * disagree. JSON Schema cannot express this cross-field equality, so it is + * checked here instead. + * + * @param {{ inputs?: unknown[], constraints?: { inputs?: unknown[] } }} instance + * @returns {unknown[] | undefined} the normalized inputs, or undefined when neither form is present + * @throws {GrammarError} when both forms are present and disagree + */ +/** + * Structural equality for plain JSON values (objects, arrays, strings, + * numbers, booleans, null). Unlike `JSON.stringify(a) === JSON.stringify(b)`, + * this does not depend on an object's member order -- two filesystem input + * entries with the same fields written in a different key order compare + * equal. Array element order still matters (an array is an ordered list, + * e.g. `accepted_extensions`, and reordering it is a real difference). + */ +export function deepEqualJson(a, b) { + if (a === b) return true; + if (typeof a !== typeof b || a === null || b === null) return false; + if (Array.isArray(a) !== Array.isArray(b)) return false; + if (Array.isArray(a)) { + return a.length === b.length && a.every((value, i) => deepEqualJson(value, b[i])); + } + if (typeof a === "object") { + const keysA = Object.keys(a); + const keysB = Object.keys(b); + return keysA.length === keysB.length && keysA.every((key) => Object.hasOwn(b, key) && deepEqualJson(a[key], b[key])); + } + return false; +} + +export function normalizeFilesystemInputs(instance) { + const topLevel = instance?.inputs; + const nested = instance?.constraints?.inputs; + if (topLevel === undefined) return nested; + if (nested === undefined) return topLevel; + + const byEnvVar = (list) => { + const map = new Map(); + for (const entry of list) map.set(entry?.env_var, entry); + return map; + }; + const a = byEnvVar(topLevel); + const b = byEnvVar(nested); + const sameKeys = a.size === b.size && [...a.keys()].every((key) => b.has(key)); + const sameEntries = sameKeys && [...a.entries()].every(([key, value]) => deepEqualJson(value, b.get(key))); + if (!sameEntries) { + throw new GrammarError("filesystem.inputs and constraints.inputs disagree; declare one, or make both equal"); + } + return topLevel; +} + +/** + * Finds duplicate JSON object keys in raw manifest text. `JSON.parse` + * silently keeps the last value for a repeated key, which hides an author + * or generator mistake and is a real interop hazard: two readers could + * reasonably disagree about which value won. This performs its own + * lightweight tokenization so the check runs on the text as written, not on + * the already-collapsed object. + * + * @param {string} jsonText + * @returns {Array<{ path: string, key: string }>} every duplicate found + */ +export function findDuplicateJsonKeys(jsonText) { + const duplicates = []; + const stack = []; // { kind: "object"|"array", keys?: Set, path: string, index?: number } + let i = 0; + const n = jsonText.length; + + function skipWhitespace() { + while (i < n && /\s/.test(jsonText[i])) i += 1; + } + + function readString() { + // Assumes jsonText[i] === '"'. + const start = i; + i += 1; + while (i < n) { + const c = jsonText[i]; + if (c === "\\") { + i += 2; + continue; + } + if (c === '"') { + i += 1; + return JSON.parse(jsonText.slice(start, i)); + } + i += 1; + } + throw new GrammarError("unterminated string in JSON text"); + } + + // The path a newly opened `{` or `[` should take: the path of the key (in + // an object) or index (in an array) whose value it is. `null` at the + // document root. + let pendingPath = "$"; + + function childPath(parentTop, segment) { + if (!parentTop) return "$"; + return parentTop.path === "$" ? String(segment) : `${parentTop.path}.${segment}`; + } + + skipWhitespace(); + while (i < n) { + const c = jsonText[i]; + const top = stack[stack.length - 1]; + if (c === "{") { + stack.push({ kind: "object", keys: new Set(), path: pendingPath }); + i += 1; + } else if (c === "[") { + // Array element paths are not tracked per-index: a duplicate *key* + // can occur only inside an object, so the generic "[]" segment below + // is precise enough for this check's error messages. + stack.push({ kind: "array", path: pendingPath }); + pendingPath = childPath({ path: pendingPath }, "[]"); + i += 1; + } else if (c === "}" || c === "]") { + stack.pop(); + i += 1; + } else if (c === '"') { + const str = readString(); + if (top?.kind === "object") { + skipWhitespace(); + if (jsonText[i] === ":") { + // This string is a key, not a value. + if (top.keys.has(str)) { + duplicates.push({ path: childPath(top, str), key: str }); + } else { + top.keys.add(str); + } + pendingPath = childPath(top, str); + } + } + } else if (c === ":" || c === ",") { + i += 1; + } else { + // number, true, false, null -- skip to the next structural character. + i += 1; + } + skipWhitespace(); + } + + return duplicates; +} diff --git a/reference-implementation/runtime/connector-manifest.schema.json b/reference-implementation/runtime/connector-manifest.schema.json new file mode 100644 index 000000000..3c3636e7d --- /dev/null +++ b/reference-implementation/runtime/connector-manifest.schema.json @@ -0,0 +1,381 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/PDP-Connect/data-connectors/schemas/connector-manifest.schema.json", + "title": "Collection Profile manifest runtime requirements", + "description": "Machine-readable half of docs/spec/collection-profile.md Section 3.3: binding instances, kinds, interface versions, typed constraints, binding features, and filesystem inputs. Other manifest fields are outside this schema's scope, except the coverage_stores declaration rule of Section 7.1.1, which reads only `streams` and `protocol_capabilities`. String patterns for constraint grammars (Section 3.3.4, 3.3.5) are a necessary but not sufficient structural check: they accept the canonical output of schemas/connector-binding-grammar.mjs, but full IDNA canonicalization, IPv4-looking-label rejection, and IPv6 normalization are that module's job, not this schema's. A string this schema accepts can still be non-canonical in a way only that module detects.", + "type": "object", + "properties": { + "runtime_requirements": { + "type": "object", + "properties": { + "bindings": { "$ref": "#/$defs/bindings" } + } + } + }, + "allOf": [ + { + "$comment": "Section 7.1.1: a stream that declares coverage_stores requires RUN_COVERAGE_V1 in protocol_capabilities.", + "if": { + "required": ["streams"], + "properties": { + "streams": { "type": "array", "contains": { "type": "object", "required": ["coverage_stores"], "properties": { "coverage_stores": {} } } } + } + }, + "then": { + "required": ["protocol_capabilities"], + "properties": { + "protocol_capabilities": { "type": "array", "contains": { "const": "RUN_COVERAGE_V1" } } + } + } + } + ], + "$defs": { + "bindings": { + "type": "object", + "propertyNames": { "$ref": "#/$defs/bindingMapKey" }, + "additionalProperties": false, + "patternProperties": { + "^browser$": { "$ref": "#/$defs/shorthandBrowserInstance" }, + "^desktop_session$": { "$ref": "#/$defs/shorthandDesktopSessionInstance" }, + "^filesystem$": { "$ref": "#/$defs/shorthandFilesystemInstance" }, + "^network$": { "$ref": "#/$defs/shorthandNetworkInstance" }, + "^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+/[a-z0-9]([a-z0-9._-]*[a-z0-9])?$": { + "$ref": "#/$defs/namespacedExtensionInstance" + }, + "^(?!browser$|desktop_session$|filesystem$|network$)[a-z][a-z0-9_]*$": { "$ref": "#/$defs/namedInstance" } + } + }, + "bindingMapKey": { + "type": "string", + "description": "A binding map key is either a registry or namespaced binding name used as shorthand for one instance of that kind, or a free-form author-local instance key that declares its kind explicitly (Section 3.3).", + "anyOf": [{ "$ref": "#/$defs/bindingName" }, { "$ref": "#/$defs/instanceKeyPattern" }] + }, + "instanceKeyPattern": { + "type": "string", + "pattern": "^(?!browser$|desktop_session$|filesystem$|network$)[a-z][a-z0-9_]*$" + }, + "bindingName": { + "type": "string", + "description": "A registry binding name, or an extension binding namespaced as /. Unqualified names are reserved for the registry.", + "anyOf": [ + { "$ref": "#/$defs/registryBinding" }, + { "pattern": "^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+/[a-z0-9]([a-z0-9._-]*[a-z0-9])?$" } + ] + }, + "registryBinding": { + "type": "string", + "description": "Binding registry:\n- browser: A runtime-managed browser surface.\n- desktop_session: The owner's active, logged-in desktop session and its operating-system facilities.\n- filesystem: Access to local files.\n- network: Outbound network access.", + "enum": ["browser", "desktop_session", "filesystem", "network"] + }, + + "instanceRequired": { + "type": "object", + "required": ["required"], + "properties": { "required": { "type": "boolean" } } + }, + "noKind": { "not": { "type": "object", "required": ["kind"], "properties": { "kind": true } } }, + + "shorthandBrowserInstance": { + "allOf": [{ "$ref": "#/$defs/instanceRequired" }, { "$ref": "#/$defs/noKind" }, { "$ref": "#/$defs/browserExtra" }] + }, + "shorthandNetworkInstance": { + "allOf": [{ "$ref": "#/$defs/instanceRequired" }, { "$ref": "#/$defs/noKind" }, { "$ref": "#/$defs/networkExtra" }] + }, + "shorthandFilesystemInstance": { + "allOf": [{ "$ref": "#/$defs/instanceRequired" }, { "$ref": "#/$defs/noKind" }, { "$ref": "#/$defs/filesystemExtra" }] + }, + "shorthandDesktopSessionInstance": { + "allOf": [ + { "$ref": "#/$defs/instanceRequired" }, + { "$ref": "#/$defs/noKind" }, + { "$ref": "#/$defs/desktopSessionExtra" } + ] + }, + "namespacedExtensionInstance": { + "allOf": [ + { "$ref": "#/$defs/instanceRequired" }, + { "$ref": "#/$defs/noKind" }, + { "$ref": "#/$defs/genericExtensionExtra" } + ] + }, + "namedInstance": { + "description": "A free-form author-local binding key. It MUST declare kind (Section 3.3), which selects the constraint and feature shape below by value, the same shape a shorthand key of that kind would use.", + "allOf": [ + { "$ref": "#/$defs/instanceRequired" }, + { "type": "object", "required": ["kind"], "properties": { "kind": true } }, + { "type": "object", "properties": { "kind": { "$ref": "#/$defs/bindingName" } } }, + { + "if": { "type": "object", "properties": { "kind": { "const": "browser" } } }, + "then": { "$ref": "#/$defs/browserExtra" } + }, + { + "if": { "type": "object", "properties": { "kind": { "const": "network" } } }, + "then": { "$ref": "#/$defs/networkExtra" } + }, + { + "if": { "type": "object", "properties": { "kind": { "const": "filesystem" } } }, + "then": { "$ref": "#/$defs/filesystemExtra" } + }, + { + "if": { "type": "object", "properties": { "kind": { "const": "desktop_session" } } }, + "then": { "$ref": "#/$defs/desktopSessionExtra" } + }, + { + "if": { + "type": "object", + "properties": { + "kind": { "type": "string", "pattern": "^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+/[a-z0-9]([a-z0-9._-]*[a-z0-9])?$" } + } + }, + "then": { "$ref": "#/$defs/genericExtensionExtra" } + } + ] + }, + + "browserExtra": { + "type": "object", + "properties": { + "interface": { "type": "string", "pattern": "^browser@[1-9][0-9]*$" }, + "features": { + "type": "array", + "items": { "anyOf": [{ "$ref": "#/$defs/browserFeature" }, { "$ref": "#/$defs/legacyBrowserFeature" }] }, + "uniqueItems": true + }, + "constraints": { "$ref": "#/$defs/browserConstraints" } + } + }, + "networkExtra": { + "type": "object", + "properties": { + "interface": { "type": "string", "pattern": "^network@[1-9][0-9]*$" }, + "features": { + "type": "array", + "items": { "$ref": "#/$defs/networkFeature" }, + "uniqueItems": true + }, + "constraints": { "$ref": "#/$defs/networkConstraints" } + } + }, + "filesystemExtra": { + "type": "object", + "not": { "type": "object", "required": ["features"], "properties": { "features": true } }, + "properties": { + "interface": { "type": "string", "pattern": "^filesystem@[1-9][0-9]*$" }, + "inputs": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/filesystemInput" } + }, + "constraints": { "$ref": "#/$defs/filesystemConstraints" } + } + }, + "desktopSessionExtra": { + "type": "object", + "not": { "type": "object", "required": ["features"], "properties": { "features": true } }, + "properties": { + "interface": { "type": "string", "pattern": "^desktop_session@[1-9][0-9]*$" }, + "constraints": { "$ref": "#/$defs/desktopSessionConstraints" } + } + }, + "genericExtensionExtra": { + "description": "An extension kind defines its own feature and constraint vocabulary. This schema only checks that a declared feature, if any, is drawn from the shared feature enum, matching the leniency extension bindings already had before binding instances existed; constraints are an open object.", + "type": "object", + "properties": { + "interface": { "type": "string", "minLength": 1 }, + "features": { + "type": "array", + "items": { "$ref": "#/$defs/bindingFeature" }, + "uniqueItems": true + }, + "constraints": { "type": "object" } + } + }, + + "browserFeature": { + "type": "string", + "description": "Features the browser kind provides (Section 3.3.1):\n- page_navigation: Navigate the active page to a URL.\n- page_script_evaluation: Run script in the active page context.\n- page_content_read: Read rendered page content.\n- page_condition_wait: Wait until a condition in the active page becomes true.\n- host_download_capture: Capture content downloaded by the active page.\n- host_archive_extraction: Extract downloaded archive contents in the host runtime.\n- host_archive_entry_chunk_read: Read an extracted archive entry in chunks.\n- page_input: Interact with the active page: click and type.\n- cookie_read: Read the active page's cookie jar.\n- page_response_observation: Observe page network responses, including response body content.\n- host_cookie_jar_request: Make an HTTP request from the host runtime using the active page's cookie jar.", + "enum": [ + "page_navigation", + "page_script_evaluation", + "page_content_read", + "page_condition_wait", + "host_download_capture", + "host_archive_extraction", + "host_archive_entry_chunk_read", + "page_input", + "cookie_read", + "page_response_observation", + "host_cookie_jar_request" + ] + }, + "legacyBrowserFeature": { + "type": "string", + "description": "Section 3.3.8: host_http_request is also valid on browser as a named legacy dual placement, not the target shape. It is not added to browserFeature, so a validator can tell a legacy placement from the one-kind-per-feature rule new features follow. github_browser and strava_browser already declare it here, and scripts/pageshim/capabilities.mjs already supports it on both kinds. A future versioned alias, coordinated with the mobile host, is expected to resolve this; this schema does not perform that move.", + "enum": ["host_http_request"] + }, + "networkFeature": { + "type": "string", + "description": "Features the network kind provides (Section 3.3.1):\n- same_origin_page_fetch: Fetch same-origin resources from the active page context.\n- host_http_request: Make an HTTP request from the host runtime outside the page context.", + "enum": ["same_origin_page_fetch", "host_http_request"] + }, + "bindingFeature": { + "type": "string", + "description": "Host capability definitions (Section 3.3.8; each belongs to exactly one kind, see browserFeature and networkFeature):\n- page_navigation: Navigate the active page to a URL.\n- page_script_evaluation: Run script in the active page context.\n- page_content_read: Read rendered page content.\n- page_condition_wait: Wait until a condition in the active page becomes true.\n- same_origin_page_fetch: Fetch same-origin resources from the active page context.\n- host_http_request: Make an HTTP request from the host runtime outside the page context.\n- host_download_capture: Capture content downloaded by the active page.\n- host_archive_extraction: Extract downloaded archive contents in the host runtime.\n- host_archive_entry_chunk_read: Read an extracted archive entry in chunks.\n- page_input: Interact with the active page: click and type.\n- cookie_read: Read the active page's cookie jar.\n- page_response_observation: Observe page network responses, including response body content.\n- host_cookie_jar_request: Make an HTTP request from the host runtime using the active page's cookie jar.", + "enum": [ + "page_navigation", + "page_script_evaluation", + "page_content_read", + "page_condition_wait", + "same_origin_page_fetch", + "host_http_request", + "host_download_capture", + "host_archive_extraction", + "host_archive_entry_chunk_read", + "page_input", + "cookie_read", + "page_response_observation", + "host_cookie_jar_request" + ] + }, + + "browserConstraints": { + "type": "object", + "additionalProperties": false, + "properties": { + "navigate": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/webHostString" } + }, + "connect": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/webConnectHostString" } + } + } + }, + "networkConstraints": { + "type": "object", + "additionalProperties": false, + "properties": { + "hosts": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "anyOf": [ + { "$ref": "#/$defs/webHostString" }, + { "$ref": "#/$defs/endpointString" }, + { "$ref": "#/$defs/setupFieldHost" } + ] + } + } + } + }, + "setupFieldHost": { + "type": "object", + "description": "A host or endpoint supplied through a manifest setup field rather than a literal (Section 3.3.4). This is a typed reference, not string interpolation: setup_field names the field, and the runtime resolves and freezes its value before the owner approves the grant.", + "additionalProperties": false, + "required": ["setup_field"], + "properties": { + "setup_field": { "type": "string", "minLength": 1 }, + "allow_private": { "type": "boolean" } + } + }, + "webHostString": { + "type": "string", + "description": "Canonical [scheme://]host[:port] for browser.navigate and an HTTP(S) network host (Section 3.3.3, 3.3.4): lowercase, IDNA ASCII, default http/https port dropped, normalized IPv6, no IPv4-looking non-canonical label, no trailing dot, exact host or one leading '*.' label, no path. This pattern is a structural approximation; schemas/connector-binding-grammar.mjs#canonicalizeWebHost is the reference implementation.", + "pattern": "^(https?://)?(\\*\\.([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?|(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(\\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}|\\[[0-9a-f:]+\\]|([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)*[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)(:[1-9][0-9]{0,4})?$" + }, + "webConnectHostString": { + "type": "string", + "description": "Canonical [scheme://]host[:port] for browser.connect (Section 3.3.3), which is scheme-aware: a page loaded over http(s) can open a ws/wss WebSocket (live capture evidence: wss://ws.chatgpt.com), so connect accepts http, https, ws, or wss where navigate accepts only http/https. Otherwise the same canonicalization as webHostString. schemas/connector-binding-grammar.mjs#canonicalizeConnectHost is the reference implementation.", + "pattern": "^((https?|wss?)://)?(\\*\\.([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?|(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(\\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}|\\[[0-9a-f:]+\\]|([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)*[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)(:[1-9][0-9]{0,4})?$" + }, + "endpointString": { + "type": "string", + "description": "Canonical scheme://host[:port] for a non-HTTP, non-WebSocket network endpoint, for example IMAP (Section 3.3.4): scheme required, no wildcard, otherwise the same host canonicalization as webHostString. ws/wss are reserved for browser.connect (Section 3.3.3) and are not valid here. schemas/connector-binding-grammar.mjs#canonicalizeEndpoint is the reference implementation.", + "pattern": "^(?!(https?|wss?)://)[a-z][a-z0-9+.-]*://((25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(\\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}|\\[[0-9a-f:]+\\]|([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)*[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)(:[1-9][0-9]{0,4})?$" + }, + + "filesystemConstraints": { + "type": "object", + "additionalProperties": false, + "properties": { + "inputs": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/filesystemInput" } + }, + "outputs": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/filesystemOutput" } + } + } + }, + "filesystemInput": { + "type": "object", + "description": "A local path the connector reads. env_var values must be unique within inputs; JSON Schema cannot express that, so the profile states it and the schema test checks it for checked-in manifests.", + "additionalProperties": false, + "required": ["env_var", "kind", "access"], + "properties": { + "env_var": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]*$" }, + "kind": { "type": "string", "enum": ["file", "dir"] }, + "access": { "type": "string", "enum": ["read"] }, + "accepted_extensions": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "pattern": "^\\.[a-z0-9]+(\\.[a-z0-9]+)*$" } + } + } + }, + "filesystemOutput": { + "type": "object", + "description": "A declared output root the connector may write under (Section 3.3.3). The reserved slot \"scratch\" names the runtime's ephemeral per-run directory and MUST NOT be durable.", + "additionalProperties": false, + "required": ["slot", "access"], + "properties": { + "slot": { "type": "string", "pattern": "^[a-z][a-z0-9_]*$" }, + "access": { "type": "string", "enum": ["write"] }, + "overwrite": { "type": "boolean" }, + "delete": { "type": "boolean" }, + "durable": { "type": "boolean" } + }, + "if": { "type": "object", "properties": { "slot": { "const": "scratch" } } }, + "then": { "type": "object", "properties": { "durable": { "const": false } } } + }, + + "desktopSessionConstraints": { + "type": "object", + "additionalProperties": false, + "required": ["items"], + "properties": { + "items": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/desktopSessionItem" } + } + } + }, + "desktopSessionItem": { + "type": "object", + "description": "One exact secret item and the operations brokered on it (Section 3.3.6). selector is an opaque, connector- and OS-specific identifier; the runtime broker resolves it, never the manifest.", + "additionalProperties": false, + "required": ["service", "selector", "operations"], + "properties": { + "service": { "type": "string", "minLength": 1 }, + "selector": { "type": "string", "minLength": 1 }, + "operations": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "enum": ["read", "unwrap"] } + } + } + } + } +} diff --git a/reference-implementation/runtime/index.ts b/reference-implementation/runtime/index.ts index 235106c22..6905047f8 100644 --- a/reference-implementation/runtime/index.ts +++ b/reference-implementation/runtime/index.ts @@ -17,12 +17,9 @@ import { DatabaseSync } from "node:sqlite"; import { type ConnectorProtocolCapability, DETAIL_COVERAGE_KEY_SETS_CAPABILITY, - HOST_BLOB_CAPABILITY, isConnectorProtocolCapabilityArray, - OBSERVATION_CAPABILITY, PERMANENT_DETAIL_GAP_REASON, RETRYABLE_DETAIL_GAP_REASONS, - STREAM_EVIDENCE_CAPABILITY, unsupportedRequiredProtocolCapabilities, validateRuntimeContinuationFact, } from "@pdpp/connector-protocol/connector-runtime-protocol"; @@ -93,9 +90,12 @@ import { cursorAuditFact } from "./cursor-audit-fact.ts"; import { describeCursorBandViolation, evaluateCursorBand } from "./cursor-band-contiguity.ts"; import { declaredReasonTokensFor } from "./declared-reason-tokens.ts"; import { createDetailGapPageReader, validateDetailGapsPageRequest } from "./detail-gap-paging.ts"; +import { normalizeBindingDeclarations, planBindingPlacement, startBindingsFor } from "./binding-declarations.ts"; import { collectionProfileDeclarationProblem, + PROTOCOL_CAPABILITY_UNSUPPORTED_FAILURE_REASON, readCollectionProfileDeclaration, + REFERENCE_COLLECTION_PROFILE_SUPPORT, } from "./collection-profile-support.ts"; import type { DoneError } from "./done-validators.ts"; import { @@ -209,14 +209,6 @@ interface AttachmentOutcome { [key: string]: unknown; } -/** Runtime-advertised child bindings (Collection Profile START.bindings). */ -interface AvailableBindings { - browser: Record; - filesystem: Record; - interactive?: Record; - network: Record; -} - /** RS ingest response, after `readIngestResponse` proves the counters and receipt vector. */ type IngestResult = StrictIngestResult; @@ -937,18 +929,6 @@ function buildScopeFields( return appendUniqueFields(streamScope.fields, [...requiredFields, ...primaryKeyFields, ...timeRangeFields]); } -function buildAvailableBindings(onInteraction: unknown): AvailableBindings { - // In this reference runtime, connectors run as local Node child processes - // with full filesystem access by virtue of being child processes. We - // advertise `filesystem` so file-based connectors can declare it as a - // required binding per the Collection Profile spec. - const bindings: AvailableBindings = { browser: {}, filesystem: {}, network: {} }; - if (typeof onInteraction === "function") { - bindings.interactive = {}; - } - return bindings; -} - // Strict, index-validating, oracle-tested parser (test/read-ingest-response-oracle.test.ts). // The hosted RS route (server/routes/rs-mutation.ts) always sets // hostedRejectionReceipts: true, so this runtime's ingest responses always @@ -2602,15 +2582,13 @@ export function buildBrowserSurfaceLaunchEnv({ /** * Protocol capabilities this runtime implements, listed in - * START.protocol_capabilities. It accepts DETAIL_COVERAGE empty_keys and - * unchanged_keys, validates STREAM_EVIDENCE and accepts OBSERVATION (Section - * 5.10). It has no BLOB transport, so it does not list it. + * START.protocol_capabilities. Key-set extension credit rules are not part of + * portable coverage, so DETAIL_COVERAGE_KEY_SETS is omitted: connectors use + * their negotiated base-profile fallback. BLOB is required and unsupported, + * so a connector requiring it is refused before spawn. */ -const RUNTIME_PROTOCOL_CAPABILITIES: readonly ConnectorProtocolCapability[] = [ - DETAIL_COVERAGE_KEY_SETS_CAPABILITY, - OBSERVATION_CAPABILITY, - STREAM_EVIDENCE_CAPABILITY, -]; +const RUNTIME_PROTOCOL_CAPABILITIES: readonly ConnectorProtocolCapability[] = + REFERENCE_COLLECTION_PROFILE_SUPPORT.protocol_capabilities; /** Type of a Collection Profile 0.2.0 OBSERVATION message (Section 5.10). */ const OBSERVATION_MESSAGE_TYPE = "OBSERVATION"; @@ -2624,50 +2602,29 @@ function isObservationMessage(value: unknown): boolean { return typeof value === "object" && value !== null && (value as { type?: unknown }).type === OBSERVATION_MESSAGE_TYPE; } -/** - * Declared capabilities this runtime starts a connector with although it does - * not implement them. BLOB is a runtime-specific extension (Collection Profile - * Section 7). chatgpt and anthropic declare it but emit BLOB only for an item - * over the inline cap, so most of their runs never send one. Such a run - * completes; a run that does send BLOB fails at that line as an unknown - * message type. BLOB stays out of START until this runtime implements it. - */ -const SPAWN_TOLERATED_PROTOCOL_CAPABILITIES: readonly ConnectorProtocolCapability[] = [HOST_BLOB_CAPABILITY]; - /** * Pre-spawn capability gate (Collection Profile Sections 3.4 and 4). An * absent `protocol_capabilities` declares none. A malformed list or an unknown * name fails closed, and so does a required capability this runtime does not - * implement, except BLOB (see SPAWN_TOLERATED_PROTOCOL_CAPABILITIES). An - * advisory capability never blocks the run. + * implement. An advisory capability never blocks the run. */ +function protocolCapabilityRefusal(message: string): Error { + return Object.assign(new Error(message), { failure_reason: PROTOCOL_CAPABILITY_UNSUPPORTED_FAILURE_REASON }); +} + function validateRequiredProtocolCapabilities(manifest: ConnectorManifest): void { const declared = manifest.protocol_capabilities === undefined ? [] : manifest.protocol_capabilities; if (!isConnectorProtocolCapabilityArray(declared)) { - throw new Error("Connector manifest protocol_capabilities must be an array of known capability names"); + throw protocolCapabilityRefusal("Connector manifest protocol_capabilities must be an array of known capability names"); } - const unsupported = unsupportedRequiredProtocolCapabilities(declared, [ - ...RUNTIME_PROTOCOL_CAPABILITIES, - ...SPAWN_TOLERATED_PROTOCOL_CAPABILITIES, - ]); + const unsupported = unsupportedRequiredProtocolCapabilities(declared, RUNTIME_PROTOCOL_CAPABILITIES); if (unsupported.length > 0) { - throw new Error( + throw protocolCapabilityRefusal( `Connector requires protocol capabilities this runtime does not support: ${unsupported.join(", ")}` ); } } -function validateRequiredRuntimeBindings( - requiredBindings: Record, - availableBindings: AvailableBindings -): void { - for (const [binding, req] of Object.entries(requiredBindings)) { - if (req.required && !(binding in availableBindings)) { - throw new Error(`Runtime cannot satisfy required binding: ${binding}`); - } - } -} - function requestedRuntimeStreams(scope: { streams?: unknown } | null): Set | null { if (!Array.isArray(scope?.streams)) { return null; @@ -3074,14 +3031,18 @@ export async function runConnector(opts: RuntimeRunConnectorOptions): Promise = new Set( + Array.isArray(declaredHumanInteractionValues) ? declaredHumanInteractionValues : [] + ); const declarationProblem = collectionProfileDeclarationProblem(manifest as unknown as Record); if (declarationProblem !== null) { - throw new Error(`Connector manifest ${declarationProblem}`); + throw protocolCapabilityRefusal(`Connector manifest ${declarationProblem}`); } validateRequiredProtocolCapabilities(manifest); const profileDeclaration = readCollectionProfileDeclaration(manifest as unknown as Record); @@ -6382,8 +6343,11 @@ export async function runConnector(opts: RuntimeRunConnectorOptions): Promise key !== null) .sort((a, b) => a.localeCompare(b)); +const legacySourceBindings = validateLegacySourceBindings( + pinnedPolyfillManifests.map(({ manifest, file }) => { + const key = manifest.connector_key; + const sourceId = manifest.connector_id; + if (typeof key !== "string") { + throw new Error(`Invalid pinned connector_key in ${file}`); + } + if (typeof sourceId !== "string") { + throw new Error(`Invalid pinned legacy connector_id in ${file}`); + } + return [key, sourceId] as const; + }) +); + function jsonArrayLiteral(values: readonly string[]): string { return `[${values.map((value) => JSON.stringify(value)).join(", ")}]`; } @@ -288,6 +306,10 @@ function jsonObjectLiteral(record: Record): string { return `{\n${lines.join("\n")}\n}`; } +function jsonTupleArrayLiteral(values: readonly (readonly [string, string])[]): string { + return `[${values.map(([key, sourceId]) => `[${JSON.stringify(key)}, ${JSON.stringify(sourceId)}]`).join(", ")}]`; +} + const output = `// Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 @@ -328,6 +350,9 @@ export const PROVIDER_AUTH_LIFECYCLE_PROVEN_KEYS: readonly string[] = Object.fre /** Manifests declaring capabilities.proven.static_secret_live.proven === true. */ export const STATIC_SECRET_LIVE_PROVEN_KEYS: readonly string[] = Object.freeze(${jsonArrayLiteral(staticSecretLiveProvenKeys)}); + +/** Reviewed historical source bindings derived from the pinned connector package. */ +export const LEGACY_FIRST_PARTY_SOURCE_ENTRIES: readonly (readonly [string, string])[] = Object.freeze((${jsonTupleArrayLiteral(legacySourceBindings)} as [string, string][]).map(([key, sourceId]) => Object.freeze([key, sourceId] as const))); `; if (process.argv[2]) { diff --git a/reference-implementation/scripts/legacy-source-binding-validation.ts b/reference-implementation/scripts/legacy-source-binding-validation.ts new file mode 100644 index 000000000..1d76ef36e --- /dev/null +++ b/reference-implementation/scripts/legacy-source-binding-validation.ts @@ -0,0 +1,40 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +export type LegacySourceBinding = readonly [connectorKey: string, sourceId: string]; + +export const LEGACY_SOURCE_BINDINGS_PIN = "7e8f726725b7a14705571339c84a52d8f05888ffc0996b77cce3d4ae13dae9e6"; +const LEGACY_BINDING_KEY = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; + +/** Validate and order the exact historical mapping approved for generation. */ +export function validateLegacySourceBindings(bindings: readonly LegacySourceBinding[]): LegacySourceBinding[] { + for (const [key, sourceId] of bindings) { + if (!LEGACY_BINDING_KEY.test(key)) { + throw new Error(`Invalid pinned connector_key: ${key}`); + } + if (!URL.canParse(sourceId)) { + throw new Error(`Invalid pinned legacy connector_id for ${key}`); + } + } + + if ( + new Set(bindings.map(([key]) => key)).size !== bindings.length || + new Set(bindings.map(([, sourceId]) => sourceId)).size !== bindings.length + ) { + throw new Error("Pinned historical source bindings contain duplicate keys or source IDs"); + } + + const ordered = [...bindings].sort(([a], [b]) => { + if (a === b) { + return 0; + } + return a < b ? -1 : 1; + }); + const pin = createHash("sha256").update(JSON.stringify(ordered), "utf8").digest("hex"); + if (pin !== LEGACY_SOURCE_BINDINGS_PIN) { + throw new Error(`Pinned historical source mapping changed: sha256:${pin}`); + } + return ordered; +} diff --git a/reference-implementation/scripts/postgres-template-eligibility.ts b/reference-implementation/scripts/postgres-template-eligibility.ts index 12c9352b4..964c1ff95 100644 --- a/reference-implementation/scripts/postgres-template-eligibility.ts +++ b/reference-implementation/scripts/postgres-template-eligibility.ts @@ -53,6 +53,9 @@ export const POSTGRES_TEMPLATE_ELIGIBLE_FILES: readonly string[] = [ "test/connector-attention-store.test.ts", // Exercises install, lock and post-install repair behavior over seeded data, not schema bootstrap. "test/connector-install-postgres.test.ts", + // Install-to-registry journey over a fresh database: Postgres is a fixture for registration and reuse, not schema bootstrap. + "test/connector-install-registry-integration.test.ts", + "test/connector-install-sql-persistence.test.ts", "test/connector-instance-credential-revoke-postgres.test.ts", "test/connector-instance-delete-upsert-two-process-race.test.ts", "test/connector-instance-delete-vs-queued-write-fence.test.ts", @@ -63,6 +66,8 @@ export const POSTGRES_TEMPLATE_ELIGIBLE_FILES: readonly string[] = [ "test/connector-manifest-revision-fence.test.ts", // Uses Postgres as a fixture for the diagnostic run store's writes, retention and locking, not schema bootstrap. "test/connector-observation-store.test.ts", + // Registration identity boundary: Postgres is a fixture for the connectors row and its persist guard. + "test/connector-source-identity-boundary.test.ts", "test/connector-summary-dirty-hooks.test.ts", "test/connector-summary-evidence-canonical-count-cancel-isolation.test.ts", "test/connector-summary-evidence-direct-write-source-revision-detection.test.ts", diff --git a/reference-implementation/server/auth.ts b/reference-implementation/server/auth.ts index 0e4d07068..aec94e612 100644 --- a/reference-implementation/server/auth.ts +++ b/reference-implementation/server/auth.ts @@ -38,8 +38,10 @@ import { clientAssertionIssuer, verifyPrivateKeyJwtClientAssertion, } from "./oauth-client-auth.ts"; +import { parseManifestJsonText } from "../runtime/binding-declarations.ts"; import { listActiveBindingsForGrant, projectBindingForWire } from "./connection-identity.ts"; import { canonicalConnectorKey, canonicalConnectorKeyFromManifest, legacyLocalAliasMap } from "./connector-key.ts"; +import { legacyFirstPartySourceIdFor } from "./connector-legacy-source-map.ts"; import { invalidConnectorManifest, resolveManifestSensitivity, @@ -77,7 +79,7 @@ import { ownerPasswordVerifierRevision, } from "./stores/owner-password-verifier-store.ts"; import { buildGrantedAuthorizationDetail } from "./source-approved-authorization.ts"; -import { snapshotSourceDeclaration } from "./source-declaration.ts"; +import { InvalidSourceDeclarationError, snapshotSourceDeclaration } from "./source-declaration.ts"; import { snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest } from "./source-declaration-legacy-collection.ts"; import { type AcceptedSourceDeclarationRevisionStore, @@ -3524,6 +3526,7 @@ async function persistManifestAndAdvanceGenerations( connectorId, ]) ).rows[0]?.manifest; + assertRetainedSourceIdentity(connectorId, previous, manifestJson); // A manifest stored before digests were retained is the only record of // its version's accepted content: keep it before it is replaced. const previousDigest = previous ? declarationDigestOf(previous) : null; @@ -3595,6 +3598,7 @@ async function persistManifestAndAdvanceGenerations( const existing = db .prepare("SELECT manifest FROM connectors WHERE connector_id = ?") .get<{ manifest?: string }>(connectorId); + assertRetainedSourceIdentity(connectorId, existing?.manifest, manifestJson); if (options.queueDerivedRepair) { const plan = derivedRepairPlan(existing?.manifest ?? null, manifestJson); const instances = db @@ -5550,8 +5554,12 @@ export async function registerConnector( } = {} ): Promise { validateConnectorManifest(manifest); - const { connectorId, storedManifest } = normalizeConnectorManifestForStorage(manifest); + // The retained identity is read here and re-checked under the publication lock in + // persistManifestAndAdvanceGenerations, which is what makes the choice atomic. + const storageContext = await retainedStorageContextForConnector(storageConnectorIdOf(manifest)); + const { connectorId, storedManifest } = normalizeConnectorManifestForStorage(manifest, storageContext); if (!options.skipManifestPersistence) { + await maybeRegisterConnectorPhaseForTest("before-manifest-persisted", { connectorId, manifest: storedManifest }); await persistManifestAndAdvanceGenerations(connectorId, JSON.stringify(storedManifest), { queueDerivedRepair: options.queueDerivedRepair === true, replaceAcceptedDigest: options.replaceAcceptedDigest === true, @@ -5642,14 +5650,138 @@ export async function registerConnector( * authority. In particular, legacy manifest projection adds a durable source * declaration that is not present in the shipped JSON. */ -export function normalizeConnectorManifestForStorage(manifest: Record): { +/** + * Facts a caller reads under the same authority that will persist the result. + * Normalization stays a pure function of its arguments: it never looks anything up. + */ +export interface ConnectorManifestStorageContext { + /** + * The SourceDeclaration an earlier accepted registration retained under this + * connector key. When the incoming manifest carries no explicit declaration, + * the retained `source.id` is kept. Its projection is re-derived from the + * incoming content, so a legitimate content update yields a new + * `declaration_version` under the same source identity. + */ + readonly retainedSourceDeclaration?: unknown; +} + +function storageConnectorIdOf(manifest: Record): unknown { + return canonicalConnectorKeyFromManifest(manifest) ?? manifest.connector_id; +} + +/** The `source_declaration` of a stored or incoming manifest; undefined when absent or unparseable. */ +function sourceDeclarationOf(manifest: unknown): Record | undefined { + try { + const parsed: unknown = typeof manifest === "string" ? JSON.parse(manifest) : manifest; + return isRecord(parsed) && isRecord(parsed.source_declaration) ? parsed.source_declaration : undefined; + } catch { + return undefined; + } +} + +function sourceIdOfDeclaration(declaration: Record | undefined): string | undefined { + const source = declaration?.source; + return isRecord(source) && typeof source.id === "string" && source.id.length > 0 ? source.id : undefined; +} + +/** + * Storage context for a writer that replaces `previousManifest`. Call it with the row read under the same + * lock or transaction that persists the result. A row with no retained declaration gives no context, so + * its write takes the fresh path: a reviewed mapping or an explicit declaration, else a refusal. + */ +export function retainedStorageContext(previousManifest: unknown): ConnectorManifestStorageContext { + const declaration = sourceDeclarationOf(previousManifest); + return declaration === undefined ? {} : { retainedSourceDeclaration: declaration }; +} + +/** The declaration currently retained under a connector key, read before the registration transaction. */ +async function retainedStorageContextForConnector(connectorId: unknown): Promise { + if (!isNonEmptyString(connectorId)) { + return {}; + } + const row = await getConnectorCatalogStore().getManifestById(connectorId); + return retainedStorageContext(row?.manifest); +} + +/** Core Section 5: a retained source identity is never replaced by a registration that names another one. */ +export function assertRetainedSourceIdentity(connectorId: string, previousManifest: unknown, incomingManifestJson: string) { + const retained = sourceIdOfDeclaration(sourceDeclarationOf(previousManifest)); + if (retained === undefined) { + return; + } + const incoming = sourceIdOfDeclaration(sourceDeclarationOf(incomingManifestJson)); + if (incoming === retained) { + return; + } + throw invalidConnectorManifest( + `Connector ${connectorId} is already bound to source ${retained}; this registration names ${incoming ?? "no source"}. A source identity change needs an explicit identity-change path.` + ); +} + +/** Validate an explicit declaration through the Core boundary before it is retained. */ +function acceptedSourceDeclaration(value: unknown): Record { + try { + return snapshotSourceDeclaration(value) as unknown as Record; + } catch (error) { + if (error instanceof InvalidSourceDeclarationError) { + throw invalidConnectorManifest(error.message); + } + throw error; + } +} + +/** + * Project the legacy manifest under the source identity a retained declaration + * already holds. The projection's content is the incoming manifest's; the + * identity is the retained one, never the manifest's provenance. + */ +function projectRetainedSourceDeclaration( + storedManifest: Record, + retainedValue: unknown +): Record { + const retained = acceptedSourceDeclaration(retainedValue); + const source = retained.source as { id: string; kind: string }; + const publisher = retained.publisher as { id: string }; + if (source.kind !== "connector") { + throw invalidConnectorManifest( + `Connector ${String(storedManifest.connector_key)} is bound to a ${source.kind} source; a connector registration cannot change its source kind` + ); + } + return snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest(storedManifest, { + connectorImplementationId: implementationIdOf(storedManifest, source.id), + publisherId: publisher.id, + sourceId: source.id, + }) as unknown as Record; +} + +/** + * The implementation id a legacy projection records: the manifest document's provenance URI when it + * is absolute, else the source id. The alias stamp in `normalizeConnectorManifestForStorage` can put a + * non-URI alias in `manifest_uri`; that value is not a provenance URI. + */ +function implementationIdOf(storedManifest: Record, sourceId: string): string { + const provenance = storedManifest.manifest_uri; + return isNonEmptyString(provenance) && URL.canParse(provenance) ? provenance : sourceId; +} + +export function normalizeConnectorManifestForStorage( + manifest: Record, + context: ConnectorManifestStorageContext = {} +): { connectorId: string; storedManifest: Record; } { - const connectorId = canonicalConnectorKeyFromManifest(manifest) ?? manifest.connector_id; + const connectorId = storageConnectorIdOf(manifest); if (!isNonEmptyString(connectorId)) { throw invalidConnectorManifest("connector_id is required"); } + if (URL.canParse(connectorId)) { + // Known aliases have already resolved to slugs. Any remaining URL is unsupported as an operational + // key, whether or not the manifest carries a source declaration. + throw invalidConnectorManifest( + `connector_id ${connectorId} is a URL with no connector_key; register a connector key, and keep the URL as manifest_uri` + ); + } const storedManifest: Record = { ...cloneJson(manifest), connector_id: connectorId, @@ -5661,21 +5793,27 @@ export function normalizeConnectorManifestForStorage(manifest: Record; - const expected = activationValues(record); + // The row's own canonical declaration is the identity it was published under. + const expected = activationValues(record, retainedStorageContext(canonicalManifest)); if ( expected.activationId !== row.activation_id || expected.manifestRevision !== row.manifest_revision || @@ -107,17 +122,26 @@ function parseRow(row: ActivationRow | undefined): ConnectorActivation | null { /** Atomically publish the registry manifest, generation advance, and executable tuple. */ export async function publishConnectorActivation(record: ConnectorInstallRecord): Promise { - const values = activationValues(record); const attemptId = randomUUID(); const id = record.connectorId; + // The retained declaration is read under the same lock that persists the new manifest, so the + // identity choice cannot race a concurrent registration. + let values: ActivationValues; if (isPostgresStorageBackend()) { - await withPostgresTransaction(async (client) => { + values = await withPostgresTransaction(async (client) => { await client.query("SELECT pg_advisory_xact_lock(hashtext('connector_activation'), hashtext($1))", [id]); + const registry = await client.query<{ manifest: unknown }>( + "SELECT manifest FROM connectors WHERE connector_id=$1 FOR UPDATE", + [id] + ); + const previous = registry.rows[0]?.manifest; + const derived = activationValues(record, retainedStorageContext(previous)); + assertRetainedSourceIdentity(id, previous, derived.canonicalManifestJson); const changed = await client.query( `INSERT INTO connectors(connector_id, manifest) VALUES($1, $2::jsonb) ON CONFLICT(connector_id) DO UPDATE SET manifest = EXCLUDED.manifest WHERE connectors.manifest IS DISTINCT FROM EXCLUDED.manifest RETURNING connector_id`, - [id, values.canonicalManifestJson] + [id, derived.canonicalManifestJson] ); if (changed.rowCount) { await client.query( @@ -135,23 +159,33 @@ export async function publishConnectorActivation(record: ConnectorInstallRecord) canonical_manifest_json=EXCLUDED.canonical_manifest_json,manifest_revision=EXCLUDED.manifest_revision, activation_id=EXCLUDED.activation_id,attempt_id=EXCLUDED.attempt_id, repair_reason='Derived manifest repair pending',repair_error_json=NULL,updated_at=EXCLUDED.updated_at`, - [id, values.recordJson, values.canonicalManifestJson, values.manifestRevision, values.activationId, attemptId] + [ + id, + derived.recordJson, + derived.canonicalManifestJson, + derived.manifestRevision, + derived.activationId, + attemptId, + ] ); await client.query( `INSERT INTO connector_installs(connector_id,record_json,updated_at) VALUES($1,$2,clock_timestamp()::text) ON CONFLICT(connector_id) DO UPDATE SET record_json=EXCLUDED.record_json,updated_at=EXCLUDED.updated_at`, - [id, values.recordJson] + [id, derived.recordJson] ); + return derived; }); } else { - getDb() + values = getDb() .transaction(() => { const db = getDb(); const old = db.prepare("SELECT manifest FROM connectors WHERE connector_id = ?").get<{ manifest: string }>(id); - if (!old || canonicalJson(JSON.parse(old.manifest)) !== values.canonicalManifestJson) { + const derived = activationValues(record, retainedStorageContext(old?.manifest)); + assertRetainedSourceIdentity(id, old?.manifest, derived.canonicalManifestJson); + if (!old || canonicalJson(JSON.parse(old.manifest)) !== derived.canonicalManifestJson) { db.prepare( "INSERT INTO connectors(connector_id,manifest) VALUES(?,?) ON CONFLICT(connector_id) DO UPDATE SET manifest=excluded.manifest" - ).run(id, values.canonicalManifestJson); + ).run(id, derived.canonicalManifestJson); db.prepare( "UPDATE connector_instances SET manifest_generation=manifest_generation+1 WHERE connector_id=?" ).run(id); @@ -163,15 +197,16 @@ export async function publishConnectorActivation(record: ConnectorInstallRecord) manifest_revision=excluded.manifest_revision,activation_id=excluded.activation_id,attempt_id=excluded.attempt_id, repair_reason='Derived manifest repair pending',repair_error_json=NULL,updated_at=excluded.updated_at`).run( id, - values.recordJson, - values.canonicalManifestJson, - values.manifestRevision, - values.activationId, + derived.recordJson, + derived.canonicalManifestJson, + derived.manifestRevision, + derived.activationId, attemptId ); db.prepare( "INSERT INTO connector_installs(connector_id,record_json,updated_at) VALUES(?,?,datetime('now')) ON CONFLICT(connector_id) DO UPDATE SET record_json=excluded.record_json,updated_at=excluded.updated_at" - ).run(id, values.recordJson); + ).run(id, derived.recordJson); + return derived; }) .immediate(); } @@ -311,11 +346,11 @@ async function migrateLegacyActivation(connectorId: string): Promise { return; } const record = JSON.parse(legacy.rows[0].record_json) as ConnectorInstallRecord; - const values = activationValues(record); const registry = await client.query<{ manifest: unknown }>( "SELECT manifest FROM connectors WHERE connector_id=$1", [connectorId] ); + const values = activationValues(record, retainedStorageContext(registry.rows[0]?.manifest)); const coherent = registry.rows[0] && canonicalJson(registry.rows[0].manifest) === values.canonicalManifestJson; await client.query( `INSERT INTO connector_activations(connector_id,state,record_json,canonical_manifest_json,manifest_revision,activation_id,attempt_id,repair_reason,updated_at) @@ -346,10 +381,10 @@ async function migrateLegacyActivation(connectorId: string): Promise { return; } const record = JSON.parse(legacy.record_json) as ConnectorInstallRecord; - const values = activationValues(record); const registry = db .prepare("SELECT manifest FROM connectors WHERE connector_id=?") .get<{ manifest: string }>(connectorId); + const values = activationValues(record, retainedStorageContext(registry?.manifest)); const coherent = registry && canonicalJson(JSON.parse(registry.manifest)) === values.canonicalManifestJson; db.prepare(`INSERT INTO connector_activations(connector_id,state,record_json,canonical_manifest_json,manifest_revision,activation_id,attempt_id,repair_reason,updated_at) VALUES(?,?,?,?,?,?,?,?,datetime('now'))`).run( diff --git a/reference-implementation/server/connector-install/index.ts b/reference-implementation/server/connector-install/index.ts index dcf2c284a..17f6b7321 100644 --- a/reference-implementation/server/connector-install/index.ts +++ b/reference-implementation/server/connector-install/index.ts @@ -22,8 +22,10 @@ import { writeFileSync, } from "node:fs"; import { dirname, join, resolve, sep } from "node:path"; +import { isDeepStrictEqual } from "node:util"; import { crashPoint } from "../crash-point.ts"; import { canonicalConnectorKey } from "../connector-key.ts"; +import { snapshotSourceDeclaration } from "../source-declaration.ts"; import { assertConnectorLifecycleAvailable, clearConnectorLifecycleUninstalled, @@ -48,6 +50,7 @@ const CORE_MODULE = "@opendatalabs/data-connectors-tools/installer-core"; const MAX_CONFIG_BYTES = 1024 * 1024; const DIGEST = /^sha256:[a-f0-9]{64}$/; const CONNECTOR_ID = /^[a-z0-9][a-z0-9-]*$/; +const SOURCE_DECLARATION_PATH = "source-declaration.json"; const MANIFEST_REQUEST = /\/manifests\/(sha256:[0-9a-f]{64})$/; const BLOB_REQUEST = /\/blobs\/(sha256:[0-9a-f]{64})$/; @@ -64,6 +67,11 @@ export interface ConnectorInstallRecord { readonly manifestSha256: string; readonly provenancePath: string; readonly provenanceSha256: string; + /** + * Digest of the verified `source-declaration.json` sidecar. Absent on legacy records: those keep their + * retained registry identity, and their sidecar bytes are never read as trusted. + */ + readonly sourceDeclarationSha256?: string; readonly registry: "ghcr.io"; readonly repository: string; readonly root: string; @@ -420,7 +428,8 @@ function assertStoredRecordMetadata(active: ConnectorInstallRecord): void { active.repository !== `pdp-connect/connector/${active.connectorId}` || active.manifestPath !== "profile/collection-profile.json" || active.entrypointPath !== "dist/collection-profile.mjs" || - active.provenancePath !== "provenance.json" + active.provenancePath !== "provenance.json" || + (active.sourceDeclarationSha256 !== undefined && !DIGEST.test(active.sourceDeclarationSha256)) ) { throw new Error("Active connector record is invalid."); } @@ -591,7 +600,58 @@ function verifyStoredRecord(root: string, active: ConnectorInstallRecord, dataDi if (manifestKey !== active.connectorId) { throw new Error("Active connector manifest identity does not match its record."); } - return { ...active, manifest, root: normalizedRoot }; + return { + ...active, + manifest: withVerifiedSourceDeclaration(normalizedRoot, active, manifest), + root: normalizedRoot, + }; +} + +/** + * Binds the verified Core source declaration to the profile manifest that registration reads. Only a + * record that carries the sidecar digest may supply it, so a legacy record (no digest) keeps its + * retained registry identity and its sidecar bytes are never trusted. + */ +function withVerifiedSourceDeclaration( + root: string, + active: ConnectorInstallRecord, + manifest: Record +): Record { + if (active.sourceDeclarationSha256 === undefined) { + return manifest; + } + const file = confinedFile(root, SOURCE_DECLARATION_PATH); + if (!file || lstatSync(file).size > MAX_CONFIG_BYTES || sha256(file) !== active.sourceDeclarationSha256) { + throw new Error("Active source declaration bytes failed integrity verification."); + } + const declaration = snapshotSourceDeclaration(JSON.parse(readFileSync(file, "utf8"))); + if (manifest.source_declaration !== undefined && !isDeepStrictEqual(manifest.source_declaration, declaration)) { + throw new Error( + "Connector profile embeds a source declaration that conflicts with the verified source declaration." + ); + } + return { ...manifest, source_declaration: declaration }; +} + +/** + * The sidecar digest when a source declaration file is present; undefined only when it is genuinely absent + * (ENOENT). Any other inspection error propagates: an unreadable sidecar is not a legacy root. + */ +function sourceDeclarationDigestOf(root: string): string | undefined { + const path = join(root, SOURCE_DECLARATION_PATH); + try { + lstatSync(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return undefined; + } + throw error; + } + const file = confinedFile(root, SOURCE_DECLARATION_PATH); + if (!file || lstatSync(file).size > MAX_CONFIG_BYTES) { + throw new Error("Verified source declaration is invalid."); + } + return sha256(file); } function assertCatalogEntry(entry: ConnectorCatalogEntry, connectorId: string, digest?: string): void { @@ -611,7 +671,35 @@ function assertCatalogEntry(entry: ConnectorCatalogEntry, connectorId: string, d } } -function readVerifiedRecord(root: string, entry: ConnectorCatalogEntry): ConnectorInstallRecord { +/** + * A pinned install must carry exactly the sidecar its installer core verified. Absence and any other digest are + * refusals; no pinned digest means a legacy install, which accepts no sidecar check here. + */ +function assertPinnedSourceDeclaration(observed: string | undefined, pinned: string | undefined): void { + if (pinned === undefined) { + return; + } + if (!DIGEST.test(pinned)) { + throw new Error("Pinned source declaration digest is invalid."); + } + if (observed === undefined) { + throw new Error("Verified source declaration is missing for a pinned install."); + } + if (observed !== pinned) { + throw new Error("Verified source declaration does not match installer-core verification."); + } +} + +/** + * Reads a root into an install record. `pinnedSourceDeclarationSha256` is the digest the pinned installer core + * verified: the sidecar must then exist and match it, and the record takes that digest as its authority. Without + * it (legacy installers), the sidecar is optional and its digest is observed from the staged bytes. + */ +function readVerifiedRecord( + root: string, + entry: ConnectorCatalogEntry, + pinnedSourceDeclarationSha256?: string +): ConnectorInstallRecord { const manifestPath = "profile/collection-profile.json"; const entrypointPath = "dist/collection-profile.mjs"; const provenancePath = "provenance.json"; @@ -634,6 +722,9 @@ function readVerifiedRecord(root: string, entry: ConnectorCatalogEntry): Connect } const manifestRecord = manifest as Record; const manifestKey = canonicalManifestKey(manifestRecord); + const observedSourceDeclarationSha256 = sourceDeclarationDigestOf(root); + assertPinnedSourceDeclaration(observedSourceDeclarationSha256, pinnedSourceDeclarationSha256); + const sourceDeclarationSha256 = pinnedSourceDeclarationSha256 ?? observedSourceDeclarationSha256; if (manifestKey !== entry.connector_key) { throw new Error("Connector profile identity does not match the catalog target."); } @@ -654,6 +745,7 @@ function readVerifiedRecord(root: string, entry: ConnectorCatalogEntry): Connect manifestSha256: sha256(manifestFile), provenancePath, provenanceSha256: sha256(provenanceFile), + ...(sourceDeclarationSha256 === undefined ? {} : { sourceDeclarationSha256 }), registry: "ghcr.io", repository: `pdp-connect/connector/${entry.connector_key}`, root, @@ -704,12 +796,37 @@ async function reuseExistingInstall( return verified; } +/** + * A pinned installer reports the config digest it verified and the digest of the source-declaration bytes it + * verified. The staged install must then carry exactly those bytes; a missing or different sidecar is refused. + */ +interface PinnedInstallResult { + readonly configDigest: string; + readonly sourceDeclarationSha256: string; +} + +type InstallArtifactResult = + | string + | PinnedInstallResult + // biome-ignore lint/suspicious/noConfusingVoidType: Test fixtures may use synchronous installers. + | void; + +/** Legacy installers report a config digest string or nothing; the sidecar digest is then optional. */ type InstallArtifact = ( root: string, entry: ConnectorCatalogEntry, signal: AbortSignal - // biome-ignore lint/suspicious/noConfusingVoidType: Test fixtures may use synchronous installers. -) => Promise | string | void; +) => Promise | InstallArtifactResult; + +function installResultOf(result: InstallArtifactResult): { + readonly configDigest?: string; + readonly sourceDeclarationSha256?: string; +} { + if (typeof result === "string") { + return { configDigest: result }; + } + return result ? { configDigest: result.configDigest, sourceDeclarationSha256: result.sourceDeclarationSha256 } : {}; +} async function installStagedEntry(options: { readonly connectorId: string; @@ -726,13 +843,13 @@ async function installStagedEntry(options: { const staged = join(transaction, "next"); let published = false; try { - const discoveredConfigDigest = await options.installArtifact(staged, options.entry, options.signal); + const installed = installResultOf(await options.installArtifact(staged, options.entry, options.signal)); options.signal.throwIfAborted(); - const verifiedEntry = discoveredConfigDigest - ? { ...options.entry, config_digest: discoveredConfigDigest } + const verifiedEntry = installed.configDigest + ? { ...options.entry, config_digest: installed.configDigest } : options.entry; assertCatalogEntry(verifiedEntry, options.connectorId, options.entry.digest); - const record = readVerifiedRecord(staged, verifiedEntry); + const record = readVerifiedRecord(staged, verifiedEntry, installed.sourceDeclarationSha256); renameSync(staged, options.root); published = true; const active = { ...record, root: options.root }; @@ -1395,7 +1512,7 @@ async function installPinnedArtifact( root: string, entry: ConnectorCatalogEntry, signal: AbortSignal -): Promise { +): Promise { const core = await loadPinnedCore(); const abortableFetch: FetchLike = (input, init) => fetch(input, { ...init, signal: init?.signal ? AbortSignal.any([init.signal, signal]) : signal }); @@ -1485,7 +1602,10 @@ async function installPinnedArtifact( ); } normalizeCoreInstallLayout(root, entry.connector_id, verifiedSourceDeclaration); - return configDigest; + return { + configDigest, + sourceDeclarationSha256: `sha256:${createHash("sha256").update(verifiedSourceDeclaration).digest("hex")}`, + }; } export function normalizeCoreInstallLayout( diff --git a/reference-implementation/server/connector-install/local-source.test.ts b/reference-implementation/server/connector-install/local-source.test.ts index c2d1790d7..e7f260bec 100644 --- a/reference-implementation/server/connector-install/local-source.test.ts +++ b/reference-implementation/server/connector-install/local-source.test.ts @@ -1,22 +1,31 @@ // Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 -import { strict as assert } from "node:assert" +import assert from "node:assert/strict" +import { createHash } from "node:crypto" import { + existsSync, mkdirSync, mkdtempSync, + readFileSync, rmSync, writeFileSync, - existsSync, } from "node:fs" import { join } from "node:path" import { afterEach, test } from "node:test" +import { withFixtureSourceDeclaration } from "../../test/helpers/fixture-source-declaration.ts" import { createFileLocalConnectorSourceStore, inspectActiveLocalConnectorSource, } from "./local-source.ts" const temporaryRoots: string[] = [] +const RE_LOCAL_EXPAND_INVALID = /query\.expand.*same-stream relationships\[\] entry/ +const RE_LOCAL_IDENTITY_MISMATCH = /connector_key must match the canonical key for connector_id/ +const RE_MALFORMED_LOCAL_BINDING = /runtime_requirements\.bindings\.usb must have required property 'kind'/ +const RE_MISSING_LOCAL_KEY = /connector_key/ +const RE_DUPLICATE_LOCAL_JSON_KEY = /duplicate JSON object keys/ +const RE_INVALID_ARTIFACT_URI = /connector_id must be an absolute URI or its declared connector_key/ afterEach(() => { for (const root of temporaryRoots.splice(0)) { @@ -27,6 +36,7 @@ afterEach(() => { function fixture( options: { readonly key?: string + readonly connectorId?: string readonly version?: string readonly binding?: string } = {} @@ -35,19 +45,18 @@ function fixture( temporaryRoots.push(root) mkdirSync(join(root, "profile")) mkdirSync(join(root, "dist")) - writeFileSync( - join(root, "profile", "collection-profile.json"), - JSON.stringify({ - connector_id: `https://registry.pdpp.dev/connectors/${options.key ?? "developer-fixture"}`, - connector_key: options.key ?? "developer-fixture", - display_name: "Developer fixture", - runtime_requirements: options.binding - ? { bindings: { [options.binding]: { required: true } } } - : { bindings: { network: { required: true } } }, - streams: [{ name: "items" }], - version: options.version ?? "0.1.0", - }) - ) + const manifest = JSON.parse( + readFileSync(new URL("../../test/fixtures/collection-profiles/github.json", import.meta.url), "utf8") + ) as Record + const connectorKey = options.key ?? "developer-fixture" + manifest.connector_id = options.connectorId ?? `https://vendor.example/connectors/${connectorKey}` + manifest.manifest_uri = manifest.connector_id + manifest.connector_key = connectorKey + manifest.version = options.version ?? "0.1.0" + if (options.binding) { + manifest.runtime_requirements = { bindings: { [options.binding]: { required: true } } } + } + writeFileSync(join(root, "profile", "collection-profile.json"), JSON.stringify(manifest)) writeFileSync( join(root, "dist", "collection-profile.mjs"), "export default {};\n" @@ -55,6 +64,16 @@ function fixture( return root } +function invalidateFirstExpand(root: string): void { + const path = join(root, "profile", "collection-profile.json") + const manifest = JSON.parse(readFileSync(path, "utf8")) as Record + const streams = manifest.streams as Record[] + const [firstStream] = streams + assert.ok(firstStream) + firstStream.query = { expand: [{ name: "missing_relationship" }] } + writeFileSync(path, JSON.stringify(manifest)) +} + test("local sources coexist by key, select explicitly, and remain outside verified installs", async () => { const dataDir = mkdtempSync(join(process.cwd(), "tmp-local-source-state-")) temporaryRoots.push(dataDir) @@ -94,6 +113,96 @@ test("local sources coexist by key, select explicitly, and remain outside verifi assert.equal((await store.list()).length, 1) }) +test("unknown URL artifacts require an explicit key and preserve original manifest bytes", async () => { + const dataDir = mkdtempSync(join(process.cwd(), "tmp-local-source-state-")) + temporaryRoots.push(dataDir) + const sourcePath = fixture({ key: "custom-provider" }) + const manifestPath = join(sourcePath, "profile", "collection-profile.json") + const originalText = readFileSync(manifestPath, "utf8") + const originalManifest = JSON.parse(originalText) as Record + originalManifest.source_declaration = withFixtureSourceDeclaration(originalManifest).source_declaration + writeFileSync(manifestPath, JSON.stringify(originalManifest)) + const originalBytes = readFileSync(manifestPath) + const store = createFileLocalConnectorSourceStore(dataDir) + + const record = await store.add(sourcePath) + + assert.equal(record.connectorId, "https://vendor.example/connectors/custom-provider") + assert.equal(record.connectorKey, "custom-provider") + assert.equal(record.manifest.manifest_uri, record.connectorId) + assert.deepEqual(record.manifest, originalManifest) + assert.deepEqual(record.manifest.source_declaration, originalManifest.source_declaration) + assert.equal(record.manifestSha256, `sha256:${createHash("sha256").update(originalBytes).digest("hex")}`) + assert.deepEqual(readFileSync(manifestPath), originalBytes) +}) + +test("local add rejects invalid profiles before publishing source state", async () => { + const dataDir = mkdtempSync(join(process.cwd(), "tmp-local-source-state-")) + temporaryRoots.push(dataDir) + const invalidPath = fixture({ key: "invalid-expand" }) + const store = createFileLocalConnectorSourceStore(dataDir) + const baseline = await store.list() + + invalidateFirstExpand(invalidPath) + await assert.rejects(() => store.add(invalidPath), RE_LOCAL_EXPAND_INVALID) + assert.deepEqual(await store.list(), baseline) +}) + +test("local reload rejects invalid profiles without changing published state", async () => { + const dataDir = mkdtempSync(join(process.cwd(), "tmp-local-source-state-")) + temporaryRoots.push(dataDir) + const validPath = fixture() + const store = createFileLocalConnectorSourceStore(dataDir) + const existing = await store.add(validPath) + const baseline = await store.list() + + invalidateFirstExpand(validPath) + await assert.rejects(() => store.reload(existing.sourceId), RE_LOCAL_EXPAND_INVALID) + assert.deepEqual(await store.list(), baseline) +}) + +test("local admission refuses a recognized URL and explicit key conflict", async () => { + const dataDir = mkdtempSync(join(process.cwd(), "tmp-local-source-state-")) + temporaryRoots.push(dataDir) + const store = createFileLocalConnectorSourceStore(dataDir) + await assert.rejects( + () => store.add(fixture({ connectorId: "https://registry.pdpp.dev/connectors/github", key: "different-key" })), + RE_LOCAL_IDENTITY_MISMATCH + ) + assert.deepEqual(await store.list(), []) + + const missingKeyPath = fixture() + const manifestPath = join(missingKeyPath, "profile", "collection-profile.json") + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record + const manifestWithoutKey = Object.fromEntries(Object.entries(manifest).filter(([key]) => key !== "connector_key")) + writeFileSync(manifestPath, JSON.stringify(manifestWithoutKey)) + await assert.rejects(() => store.add(missingKeyPath), RE_MISSING_LOCAL_KEY) + assert.deepEqual(await store.list(), []) + + const duplicateKeyPath = fixture() + const duplicateManifestPath = join(duplicateKeyPath, "profile", "collection-profile.json") + const validText = readFileSync(duplicateManifestPath, "utf8") + const duplicateText = validText.replace( + '"connector_key":"developer-fixture"', + '"connector_key":"developer-fixture","connector_key":"other-key"' + ) + assert.notEqual(duplicateText, validText) + writeFileSync(duplicateManifestPath, duplicateText) + await assert.rejects(() => store.add(duplicateKeyPath), RE_DUPLICATE_LOCAL_JSON_KEY) + assert.deepEqual(await store.list(), []) +}) + +test("local admission rejects a malformed URL before publishing source state", async () => { + const dataDir = mkdtempSync(join(process.cwd(), "tmp-local-source-state-")) + temporaryRoots.push(dataDir) + const store = createFileLocalConnectorSourceStore(dataDir) + await assert.rejects( + () => store.add(fixture({ connectorId: "https://[invalid", key: "malformed-uri" })), + RE_INVALID_ARTIFACT_URI + ) + assert.deepEqual(await store.list(), []) +}) + test("reload refreshes hashes and drift blocks a selected local run until reload", async () => { const dataDir = mkdtempSync(join(process.cwd(), "tmp-local-source-state-")) temporaryRoots.push(dataDir) @@ -119,12 +228,12 @@ test("reload refreshes hashes and drift blocks a selected local run until reload ) }) -test("local admission rejects unsupported runtime bindings", async () => { +test("local admission rejects malformed runtime binding declarations", async () => { const dataDir = mkdtempSync(join(process.cwd(), "tmp-local-source-state-")) temporaryRoots.push(dataDir) const store = createFileLocalConnectorSourceStore(dataDir) await assert.rejects( - () => store.add(fixture({ key: "bad-binding", binding: "usb" })), - /unsupported runtime binding/ + () => store.add(fixture({ binding: "usb", key: "bad-binding" })), + RE_MALFORMED_LOCAL_BINDING ) }) diff --git a/reference-implementation/server/connector-install/local-source.ts b/reference-implementation/server/connector-install/local-source.ts index aaf490a92..56a8644a7 100644 --- a/reference-implementation/server/connector-install/local-source.ts +++ b/reference-implementation/server/connector-install/local-source.ts @@ -20,14 +20,18 @@ import { writeFileSync, } from "node:fs" import { isAbsolute, join, relative, resolve, sep } from "node:path" +import { findDuplicateJsonKeys } from "../../runtime/connector-binding-grammar.mjs" import { canonicalConnectorKey } from "../connector-key.ts" +import { + projectLocalConnectorArtifactForManifestValidation, + validateConnectorManifest, +} from "../connector-manifest-validation.ts" const MANIFEST_PATH = "profile/collection-profile.json" const ENTRYPOINT_PATH = "dist/collection-profile.mjs" const PROVENANCE_PATH = "provenance.json" const MAX_PROFILE_BYTES = 1024 * 1024 const CONNECTOR_KEY = /^[a-z0-9][a-z0-9._-]*$/ -const LOCAL_BINDINGS = new Set(["browser", "filesystem", "network"]) export interface LocalConnectorSourceRecord { readonly connectorId: string @@ -130,6 +134,23 @@ function canonicalRoot(sourcePath: string): string { return root } +type LocalManifestJson = + | { readonly kind: "parsed"; readonly value: unknown } + | { readonly kind: "invalid_json" } + | { readonly kind: "duplicate_key" } + +/** Parser failures stay inside this boundary because their messages can contain file contents. */ +function decodeLocalManifestJson(text: string): LocalManifestJson { + try { + if (findDuplicateJsonKeys(text).length > 0) { + return { kind: "duplicate_key" } + } + return { kind: "parsed", value: JSON.parse(text) } + } catch { + return { kind: "invalid_json" } + } +} + function readManifest(root: string): { readonly connectorId: string readonly connectorKey: string @@ -145,15 +166,14 @@ function readManifest(root: string): { if (lstatSync(manifestPath).size > MAX_PROFILE_BYTES) { throw new Error("Collection Profile manifest exceeds the 1 MiB limit.") } - let parsed: unknown - try { - parsed = JSON.parse(readFileSync(manifestPath, "utf8")) - } catch (error) { - throw new Error( - `Collection Profile manifest is invalid: ${error instanceof Error ? error.message : String(error)}`, - { cause: error } - ) + const decoded = decodeLocalManifestJson(readFileSync(manifestPath, "utf8")) + if (decoded.kind === "invalid_json") { + throw new Error("Collection Profile manifest is invalid JSON.") } + if (decoded.kind === "duplicate_key") { + throw new Error("Collection Profile manifest contains duplicate JSON object keys.") + } + const parsed = decoded.value assertRecord(parsed, "Collection Profile manifest") const connectorId = parsed.connector_id const connectorKey = parsed.connector_key @@ -170,39 +190,13 @@ function readManifest(root: string): { "Collection Profile manifest must declare a valid connector_id, connector_key, and version." ) } + validateConnectorManifest(projectLocalConnectorArtifactForManifestValidation(parsed)) const streams = parsed.streams if (!Array.isArray(streams) || streams.length === 0) { throw new Error( "Collection Profile manifest must declare at least one stream." ) } - const requirements = parsed.runtime_requirements - if (requirements !== undefined && requirements !== null) { - assertRecord(requirements, "Collection Profile runtime_requirements") - const bindings = requirements.bindings - if (bindings !== undefined && bindings !== null) { - assertRecord(bindings, "Collection Profile runtime_requirements.bindings") - for (const [binding, requirement] of Object.entries(bindings)) { - if (!LOCAL_BINDINGS.has(binding)) { - throw new Error( - `Collection Profile declares unsupported runtime binding: ${binding}` - ) - } - assertRecord( - requirement, - `Collection Profile runtime binding ${binding}` - ) - if ( - requirement.required !== undefined && - typeof requirement.required !== "boolean" - ) { - throw new Error( - `Collection Profile runtime binding ${binding}.required must be a boolean.` - ) - } - } - } - } return { connectorId, connectorKey, diff --git a/reference-implementation/server/connector-legacy-source-map.ts b/reference-implementation/server/connector-legacy-source-map.ts new file mode 100644 index 000000000..8027e6786 --- /dev/null +++ b/reference-implementation/server/connector-legacy-source-map.ts @@ -0,0 +1,23 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import { LEGACY_FIRST_PARTY_SOURCE_ENTRIES } from "./generated/connector-registry.generated.ts"; + +export const LEGACY_SOURCE_MAP_VERSION = 1; + +const legacySourceIds: Readonly> = Object.freeze( + Object.fromEntries(LEGACY_FIRST_PARTY_SOURCE_ENTRIES) +); + +/** Return the reviewed historical source identity for a canonical key. */ +export function legacyFirstPartySourceIdFor(connectorKey: string): string | undefined { + return Object.hasOwn(legacySourceIds, connectorKey) ? legacySourceIds[connectorKey] : undefined; +} + +export const LEGACY_FIRST_PARTY_KEYS: readonly string[] = Object.freeze( + LEGACY_FIRST_PARTY_SOURCE_ENTRIES.map(([key]) => key) +); + +export const LEGACY_FIRST_PARTY_ENTRIES: ReadonlyArray = Object.freeze( + LEGACY_FIRST_PARTY_SOURCE_ENTRIES.map(([key, sourceId]) => Object.freeze([key, sourceId] as const)) +); diff --git a/reference-implementation/server/connector-manifest-validation.ts b/reference-implementation/server/connector-manifest-validation.ts index b2213ea93..7b59160df 100644 --- a/reference-implementation/server/connector-manifest-validation.ts +++ b/reference-implementation/server/connector-manifest-validation.ts @@ -10,7 +10,8 @@ * logic. */ -import { collectionProfileDeclarationError } from "@pdpp/connector-protocol"; +import { BindingDeclarationError, normalizeBindingDeclarations } from "../runtime/binding-declarations.ts"; +import { collectionProfileDeclarationProblem } from "../runtime/collection-profile-support.ts"; import { tuningEnvironmentKeyOwner } from "../runtime/connector-child-environment.ts"; import { normalizeStaticSecretCredentialCapture, @@ -183,6 +184,36 @@ export function invalidConnectorManifest(message: string, code = "invalid_reques return err; } +/** + * Prepare a developer-local artifact for the host's full manifest validator. + * + * Unknown URL identities are not inferred into connector keys. The artifact + * must declare its own key; this copy uses that key as the operational id only + * for validation. The returned projection is not a trusted or retained source + * identity, and callers must keep the original manifest unchanged. + */ +export function projectLocalConnectorArtifactForManifestValidation( + manifest: Record +): Record { + const connectorId = manifest.connector_id; + const connectorKey = manifest.connector_key; + if (!isNonEmptyString(connectorId)) { + throw invalidConnectorManifest("Collection Profile manifest must declare connector_id."); + } + if (typeof connectorKey !== "string" || !isConnectorKey(connectorKey)) { + throw invalidConnectorManifest("Collection Profile manifest must declare a valid connector_key."); + } + const normalizedKey = connectorKey.trim(); + const recognizedKey = canonicalConnectorKey(connectorId); + if (recognizedKey !== null && recognizedKey !== normalizedKey) { + throw invalidConnectorManifest("connector_key must match the canonical key for connector_id"); + } + if (recognizedKey === null && connectorId.trim() !== normalizedKey && !URL.canParse(connectorId)) { + throw invalidConnectorManifest("connector_id must be an absolute URI or its declared connector_key"); + } + return { ...manifest, connector_id: normalizedKey }; +} + /** * Collection Profile declaration gate (profile Sections 3.1 and 3.4): * `protocol_version` names a supported profile version, every @@ -191,12 +222,9 @@ export function invalidConnectorManifest(message: string, code = "invalid_reques * manifest without `protocol_version`; it is checked as a 0.1.0 manifest. */ function validateCollectionProfileDeclaration(manifest: Record, code: string): void { - const error = collectionProfileDeclarationError( - manifest.protocol_version === undefined ? "0.1.0" : manifest.protocol_version, - manifest.protocol_capabilities - ); - if (error !== null) { - throw invalidConnectorManifest(error, code); + const problem = collectionProfileDeclarationProblem(manifest); + if (problem !== null) { + throw invalidConnectorManifest(problem, code); } } @@ -283,13 +311,6 @@ export const REFRESH_POLICY_ALLOWED_KEYS = new Set([ // production code, but the read-site clamp holds even if it does. export const REFRESH_POLICY_MAX_COOLDOWN_CYCLES_RANGE = { max: 24, min: 1 } as const; export const REFRESH_POLICY_MAX_RECOVERY_ATTEMPTS_RANGE = { max: 20, min: 1 } as const; -export const RUNTIME_REQUIREMENT_BINDINGS = new Set([ - "browser", - "desktop_session", - "filesystem", - "interactive", - "network", -]); export const STREAM_AVAILABILITY_STATES = new Set(["supported", "unsupported_in_mode", "experimental", "deprecated"]); export const STREAM_AVAILABILITY_ALLOWED_KEYS = new Set(["future_modes", "mode", "reason", "state"]); export const STREAM_COVERAGE_POLICIES = new Set([ @@ -326,38 +347,19 @@ const SUPPORTED_RANGE_OPERATORS = new Set(["gte", "gt", "lte", "lt"]); const EXTERNAL_TOOL_ALLOWED_KEYS = new Set(["detect", "install_hint", "license", "min_version", "name", "purpose"]); const EXTERNAL_TOOL_DETECT_ALLOWED_KEYS = new Set(["args", "executable", "executable_env_override", "exit_code"]); -// Validates `runtime_requirements.bindings` (order-preserving). Returns `false` -// when `bindings` is absent — the original validator returns from the whole -// function in that case, so the caller must NOT go on to external_tools (a -// runtime_requirements object without bindings is accepted verbatim). Returns -// `true` once bindings are present and fully validated. -function validateRuntimeBindings(req: Record, code: string): boolean { - // biome-ignore lint/style/useDestructuring: Explicit property or positional access documents this compatibility boundary. - const bindings = req.bindings; - if (bindings === undefined || bindings === null) { - return false; - } - if (typeof bindings !== "object" || Array.isArray(bindings)) { - throw invalidConnectorManifest("runtime_requirements.bindings must be an object when declared", code); - } - const bindingsObj = bindings as Record; - const unknownBindings = Object.keys(bindingsObj).filter((binding) => !RUNTIME_REQUIREMENT_BINDINGS.has(binding)); - if (unknownBindings.length) { - throw invalidConnectorManifest( - `runtime_requirements.bindings has unsupported keys: ${unknownBindings.join(", ")}`, - code - ); - } - for (const [binding, requirement] of Object.entries(bindingsObj)) { - if (!requirement || typeof requirement !== "object" || Array.isArray(requirement)) { - throw invalidConnectorManifest(`runtime_requirements.bindings.${binding} must be an object`, code); - } - const reqObj = requirement as Record; - if (reqObj.required !== undefined && typeof reqObj.required !== "boolean") { - throw invalidConnectorManifest(`runtime_requirements.bindings.${binding}.required must be a boolean`, code); +// Validates `runtime_requirements.bindings` with the shared canonical validator +// (runtime/binding-declarations.ts), the same one `runConnector` runs before spawn. +// An absent map is valid. A present one must satisfy the canonical schema and +// the grammar rules, or registration refuses it with the validator's message. +function validateRuntimeBindings(req: Record, code: string): void { + try { + normalizeBindingDeclarations(req.bindings); + } catch (error) { + if (error instanceof BindingDeclarationError) { + throw invalidConnectorManifest(error.message, code); } + throw error; } - return true; } function validateExternalToolDetectExitCode(detect: Record, index: number, code: string): void { @@ -652,9 +654,7 @@ export function validateRuntimeRequirements(manifest: Record, c } const req = requirements as Record; validateTuningEnvironment(manifest, req, code); - if (!validateRuntimeBindings(req, code)) { - return; - } + validateRuntimeBindings(req, code); validateExternalTools(req, code, { allowLegacyCommand: !Array.isArray(manifest.streams) }); validateLocalPaths(req, code); } @@ -797,6 +797,37 @@ function validateRefreshPolicyInternalConsistency(pol: Record, } } +// Section 3.4: `capabilities.human_interaction` is an optional array of distinct +// values from this closed set. An INTERACTION.kind outside it is refused at run time. +const HUMAN_INTERACTION_VALUES = new Set(["credentials", "otp", "manual_action"]); + +function validateHumanInteractionCapability(manifest: Record, code: string): void { + const { capabilities } = manifest; + if (typeof capabilities !== "object" || capabilities === null || Array.isArray(capabilities)) { + return; + } + const values = (capabilities as Record).human_interaction; + if (values === undefined) { + return; + } + if (!Array.isArray(values)) { + throw invalidConnectorManifest("capabilities.human_interaction must be an array when declared", code); + } + const seen = new Set(); + for (const value of values) { + if (!HUMAN_INTERACTION_VALUES.has(value as string)) { + throw invalidConnectorManifest( + `capabilities.human_interaction value ${JSON.stringify(value)} must be one of: credentials, otp, manual_action`, + code + ); + } + if (seen.has(value)) { + throw invalidConnectorManifest(`capabilities.human_interaction repeats ${String(value)}`, code); + } + seen.add(value); + } +} + export function validateRefreshPolicyCapability(manifest: Record, code: string): void { validatePublicListingTier(manifest, code); // biome-ignore lint/style/useDestructuring: Explicit property or positional access documents this compatibility boundary. @@ -2232,6 +2263,7 @@ export function validateConnectorManifest( validateCollectionProfileDeclaration(manifest, code); } validateRuntimeRequirements(manifest, code); + validateHumanInteractionCapability(manifest, code); validateRefreshPolicyCapability(manifest, code); validateDeclaredReasonTokensCapability(manifest, code); validateProvenCapability(manifest, code); diff --git a/reference-implementation/server/generated/connector-registry.generated.ts b/reference-implementation/server/generated/connector-registry.generated.ts index 9c0c8a220..cc915daf9 100644 --- a/reference-implementation/server/generated/connector-registry.generated.ts +++ b/reference-implementation/server/generated/connector-registry.generated.ts @@ -45,3 +45,6 @@ export const PROVIDER_AUTH_LIFECYCLE_PROVEN_KEYS: readonly string[] = Object.fre /** Manifests declaring capabilities.proven.static_secret_live.proven === true. */ export const STATIC_SECRET_LIVE_PROVEN_KEYS: readonly string[] = Object.freeze(["github", "gmail", "slack", "ynab"]); + +/** Reviewed historical source bindings derived from the pinned connector package. */ +export const LEGACY_FIRST_PARTY_SOURCE_ENTRIES: readonly (readonly [string, string])[] = Object.freeze(([["amazon", "https://registry.pdpp.dev/connectors/amazon"], ["anthropic", "https://registry.pdpp.dev/connectors/anthropic"], ["apple-contacts", "https://registry.pdpp.dev/connectors/apple-contacts"], ["apple-health", "https://registry.pdpp.dev/connectors/apple-health"], ["apple-photos", "https://registry.pdpp.dev/connectors/apple-photos"], ["chase", "https://registry.pdpp.dev/connectors/chase"], ["chatgpt", "https://registry.pdpp.dev/connectors/chatgpt"], ["claude-code", "https://registry.pdpp.dev/connectors/claude-code"], ["codex", "https://registry.pdpp.dev/connectors/codex"], ["doordash", "https://registry.pdpp.dev/connectors/doordash"], ["github", "https://registry.pdpp.dev/connectors/github"], ["gmail", "https://registry.pdpp.dev/connectors/gmail"], ["google-calendar", "https://registry.pdpp.dev/connectors/google-calendar"], ["google-contacts", "https://registry.pdpp.dev/connectors/google-contacts"], ["google-maps", "https://registry.pdpp.dev/connectors/google-maps"], ["google-maps-data-portability", "https://registry.pdpp.dev/connectors/google-maps-data-portability"], ["google-messages", "https://registry.pdpp.dev/connectors/google-messages"], ["google-takeout", "https://registry.pdpp.dev/connectors/google-takeout"], ["groupme", "https://registry.pdpp.dev/connectors/groupme"], ["heb", "https://registry.pdpp.dev/connectors/heb"], ["ical", "https://registry.pdpp.dev/connectors/ical"], ["imessage", "https://registry.pdpp.dev/connectors/imessage"], ["jellyfin", "https://registry.pdpp.dev/connectors/jellyfin"], ["linkedin", "https://registry.pdpp.dev/connectors/linkedin"], ["loom", "https://registry.pdpp.dev/connectors/loom"], ["meta", "https://registry.pdpp.dev/connectors/meta"], ["netflix-export", "https://registry.pdpp.dev/connectors/netflix-export"], ["notion", "https://registry.pdpp.dev/connectors/notion"], ["oura", "https://registry.pdpp.dev/connectors/oura"], ["pocket", "https://registry.pdpp.dev/connectors/pocket"], ["reddit", "https://registry.pdpp.dev/connectors/reddit"], ["shopify", "https://registry.pdpp.dev/connectors/shopify"], ["signal", "https://registry.pdpp.dev/connectors/signal"], ["slack", "https://registry.pdpp.dev/connectors/slack"], ["spotify", "https://registry.pdpp.dev/connectors/spotify"], ["steam", "https://registry.pdpp.dev/connectors/steam"], ["strava", "https://registry.pdpp.dev/connectors/strava"], ["twitter-archive", "https://registry.pdpp.dev/connectors/twitter-archive"], ["uber", "https://registry.pdpp.dev/connectors/uber"], ["usaa", "https://registry.pdpp.dev/connectors/usaa"], ["venmo", "https://registry.pdpp.dev/connectors/venmo"], ["whatsapp", "https://registry.pdpp.dev/connectors/whatsapp"], ["wholefoods", "https://registry.pdpp.dev/connectors/wholefoods"], ["whoop", "https://registry.pdpp.dev/connectors/whoop"], ["ynab", "https://registry.pdpp.dev/connectors/ynab"]] as [string, string][]).map(([key, sourceId]) => Object.freeze([key, sourceId] as const))); diff --git a/reference-implementation/server/index.ts b/reference-implementation/server/index.ts index 43c5ace67..87c9292e2 100644 --- a/reference-implementation/server/index.ts +++ b/reference-implementation/server/index.ts @@ -69,6 +69,7 @@ import { type BrowserSurfaceReadinessProbe, createDefaultBrowserSurfaceReadinessProbe, } from "../runtime/browser-surface-readiness.ts"; +import { REFERENCE_COLLECTION_PROFILE_SUPPORT } from "../runtime/collection-profile-support.ts"; import { type ConnectorEnvironmentPolicy, parseConnectorEnvironmentPolicy, @@ -5954,6 +5955,14 @@ export function buildAsApp(opts: ServerOpts = {}) { pdppError, requireOwnerSession: ownerAuth.requireOwnerSession, } as unknown as Parameters[1]); + // Owner-only statement of the Collection Profile this runtime implements, with what is outstanding. + app.get( + "/_ref/collection-profile", + ownerAuth.requireOwnerSession, + (_req: unknown, res: { json: (body: unknown) => unknown }) => { + res.json({ object: "collection_profile_support", ...REFERENCE_COLLECTION_PROFILE_SUPPORT }); + } + ); // Owner-only diagnostic read of a run's OBSERVATION record (Collection Profile 0.2.0 Section 5.10). mountRefRunObservations(app, { createRequestConnectorInstanceStore, diff --git a/reference-implementation/server/transport.ts b/reference-implementation/server/transport.ts index 8f8f4ec7d..eb64bb513 100644 --- a/reference-implementation/server/transport.ts +++ b/reference-implementation/server/transport.ts @@ -53,6 +53,7 @@ declare module "fastify" { } import { publicManifests, referenceManifests } from "@pdpp/reference-contract"; +import { parseManifestJsonText } from "../runtime/binding-declarations.ts"; import { applyRequestValidation, applyResponseValidation, @@ -436,13 +437,18 @@ function buildFastify({ loggerInstance }: { loggerInstance: FastifyBaseLogger }) // payloads so routes like `POST /grants/:id/revoke` (no body, header still // JSON) reach their handlers instead of failing at the transport. fastify.removeContentTypeParser("application/json"); - fastify.addContentTypeParser("application/json", { parseAs: "string" }, (_req, body, done) => { + // Routes whose body is a Collection Profile manifest also reject any repeated + // object key in the raw text, so last-wins cannot change what gets validated. + const duplicateKeyRejectingRoutes = new Set(["POST /connectors"]); + fastify.addContentTypeParser("application/json", { parseAs: "string" }, (req, body, done) => { if (!body) { done(null, {}); return; } try { - done(null, JSON.parse(typeof body === "string" ? body : body.toString())); + const text = typeof body === "string" ? body : body.toString(); + const routeKey = `${req.method} ${req.routeOptions?.url ?? ""}`; + done(null, duplicateKeyRejectingRoutes.has(routeKey) ? parseManifestJsonText(text) : JSON.parse(text)); } catch (error: unknown) { const err = error instanceof Error ? error : new Error(String(error)); Object.assign(err, { statusCode: 400 }); diff --git a/reference-implementation/test/approval-review-artifact-rendering.test.ts b/reference-implementation/test/approval-review-artifact-rendering.test.ts index d77de19c2..34b2fc982 100644 --- a/reference-implementation/test/approval-review-artifact-rendering.test.ts +++ b/reference-implementation/test/approval-review-artifact-rendering.test.ts @@ -134,7 +134,7 @@ function sourceManifest() { async function setup(): Promise<{ asUrl: string; server: TestServerHandle }> { const server = (await startServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 })) as TestServerHandle; const asUrl = `http://localhost:${server.asPort}`; - const manifest = loadSpotifyManifest(); + const manifest = { ...loadSpotifyManifest(), source_declaration: sourceManifest().source_declaration }; const registered = await fetch(`${asUrl}/connectors`, { body: JSON.stringify(manifest), headers: { Accept: "application/json", "Content-Type": "application/json" }, diff --git a/reference-implementation/test/approval-review-seam.test.ts b/reference-implementation/test/approval-review-seam.test.ts index c2400e93f..147ae2ad0 100644 --- a/reference-implementation/test/approval-review-seam.test.ts +++ b/reference-implementation/test/approval-review-seam.test.ts @@ -79,8 +79,10 @@ async function setup(backend: Backend = "sqlite", databaseUrl = POSTGRES_URL) { } else { initDb(":memory:"); } + // Register the fixture's own source from the start. Otherwise the first registration retains the reviewed + // legacy source for `spotify`, and every later registration that names SOURCE_ID is a source rebinding. const manifest = loadSpotifyManifest(); - await registerConnector(manifest); + await registerConnector({ ...manifest, source_declaration: nativeManifest().source_declaration }); await seedPreRegisteredClients([ { client_id: CLIENT_ID, @@ -603,7 +605,7 @@ test("HTTP batch final review resumes the exact result without duplicate issuanc const asUrl = `http://localhost:${server.asPort}`; try { const register = await fetch(`${asUrl}/connectors`, { - body: JSON.stringify(manifest), + body: JSON.stringify({ ...manifest, source_declaration: nativeManifest().source_declaration }), headers: { Accept: "application/json", "Content-Type": "application/json" }, method: "POST", }); @@ -817,7 +819,7 @@ test("HTTP single consent must be reviewed before approve", async () => { const asUrl = `http://localhost:${server.asPort}`; try { const register = await fetch(`${asUrl}/connectors`, { - body: JSON.stringify(manifest), + body: JSON.stringify({ ...manifest, source_declaration: nativeManifest().source_declaration }), headers: { Accept: "application/json", "Content-Type": "application/json" }, method: "POST", }); @@ -914,7 +916,7 @@ test("HTTP single final approval derives custom subject from persisted review", const asUrl = `http://localhost:${server.asPort}`; try { const register = await fetch(`${asUrl}/connectors`, { - body: JSON.stringify(manifest), + body: JSON.stringify({ ...manifest, source_declaration: nativeManifest().source_declaration }), headers: { "Content-Type": "application/json" }, method: "POST", }); diff --git a/reference-implementation/test/attention-writer.test.ts b/reference-implementation/test/attention-writer.test.ts index 9d971d394..3bb64eda6 100644 --- a/reference-implementation/test/attention-writer.test.ts +++ b/reference-implementation/test/attention-writer.test.ts @@ -496,8 +496,8 @@ test( chmodSync(stubPath, 0o755); const manifest = { + capabilities: { human_interaction: ["otp"] }, connector_id: "codex", - runtime_requirements: { bindings: { interactive: { required: false } } }, streams: [ { name: "conversations", diff --git a/reference-implementation/test/auto-enroll-eligible-schedules-integration.test.ts b/reference-implementation/test/auto-enroll-eligible-schedules-integration.test.ts index b99b158dd..5bf1841aa 100644 --- a/reference-implementation/test/auto-enroll-eligible-schedules-integration.test.ts +++ b/reference-implementation/test/auto-enroll-eligible-schedules-integration.test.ts @@ -31,6 +31,7 @@ import { canonicalConnectorKey } from "../server/connector-key.ts"; import { closeDb, initDb } from "../server/db.ts"; import { createSqliteConnectorInstanceStore } from "../server/stores/connector-instance-store.ts"; import { getDefaultSchedulerStore } from "../server/stores/scheduler-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; const OWNER_SUBJECT_ID = "owner_local"; const NOW = "2026-08-16T00:00:00.000Z"; @@ -42,34 +43,37 @@ const NOW = "2026-08-16T00:00:00.000Z"; // stored it under (matching how `canonicalConnectorKey` derives short keys // from `https://registry.pdpp.dev/connectors/` in production). async function registerConnectorWithKey(slug: string): Promise { - const manifest = { - capabilities: { - auth: { kind: "env", required: [] }, - public_listing: { tier: "supported" }, - refresh_policy: { - background_safe: true, - interaction_posture: "none", - rationale: "Synthetic manifest for the getSchedule active-connection filter contract.", - recommended_interval_seconds: 3600, - recommended_mode: "automatic", + const manifest = withFixtureSourceDeclaration( + { + capabilities: { + auth: { kind: "env", required: [] }, + public_listing: { tier: "supported" }, + refresh_policy: { + background_safe: true, + interaction_posture: "none", + rationale: "Synthetic manifest for the getSchedule active-connection filter contract.", + recommended_interval_seconds: 3600, + recommended_mode: "automatic", + }, }, + connector_id: slug, + display_name: slug, + manifest_uri: `https://registry.pdpp.dev/connectors/${slug}`, + protocol_version: "0.1.0", + runtime_requirements: {}, + streams: [ + { + name: "records", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "1.0.0", }, - connector_id: `https://registry.pdpp.dev/connectors/${slug}`, - display_name: slug, - manifest_uri: `https://registry.pdpp.dev/connectors/${slug}`, - protocol_version: "0.1.0", - runtime_requirements: {}, - streams: [ - { - name: "records", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "1.0.0", - }; + { sourceId: `https://registry.pdpp.dev/connectors/${slug}` } + ); const connectorId = await registerConnector(manifest); return canonicalConnectorKey(connectorId) ?? connectorId; } @@ -129,39 +133,42 @@ async function seedSchedule({ connectorId, connectorInstanceId }: SeedScheduleOp const TEST_ENV_KEY = "TEST_AUTO_ENROLL_TOKEN"; function testManifest(recommendedMode: "automatic" | "manual" = "automatic"): Record { - return { - capabilities: { - auth: { kind: "env", required: [TEST_ENV_KEY] }, - public_listing: { tier: "supported" }, - // Mode is DERIVED from these facts, so the synthetic manifest has to - // declare a coherent pair rather than just asserting a mode string: - // the manual variant is an interactive-login connector that has not - // declared session persistence (the Chase/USAA shape), and the - // automatic variant needs no per-run owner gesture. - refresh_policy: { - background_safe: recommendedMode === "automatic", - interaction_posture: recommendedMode === "automatic" ? "none" : "otp_likely", - rationale: "Synthetic manifest for the scheduler integration contract.", - recommended_interval_seconds: 3600, - recommended_mode: recommendedMode, + return withFixtureSourceDeclaration( + { + capabilities: { + auth: { kind: "env", required: [TEST_ENV_KEY] }, + public_listing: { tier: "supported" }, + // Mode is DERIVED from these facts, so the synthetic manifest has to + // declare a coherent pair rather than just asserting a mode string: + // the manual variant is an interactive-login connector that has not + // declared session persistence (the Chase/USAA shape), and the + // automatic variant needs no per-run owner gesture. + refresh_policy: { + background_safe: recommendedMode === "automatic", + interaction_posture: recommendedMode === "automatic" ? "none" : "otp_likely", + rationale: "Synthetic manifest for the scheduler integration contract.", + recommended_interval_seconds: 3600, + recommended_mode: recommendedMode, + }, }, + connector_id: `test-auto-enroll-${recommendedMode}`, + display_name: `Test auto-enroll (${recommendedMode})`, + manifest_uri: `https://registry.pdpp.dev/connectors/test-auto-enroll-${recommendedMode}`, + protocol_version: "0.1.0", + runtime_requirements: {}, + streams: [ + { + name: "records", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "1.0.0", }, - connector_id: `https://registry.pdpp.dev/connectors/test-auto-enroll-${recommendedMode}`, - display_name: `Test auto-enroll (${recommendedMode})`, - manifest_uri: `https://registry.pdpp.dev/connectors/test-auto-enroll-${recommendedMode}`, - protocol_version: "0.1.0", - runtime_requirements: {}, - streams: [ - { - name: "records", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "1.0.0", - }; + { sourceId: `https://registry.pdpp.dev/connectors/test-auto-enroll-${recommendedMode}` } + ); } function withTmpDb(fn: () => Promise): () => Promise { @@ -281,7 +288,7 @@ test( ); test( - "a manual-default manifest does not auto-enroll but accepts an explicit owner schedule", + "a background-unsafe manual manifest neither auto-enrolls nor accepts an enabled owner schedule", withTmpDb(async () => { const manifest = testManifest("manual"); const connectorId = manifest.connector_id as string; @@ -295,6 +302,30 @@ test( assert.equal(summary.enrolled, 0, "manual-default connectors never auto-enroll on boot"); assert.equal(await controller.getSchedule(connectorId), null); + await assert.rejects(controller.upsertSchedule(connectorId, { enabled: true, interval_seconds: 21_600 }), { + code: "invalid_request", + message: "Connector refresh policy is not background-safe; automatic scheduling is disabled.", + }); + assert.equal(await controller.getSchedule(connectorId), null, "rejected scheduling must not persist a row"); + }) +); + +test( + "a background-safe connector accepts an explicit owner schedule when default enrollment is disabled", + withTmpDb(async () => { + const manifest = testManifest(); + const connectorId = manifest.connector_id as string; + await registerConnector(manifest); + const controller = createController({}); + const summary = await autoEnrollEligibleSchedules({ + controller, + enabled: false, + env: { [TEST_ENV_KEY]: "integration-token" }, + listConnectors: buildListConnectors(manifest), + }); + assert.equal(summary.enrolled, 0, "disabled default enrollment must not create a schedule"); + assert.equal(await controller.getSchedule(connectorId), null); + const result = await controller.upsertSchedule(connectorId, { enabled: true, interval_seconds: 21_600, diff --git a/reference-implementation/test/browser-enrollment-shell-route.test.ts b/reference-implementation/test/browser-enrollment-shell-route.test.ts index e6e9ce8c4..9caccf28a 100644 --- a/reference-implementation/test/browser-enrollment-shell-route.test.ts +++ b/reference-implementation/test/browser-enrollment-shell-route.test.ts @@ -15,6 +15,7 @@ import { getDb } from "../server/db.ts"; import { startServer } from "../server/index.ts"; import { BROWSER_ENROLLMENT_SHELL_TTL_MS } from "../server/routes/ref-browser-enrollment-shell.ts"; import { createSqliteConnectorInstanceStore } from "../server/stores/connector-instance-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; // Integration coverage for the browser-enrollment shell routes: // POST /_ref/connectors/:connectorId/browser-enrollment-shell (on AS) @@ -309,12 +310,12 @@ test("browser-enrollment shell: creates draft for a browser-bound connector outs await withServer(async ({ asUrl }) => { // A connector installed from the runtime catalog: its registered manifest // declares a browser binding, and no generated key list names it. - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...(loadManifest("amazon") as Record), connector_id: "acme-shop", connector_key: "acme-shop", display_name: "Acme Shop", - }; + }); const registered = await fetch(`${asUrl}/connectors`, { body: JSON.stringify(manifest), headers: { "Content-Type": "application/json" }, diff --git a/reference-implementation/test/checkpoint-dependency-profile-conformance-postgres.test.ts b/reference-implementation/test/checkpoint-dependency-profile-conformance-postgres.test.ts index 8ef5a08fb..57247ad08 100644 --- a/reference-implementation/test/checkpoint-dependency-profile-conformance-postgres.test.ts +++ b/reference-implementation/test/checkpoint-dependency-profile-conformance-postgres.test.ts @@ -138,7 +138,7 @@ if (POSTGRES_URL) { }); }); - test("SQLite/Postgres parity scenario (Postgres side): only a DETAIL_COVERAGE_KEY_SETS connector needs a record behind a hydrated key", async () => { + test("SQLite/Postgres parity scenario (Postgres side): a DETAIL_COVERAGE_KEY_SETS-declaring connector still needs an accepted record behind a hydrated key when START omits the extension; a nondeclaring connector's hydration credits by assertion", async () => { await withScenarioServer(baseUrl, async (server) => { const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; diff --git a/reference-implementation/test/checkpoint-dependency-profile-conformance.test.ts b/reference-implementation/test/checkpoint-dependency-profile-conformance.test.ts index 4eb01108f..92eec57c5 100644 --- a/reference-implementation/test/checkpoint-dependency-profile-conformance.test.ts +++ b/reference-implementation/test/checkpoint-dependency-profile-conformance.test.ts @@ -56,6 +56,7 @@ import { runSubsetParentCoverageScenario, runUndeclaredParentScenario, } from "./helpers/checkpoint-dependency-multi-parent-scenario.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; const FAILING_STREAM_NAME_PATTERN = /sibling_b/; @@ -152,30 +153,33 @@ function fakeAdmitRunConnection( // stream-scoped-failure exception (see runtime-stream-collection-failed-commit.test.ts // for the un-cancelled counterpart of this exact shape). function twoSiblingManifest(connectorId: string) { - return { - capabilities: { human_interaction: [] }, - connector_id: connectorId, - display_name: connectorId, - manifest_uri: `https://sources.example/${connectorId}`, - protocol_version: "0.1.0", - streams: [ - { - name: "sibling_a", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - name: "sibling_b", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "0.1.0", - }; + return withFixtureSourceDeclaration( + { + capabilities: { human_interaction: [] }, + connector_id: connectorId, + display_name: connectorId, + manifest_uri: `https://sources.example/${connectorId}`, + protocol_version: "0.1.0", + streams: [ + { + name: "sibling_a", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + name: "sibling_b", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "0.1.0", + }, + { sourceId: `https://sources.example/${connectorId}` } + ); } function writeConnectorStub(tmpDir: string, script: string): string { @@ -424,7 +428,7 @@ test("a stream declaring state_stream/parent_streams without the matching covera try { // state_stream requires coverage_strategy: "checkpoint_window" on the // SAME stream (spec Validation rule 7). Here it is entirely absent. - const missingStrategyManifest = { + const missingStrategyManifest = withFixtureSourceDeclaration({ capabilities: { human_interaction: [] }, connector_id: "checkpoint-profile-coverage-strategy-missing-test", display_name: "checkpoint-profile-coverage-strategy-missing-test", @@ -439,6 +443,7 @@ test("a stream declaring state_stream/parent_streams without the matching covera semantics: "mutable_state", }, { + coverage_strategy: "checkpoint_window", name: "child", primary_key: ["id"], schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, @@ -448,7 +453,12 @@ test("a stream declaring state_stream/parent_streams without the matching covera }, ], version: "0.1.0", - }; + }); + const missingStrategyStream = missingStrategyManifest.streams.find((stream) => stream.name === "child"); + if (!missingStrategyStream) { + throw new Error("missing child fixture stream"); + } + Reflect.deleteProperty(missingStrategyStream, "coverage_strategy"); const missingStrategyResp = await fetch(`${asUrl}/connectors`, { body: JSON.stringify(missingStrategyManifest), headers: { "Content-Type": "application/json" }, @@ -464,7 +474,7 @@ test("a stream declaring state_stream/parent_streams without the matching covera // on the SAME stream. Here it is set to an unrelated valid value // ("full_inventory") instead of being absent, proving the mismatch // check — not merely an absence check. - const mismatchedStrategyManifest = { + const mismatchedStrategyManifest = withFixtureSourceDeclaration({ capabilities: { human_interaction: [] }, connector_id: "checkpoint-profile-coverage-strategy-mismatch-test", display_name: "checkpoint-profile-coverage-strategy-mismatch-test", @@ -486,7 +496,7 @@ test("a stream declaring state_stream/parent_streams without the matching covera semantics: "mutable_state", }, { - coverage_strategy: "full_inventory", + coverage_strategy: "parent_detail_accounting", name: "detail", parent_streams: ["parent_a", "parent_b"], primary_key: ["id"], @@ -496,7 +506,12 @@ test("a stream declaring state_stream/parent_streams without the matching covera }, ], version: "0.1.0", - }; + }); + const mismatchedStrategyStream = mismatchedStrategyManifest.streams.find((stream) => stream.name === "detail"); + if (!mismatchedStrategyStream) { + throw new Error("missing detail fixture stream"); + } + mismatchedStrategyStream.coverage_strategy = "full_inventory"; const mismatchedStrategyResp = await fetch(`${asUrl}/connectors`, { body: JSON.stringify(mismatchedStrategyManifest), headers: { "Content-Type": "application/json" }, @@ -509,7 +524,7 @@ test("a stream declaring state_stream/parent_streams without the matching covera ); // Positive control: the correct coverage_strategy on each shape registers successfully. - const validManifest = { + const validManifest = withFixtureSourceDeclaration({ capabilities: { human_interaction: [] }, connector_id: "checkpoint-profile-coverage-strategy-valid-test", display_name: "checkpoint-profile-coverage-strategy-valid-test", @@ -534,7 +549,7 @@ test("a stream declaring state_stream/parent_streams without the matching covera }, ], version: "0.1.0", - }; + }); const validResp = await fetch(`${asUrl}/connectors`, { body: JSON.stringify(validManifest), headers: { "Content-Type": "application/json" }, @@ -568,7 +583,7 @@ test("SQLite/Postgres parity scenario (SQLite side): one detail stream independe } }); -test("SQLite/Postgres parity scenario (SQLite side): only a DETAIL_COVERAGE_KEY_SETS connector needs a record behind a hydrated key", async () => { +test("SQLite/Postgres parity scenario (SQLite side): a DETAIL_COVERAGE_KEY_SETS-declaring connector still needs an accepted record behind a hydrated key when START omits the extension; a nondeclaring connector's hydration credits by assertion", async () => { const server = await startServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; diff --git a/reference-implementation/test/cli.test.ts b/reference-implementation/test/cli.test.ts index 3c78d2135..0e6c059ce 100644 --- a/reference-implementation/test/cli.test.ts +++ b/reference-implementation/test/cli.test.ts @@ -7181,7 +7181,8 @@ rl.on('line', (line) => { await t.test("run timeline keeps partial checkpoint commit artifacts inspectable", async () => { const manifest = { - connector_id: "https://registry.pdpp.dev/connectors/cli-run-partial-checkpoint-test", + connector_id: "cli-run-partial-checkpoint-test", + connector_key: "cli-run-partial-checkpoint-test", source_declaration: { declaration_version: "cli-run-partial-checkpoint-test-v1", display: { name: "CLI Run Partial Checkpoint Test" }, diff --git a/reference-implementation/test/collection-profile-020.test.ts b/reference-implementation/test/collection-profile-020.test.ts index d1504adb7..337ac50ed 100644 --- a/reference-implementation/test/collection-profile-020.test.ts +++ b/reference-implementation/test/collection-profile-020.test.ts @@ -34,6 +34,9 @@ const PROTOCOL_VERSION_NAMED = /protocol_version/; const OBSERVATION_NEEDS_020 = /OBSERVATION require protocol_version 0\.2\.0/; const DIAGNOSTIC_DESCRIPTORS_NEEDS_020 = /diagnostic_descriptors require protocol_version 0\.2\.0/; const UNDECLARED_STEP = /descriptor_undeclared_step/; +const BLOB_REFUSED = /BLOB/; +// A refusal before spawn carries a stable code the scheduler and console can key on. +const CAPABILITY_REFUSAL_REASON = "protocol_capability_unsupported"; const RETRYABLE_FIELD = /DONE\.error\.retryable/; // Backend under test. With the dedicated private PostgreSQL URL the runtime, the // admission store and the observation store all use Postgres. Without it the @@ -132,6 +135,7 @@ type RunOutcome = readonly ok: false; readonly message: string; readonly connectorError: { readonly message?: string } | null; + readonly failureReason: string | null; }; // A runtime rejection becomes a value, so a failing assertion names the @@ -187,7 +191,8 @@ async function runStub( return { ok: true, result: result as unknown as Record }; } catch (caught) { const connectorError = (caught as { connector_error?: { message?: string } | null }).connector_error ?? null; - return { connectorError, message: (caught as Error).message, ok: false }; + const failureReason = (caught as { failure_reason?: string }).failure_reason ?? null; + return { connectorError, failureReason, message: (caught as Error).message, ok: false }; } finally { for (const [key, value] of Object.entries(savedEnv)) { if (value === undefined) { @@ -241,6 +246,11 @@ test("declared 0.2.0 OBSERVATION facts are accepted and the run succeeds", async assert.equal(outcome.result.status, "succeeded"); const start = JSON.parse(readFileSync(stub.markerPath, "utf8")) as { protocol_capabilities: string[] }; assert.ok(start.protocol_capabilities.includes("OBSERVATION"), "START advertises the implemented message"); + assert.equal( + start.protocol_capabilities.includes("DETAIL_COVERAGE_KEY_SETS"), + false, + "the runtime must not promise key-set extension credit rules it does not implement" + ); const record = await persistedRecord("run_c020_accept"); assert.ok(record, "the run's bounded diagnostic record must be persisted"); assert.deepEqual( @@ -513,6 +523,7 @@ test("OBSERVATION or diagnostic_descriptors without protocol_version 0.2.0 are r assert.equal(outcome.ok, false, "a 0.1.0 manifest may not declare OBSERVATION"); if (!outcome.ok) { assert.match(outcome.message, OBSERVATION_NEEDS_020); + assert.equal(outcome.failureReason, CAPABILITY_REFUSAL_REASON); } assert.equal(existsMarker(stub.markerPath), false, "the child must not execute"); } finally { @@ -556,6 +567,25 @@ test("an expectation naming an undeclared step is rejected before spawn", async } }); +test("a required BLOB capability is rejected before spawn, not waived", async () => { + const stub = makeStub([{ records_emitted: 0, status: "succeeded", type: "DONE" }], 0); + try { + const outcome = await runStub( + stub, + "run_c020_blob", + manifest({ diagnostic_descriptors: undefined, protocol_capabilities: ["BLOB"], protocol_version: "0.1.0" }) + ); + assert.equal(outcome.ok, false, "BLOB is not implemented, so a connector that requires it must not start"); + if (!outcome.ok) { + assert.match(outcome.message, BLOB_REFUSED); + assert.equal(outcome.failureReason, CAPABILITY_REFUSAL_REASON); + } + assert.equal(existsMarker(stub.markerPath), false, "the child must not execute"); + } finally { + rmSync(stub.dir, { force: true, recursive: true }); + } +}); + test("a failed DONE that cites an accepted fact yields a connector-attributed diagnosis and keeps the original text", async () => { const original = "Sign-in was refused by the provider"; const stub = makeStub( diff --git a/reference-implementation/test/collection-profile-binding-declarations.test.ts b/reference-implementation/test/collection-profile-binding-declarations.test.ts new file mode 100644 index 000000000..cd3b0415c --- /dev/null +++ b/reference-implementation/test/collection-profile-binding-declarations.test.ts @@ -0,0 +1,399 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Collection Profile binding declarations, one canonical validator for every + * entry point (data-connectors a87edcdc099284e8582d24a60c9630474bd2af8b). + * + * Three entry points are covered: the registration and stored-manifest gate + * (`validateConnectorManifest`), the direct `runConnector` path before spawn, + * and raw manifest JSON text (duplicate keys). A spawn is proven absent by a + * capture file that only the spawned connector writes. + * + * What passing means: a declaration is well formed, or the runtime can supply + * its required contract. It never means the runtime enforces an authority. + * Section 3.3.10 keeps every channel's coverage separate from this check. + */ + +import assert from "node:assert/strict"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test, { after, before } from "node:test"; + +import { + BindingDeclarationError, + BindingPlacementError, + normalizeBindingDeclarations, + parseManifestJsonText, + startBindingsFor, +} from "../runtime/binding-declarations.ts"; +import { type RuntimeRunConnectorOptions, runConnector } from "../runtime/index.ts"; +import { validateConnectorManifest } from "../server/connector-manifest-validation.ts"; +import { closeDb, initDb } from "../server/db.ts"; + +const MINIMAL_MANIFEST = { + connector_id: "test", + display_name: "Test Connector", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + semantics: "append_only", + }, + ], + version: "1.0.0", +}; + +function withBindings(bindings: unknown): Record { + return { ...MINIMAL_MANIFEST, runtime_requirements: { bindings } }; +} + +// Assertion patterns, hoisted to module scope. +const REQUIRED_FLAG = /required/; +const INTERACTIVE_INSTANCE = /interactive/; +const SHORTHAND_KIND_FORBIDDEN = /browser must NOT be valid/; +const INSTANCE_KIND = /kind/; +const UNKNOWN_SHORTHAND = /scanner/; +const BINDINGS_OBJECT = /bindings/; +const EXTERNAL_TOOLS = /external_tools/; +const INTERFACE_SHAPE = /interface/; +const CANONICAL_NAVIGATION_HOST = /navigate\.0 must match pattern/; +const RATIONALE_REQUIRED = /rationale/; + +// runConnector reads detail-gap state from the DB before spawn, so it needs one. +before(() => { + initDb(":memory:"); +}); + +after(() => { + closeDb(); +}); + +// --------------------------------------------------------------------------- +// 1. Registration and stored-manifest gate: validateConnectorManifest +// --------------------------------------------------------------------------- + +test("registration gate: a required binding must state a boolean required flag", () => { + // Canonical 3.3: `required` is REQUIRED. Old gate accepted an absent flag. + assert.throws(() => validateConnectorManifest(withBindings({ browser: {} })), REQUIRED_FLAG); +}); + +test("registration gate: `interactive` is removed as a binding and is not accepted", () => { + // Canonical 3.3 removed-name table. Old gate accepted it as a registry name. + assert.throws( + () => validateConnectorManifest(withBindings({ interactive: { required: true } })), + INTERACTIVE_INSTANCE + ); + assert.throws( + () => validateConnectorManifest(withBindings({ interactive: { required: false } })), + INTERACTIVE_INSTANCE + ); +}); + +test("registration gate: a shorthand key must not carry kind; a named instance must", () => { + assert.throws( + () => validateConnectorManifest(withBindings({ browser: { kind: "network", required: false } })), + SHORTHAND_KIND_FORBIDDEN + ); + assert.throws(() => validateConnectorManifest(withBindings({ chase_site: { required: true } })), INSTANCE_KIND); +}); + +test("registration gate: a named instance with an explicit kind is valid", () => { + assert.doesNotThrow(() => + validateConnectorManifest(withBindings({ chase_site: { kind: "browser", required: true } })) + ); +}); + +test("registration gate: an unqualified unknown optional shorthand is rejected", () => { + assert.throws(() => validateConnectorManifest(withBindings({ scanner: { required: false } })), UNKNOWN_SHORTHAND); +}); + +test("registration gate: a canonical namespaced extension is shape-valid", () => { + // Shape-valid only. Placement, not validation, refuses it when required. + assert.doesNotThrow(() => validateConnectorManifest(withBindings({ "example.com/scanner": { required: true } }))); +}); + +test("registration gate: null bindings is rejected, since the canonical map is an object", () => { + // Canonical schema: `bindings` is type object. Old gate treated null as absent. + assert.throws(() => validateConnectorManifest(withBindings(null)), BINDINGS_OBJECT); +}); + +test("registration gate: external_tools is validated even when bindings is absent", () => { + // Old gate returned early on absent bindings and skipped external_tools. + const manifest = { ...MINIMAL_MANIFEST, runtime_requirements: { external_tools: "not-an-array" } }; + assert.throws(() => validateConnectorManifest(manifest), EXTERNAL_TOOLS); +}); + +test("registration gate: interface must match the resolved kind, with no leading zero", () => { + assert.throws( + () => + validateConnectorManifest( + withBindings({ chase_site: { interface: "browser@01", kind: "browser", required: true } }) + ), + INTERFACE_SHAPE + ); + assert.throws( + () => + validateConnectorManifest( + withBindings({ chase_site: { interface: "network@1", kind: "browser", required: true } }) + ), + INTERFACE_SHAPE + ); + assert.doesNotThrow(() => + validateConnectorManifest(withBindings({ chase_site: { interface: "browser@2", kind: "browser", required: true } })) + ); +}); + +test("registration gate: a web host constraint must be in canonical form", () => { + assert.throws( + () => + validateConnectorManifest( + withBindings({ browser: { constraints: { navigate: ["https://Secure.Chase.com"] }, required: true } }) + ), + CANONICAL_NAVIGATION_HOST + ); +}); + +test("registration gate: a desktop_session item is an exceptional grant and needs a rationale", () => { + const item = { operations: ["unwrap"], selector: "signal-safestorage", service: "os_keyring" }; + assert.throws( + () => + validateConnectorManifest( + withBindings({ signal_keyring: { constraints: { items: [item] }, kind: "desktop_session", required: true } }) + ), + RATIONALE_REQUIRED + ); + assert.doesNotThrow(() => + validateConnectorManifest( + withBindings({ + signal_keyring: { + constraints: { items: [item] }, + kind: "desktop_session", + rationale: "Keyring unwraps the local database key.", + required: true, + }, + }) + ) + ); +}); + +test("registration gate: a valid shorthand manifest keeps its meaning (baseline)", () => { + assert.doesNotThrow(() => validateConnectorManifest(withBindings({ network: { required: true } }))); + assert.doesNotThrow(() => validateConnectorManifest(withBindings({ filesystem: { required: false } }))); +}); + +// --------------------------------------------------------------------------- +// 2. Direct runConnector before spawn: the same validator, no child process +// --------------------------------------------------------------------------- + +function fakeAdmitRunConnection(): NonNullable { + return ({ connectorId, connectorInstanceId, ownerSubjectId }) => { + const owner = ownerSubjectId || "owner_binding_test"; + const exactId = connectorInstanceId ?? `cin_${owner}_${connectorId.replace(/[^a-z0-9]+/gi, "_")}`; + return Promise.resolve({ connectorId, connectorInstanceId: exactId, ownerSubjectId: owner }); + }; +} + +/** A connector that writes the START it receives to `capturePath`, then succeeds. */ +function startCaptureConnector(capturePath: string): { cleanup: () => void; connectorPath: string } { + const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-binding-start-capture-")); + const connectorPath = join(tmpDir, "connector.mjs"); + writeFileSync( + connectorPath, + ` +import { createInterface } from 'readline'; +import { writeFileSync } from 'node:fs'; +writeFileSync(${JSON.stringify(`${capturePath}.spawned`)}, 'spawned'); +const rl = createInterface({ input: process.stdin, terminal: false }); +rl.on('line', (line) => { + const msg = JSON.parse(line); + if (msg.type === 'START') { + writeFileSync(${JSON.stringify(capturePath)}, JSON.stringify(msg, null, 2)); + process.stdout.write(JSON.stringify({ type: 'DONE', status: 'succeeded', records_emitted: 0 }) + '\\n'); + rl.close(); + process.exit(0); + } +}); +`, + "utf-8" + ); + return { cleanup: () => rmSync(tmpDir, { force: true, recursive: true }), connectorPath }; +} + +interface DirectRunResult { + error: Error | null; + spawned: boolean; + start: Record | null; +} + +async function runWithBindings(bindings: unknown): Promise { + const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-binding-direct-")); + const capturePath = join(tmpDir, "start.json"); + const { cleanup, connectorPath } = startCaptureConnector(capturePath); + let error: Error | null = null; + try { + await runConnector({ + admitRunConnection: fakeAdmitRunConnection(), + collectionMode: "full_refresh", + connectorId: "test", + connectorPath, + manifest: withBindings(bindings), + onInteraction: async () => ({ data: {}, request_id: "unused", status: "success", type: "INTERACTION_RESPONSE" }), + ownerToken: "test", + persistState: true, + rsUrl: "http://localhost:9999", + state: null, + } as unknown as RuntimeRunConnectorOptions); + } catch (caught) { + error = caught instanceof Error ? caught : new Error(String(caught)); + } + // The launch marker is independent of START; a failure before START cannot hide a spawn. + const spawned = existsSync(`${capturePath}.spawned`); + const start = existsSync(capturePath) + ? (JSON.parse(readFileSync(capturePath, "utf8")) as Record) + : null; + cleanup(); + rmSync(tmpDir, { force: true, recursive: true }); + return { error, spawned, start }; +} + +test("direct run: a malformed binding (missing required) is refused before spawn", async () => { + const result = await runWithBindings({ browser: {} }); + assert.ok(result.error instanceof BindingDeclarationError); + assert.match(result.error.message, REQUIRED_FLAG); + assert.equal(result.spawned, false); +}); + +test("direct run: `interactive` is refused before spawn", async () => { + const result = await runWithBindings({ interactive: { required: true } }); + assert.ok(result.error instanceof BindingDeclarationError); + assert.match(result.error.message, INTERACTIVE_INSTANCE); + assert.equal(result.spawned, false); +}); + +test("direct run: a required unsupported feature is refused before spawn", async () => { + // Old direct path ignored features and spawned the connector. + const result = await runWithBindings({ browser: { features: ["page_navigation"], required: true } }); + assert.ok(result.error instanceof BindingPlacementError); + assert.deepEqual(result.error.bindingKeys, ["browser"]); + assert.ok(result.error.message.includes("features page_navigation are not provided")); + assert.equal(result.spawned, false); +}); + +test("direct run: a required unsupported constraint is refused before spawn", async () => { + const result = await runWithBindings({ + filesystem: { + constraints: { outputs: [{ access: "write", slot: "statements" }] }, + required: true, + }, + }); + assert.ok(result.error instanceof BindingPlacementError); + assert.deepEqual(result.error.bindingKeys, ["filesystem"]); + assert.ok(result.error.message.includes("does not enforce them")); + assert.equal(result.spawned, false); +}); + +test("direct run: a required unsupported interface version is refused before spawn", async () => { + const result = await runWithBindings({ chase_site: { interface: "browser@2", kind: "browser", required: true } }); + assert.ok(result.error instanceof BindingPlacementError); + assert.deepEqual(result.error.bindingKeys, ["chase_site"]); + assert.ok(result.error.message.includes("interface browser@2 is not implemented")); + assert.equal(result.spawned, false); +}); + +test("direct run: a required namespaced extension is refused before spawn", async () => { + const result = await runWithBindings({ "example.com/scanner": { required: true } }); + assert.ok(result.error instanceof BindingPlacementError); + assert.deepEqual(result.error.bindingKeys, ["example.com/scanner"]); + assert.ok(result.error.message.includes("kind example.com/scanner is not implemented")); + assert.equal(result.spawned, false); +}); + +test("direct run: an optional unsupported extension is omitted and START does not advertise it", async () => { + const result = await runWithBindings({ "example.com/scanner": { required: false } }); + assert.equal(result.error, null); + assert.equal(result.spawned, true); + assert.equal(Object.hasOwn(result.start?.bindings as object, "example.com/scanner"), false); +}); + +test("direct run: a required named instance is refused until the runtime supplies per-instance handles", async () => { + const result = await runWithBindings({ chase_site: { kind: "browser", required: true } }); + assert.ok(result.error instanceof BindingPlacementError); + assert.deepEqual(result.error.bindingKeys, ["chase_site"]); + assert.ok(result.error.message.includes("per-instance handles")); + assert.equal(result.spawned, false); +}); + +test("START encoding keeps resolved instance keys and omits a same-kind instance separately", () => { + // Encoder-only test with a supplied plan. This is not evidence that native + // execution can resolve these handles; the direct-run refusal above proves + // that this runtime currently cannot. + const instances = normalizeBindingDeclarations({ + optional_page: { features: ["page_navigation"], kind: "browser", required: false }, + provider_page: { kind: "browser", required: true }, + }); + const placed = instances.filter((instance) => instance.key === "provider_page"); + const optional = instances.find((instance) => instance.key === "optional_page"); + assert.ok(optional); + const bindings = startBindingsFor(instances, { + omitted: [{ instance: optional, reason: "unsupported feature" }], + placed, + }); + assert.deepEqual(bindings.provider_page, {}); + assert.equal(Object.hasOwn(bindings, "optional_page"), false); + assert.equal(Object.hasOwn(bindings, "browser"), false); +}); + +test("direct run: an omitted optional browser with unsupported features is not advertised in START", async () => { + const result = await runWithBindings({ browser: { features: ["page_navigation"], required: false } }); + assert.equal(result.error, null); + assert.equal(result.spawned, true); + assert.equal(Object.hasOwn(result.start?.bindings as object, "browser"), false); +}); + +test("direct run: a valid shorthand baseline spawns and advertises the static kinds", async () => { + const result = await runWithBindings({ network: { required: true } }); + assert.equal(result.error, null); + assert.equal(result.spawned, true); + const bindings = result.start?.bindings as Record; + assert.ok(Object.hasOwn(bindings, "network")); + assert.ok(Object.hasOwn(bindings, "browser")); + assert.ok(Object.hasOwn(bindings, "filesystem")); + assert.equal(Object.hasOwn(bindings, "interactive"), false); +}); + +// --------------------------------------------------------------------------- +// 3. Raw manifest JSON text: duplicate keys are rejected before last-wins +// --------------------------------------------------------------------------- + +test("raw JSON: a repeated instance key is rejected, not last-wins", () => { + const text = '{"runtime_requirements":{"bindings":{"browser":{"required":true},"browser":{"required":false}}}}'; + assert.throws( + () => parseManifestJsonText(text), + (error: unknown) => (error as { code?: string }).code === "duplicate_json_key" + ); +}); + +test("raw JSON: an escaped spelling of the same key is a duplicate", () => { + const text = '{"bindings":{"browser":{"required":true},"\\u0062rowser":{"required":false}}}'; + assert.throws( + () => parseManifestJsonText(text), + (error: unknown) => (error as { code?: string }).code === "duplicate_json_key" + ); +}); + +test("raw JSON: a repeated nested member is rejected anywhere in the text", () => { + const text = '{"runtime_requirements":{"bindings":{"network":{"required":true,"required":false}}}}'; + assert.throws( + () => parseManifestJsonText(text), + (error: unknown) => (error as { code?: string }).code === "duplicate_json_key" + ); +}); + +test("raw JSON: a text without duplicate keys parses to its value (baseline)", () => { + const parsed = parseManifestJsonText('{"runtime_requirements":{"bindings":{"network":{"required":true}}}}'); + assert.deepEqual(parsed, { runtime_requirements: { bindings: { network: { required: true } } } }); +}); diff --git a/reference-implementation/test/collection-profile-owner-journey.test.ts b/reference-implementation/test/collection-profile-owner-journey.test.ts index cfc215e6f..05973121c 100644 --- a/reference-implementation/test/collection-profile-owner-journey.test.ts +++ b/reference-implementation/test/collection-profile-owner-journey.test.ts @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 // Real HTTP/session + durable store boundary, beyond the route's fake adapter. -// Collection Profile 5.12: keep diagnostic records on the owner surface, -// uncached and namespace-scoped. +// Collection Profile 3.4 and 5.12: advertise support before any run, and keep +// diagnostic records on the owner surface, uncached and namespace-scoped. import assert from "node:assert/strict"; import test from "node:test"; import { closeDb, getDb } from "../server/db.ts"; @@ -24,7 +24,7 @@ import { dedicatedPostgresTestUrl } from "./helpers/dedicated-postgres-test-url. const POSTGRES_URL = dedicatedPostgresTestUrl(process.env.PDPP_TEST_POSTGRES_URL); const OWNER_PASSWORD = "collection-profile-owner-fixture-password"; -test("owner can read diagnostics through the real session gate", async () => { +test("owner can discover profile support and read diagnostics through the real session gate", async () => { if (POSTGRES_URL) { await initPostgresStorage({ backend: "postgres", databaseUrl: POSTGRES_URL }); } @@ -54,6 +54,20 @@ test("owner can read diagnostics through the real session gate", async () => { assert.ok(cookie, "runtime-owned sign-in must issue a real owner session"); const headers = { Accept: "application/json", Cookie: cookie }; + const supportResponse = await fetch(`${origin}/_ref/collection-profile`, { headers }); + assert.equal(supportResponse.status, 200); + assert.deepEqual( + await supportResponse.json(), + { + compatible_protocol_versions: ["0.1.0", "0.2.0"], + conformance: { outstanding: ["binding_authority"], status: "partial" }, + object: "collection_profile_support", + protocol_capabilities: ["OBSERVATION", "STREAM_EVIDENCE"], + protocol_version: "0.2.0", + }, + "profile support is discoverable before a connector starts" + ); + const connectorId = "owner-observations-fixture"; if (POSTGRES_URL) { await postgresQuery("INSERT INTO connectors(connector_id, manifest) VALUES ($1, '{}'::jsonb)", [connectorId]); diff --git a/reference-implementation/test/collection-profile.test.ts b/reference-implementation/test/collection-profile.test.ts index 7437b7f49..5035cb897 100644 --- a/reference-implementation/test/collection-profile.test.ts +++ b/reference-implementation/test/collection-profile.test.ts @@ -26,7 +26,6 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; - import { RECOVERY_ACTIONS } from "../runtime/connector-gap-bounding.ts"; import { cursorAuditFact } from "../runtime/cursor-audit-fact.ts"; import { @@ -44,6 +43,7 @@ import { createSqliteConnectorInstanceStore, } from "../server/stores/connector-instance-store.ts"; import { TEST_PRE_REGISTERED_PUBLIC_CLIENTS } from "./fixtures/demo-clients.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; type TestServer = Awaited>; @@ -712,6 +712,13 @@ const MINIMAL_MANIFEST = { version: "1.0.0", }; +// Section 3.4: an INTERACTION.kind must be declared in capabilities.human_interaction. +// Tests that emit INTERACTION use this manifest. Undeclared-kind refusal uses MINIMAL_MANIFEST. +const HUMAN_INTERACTION_MANIFEST = { + ...MINIMAL_MANIFEST, + capabilities: { human_interaction: ["credentials", "otp", "manual_action"] }, +}; + function buildMultiStreamManifest(connectorId = "test-multi-stream") { return { ...MINIMAL_MANIFEST, @@ -809,7 +816,6 @@ test("Collection Profile conformance", async (t) => { assert.deepEqual(captured.bindings, { browser: {}, filesystem: {}, - interactive: {}, network: {}, }); assert.ok(!("config" in captured), "legacy START.config should be absent"); @@ -2803,11 +2809,17 @@ rl.on('line', (line) => { assert.equal(advancedEvents.length, 2); assert.deepEqual( new Set(stagedEvents.map((event) => eventCursorKey(event))), - new Set([cursorEventKey("items", "cursor_items_success"), cursorEventKey("other_items", "cursor_other_success")]) + new Set([ + cursorEventKey("items", "cursor_items_success"), + cursorEventKey("other_items", "cursor_other_success"), + ]) ); assert.deepEqual( new Set(advancedEvents.map((event) => eventCursorKey(event))), - new Set([cursorEventKey("items", "cursor_items_success"), cursorEventKey("other_items", "cursor_other_success")]) + new Set([ + cursorEventKey("items", "cursor_items_success"), + cursorEventKey("other_items", "cursor_other_success"), + ]) ); assert.deepEqual( advancedEvents.map((event) => event.data.state_streams_committed as number).sort((a, b) => a - b), @@ -2943,6 +2955,8 @@ rl.on('line', (line) => { required: ["id"], type: "object", }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", }, { name: "other_items", @@ -2955,6 +2969,8 @@ rl.on('line', (line) => { required: ["id"], type: "object", }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", }, ], version: "0.1.0", @@ -3583,55 +3599,53 @@ rl.on('line', (line) => { } }); - await t.test("runtime satisfies required interactive bindings and advertises them in START", async () => { - const manifestWithInteractiveBinding = { - ...MINIMAL_MANIFEST, - runtime_requirements: { - bindings: { - interactive: { required: true }, - }, - }, - }; + await t.test( + "START.bindings never advertises interactive, even when the manifest declares human_interaction", + async () => { + const manifestWithInteractiveBinding = { + ...MINIMAL_MANIFEST, + capabilities: { human_interaction: ["credentials"] }, + }; - const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-interactive-binding-capture-")); - const capturePath = join(tmpDir, "start.json"); - const { connectorPath, cleanup } = createStartCaptureConnector(capturePath); + const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-interactive-binding-capture-")); + const capturePath = join(tmpDir, "start.json"); + const { connectorPath, cleanup } = createStartCaptureConnector(capturePath); - try { - const result = await runTestConnector({ - // Bare runtime unit test: no server, no registered connector, an - // unreachable rsUrl. A pure-echo admission just needs to clear the - // gate so the START.bindings capture under test is reached. - admitRunConnection: fakeEchoAdmitRunConnection(), - collectionMode: "full_refresh", - connectorId: "test", - connectorPath, - manifest: manifestWithInteractiveBinding, - onInteraction: async () => ({ - data: {}, - request_id: "unused", - status: "success", - type: "INTERACTION_RESPONSE", - }), - ownerToken: "test", - persistState: true, - rsUrl: "http://localhost:9999", - state: null, - }); + try { + const result = await runTestConnector({ + // Bare runtime unit test: no server, no registered connector, an + // unreachable rsUrl. A pure-echo admission just needs to clear the + // gate so the START.bindings capture under test is reached. + admitRunConnection: fakeEchoAdmitRunConnection(), + collectionMode: "full_refresh", + connectorId: "test", + connectorPath, + manifest: manifestWithInteractiveBinding, + onInteraction: async () => ({ + data: {}, + request_id: "unused", + status: "success", + type: "INTERACTION_RESPONSE", + }), + ownerToken: "test", + persistState: true, + rsUrl: "http://localhost:9999", + state: null, + }); - assert.equal(result.status, "succeeded"); - const captured = JSON.parse(readFileSync(capturePath, "utf8")); - assert.deepEqual(captured.bindings, { - browser: {}, - filesystem: {}, - interactive: {}, - network: {}, - }); - } finally { - cleanup(); - rmSync(tmpDir, { force: true, recursive: true }); + assert.equal(result.status, "succeeded"); + const captured = JSON.parse(readFileSync(capturePath, "utf8")); + assert.deepEqual(captured.bindings, { + browser: {}, + filesystem: {}, + network: {}, + }); + } finally { + cleanup(); + rmSync(tmpDir, { force: true, recursive: true }); + } } - }); + ); await t.test( "runtime omits interactive from START.bindings when interaction handling is explicitly disabled", @@ -5631,7 +5645,7 @@ rl.on('line', (line) => { await t.test("runtime reports manual-action known gaps without persisting interaction responses", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-manual-gap-")); @@ -5673,7 +5687,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => ({ data: { otp: "123456", password: "supersecret" }, request_id: "manual_login", @@ -6678,7 +6692,7 @@ rl.on('line', (line) => { await t.test("INTERACTION round-trip allows connector to continue collecting", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; // Connector that emits INTERACTION, waits for response, then emits RECORD + DONE @@ -6713,7 +6727,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, // biome-ignore lint/suspicious/useAwait: localized test assertion preserves its explicit contract. onInteraction: async (message: unknown) => { const msg = message as InteractionMessage; @@ -6803,7 +6817,7 @@ rl.on('line', (line) => { await t.test("resolved OTP assistance completes a pending interaction after page advancement", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const tmpDir = mkdtempSync(join(process.cwd(), ".scratch-otp-manual-")); const connectorPath = join(tmpDir, "connector.mjs"); writeFileSync( @@ -6834,7 +6848,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => new Promise(() => { // Keep the interaction pending until the connector advances the browser page. @@ -6854,7 +6868,7 @@ rl.on('line', (line) => { await t.test("late OTP assistance status is ignored after a console response", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-otp-late-status-")); const connectorPath = join(tmpDir, "connector.mjs"); writeFileSync( @@ -6884,7 +6898,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async (message: unknown) => ({ data: { code: "123456" }, request_id: (message as InteractionMessage).request_id, @@ -6906,7 +6920,7 @@ rl.on('line', (line) => { await t.test("browser-surface-backed otp INTERACTION projects streamable assistance with secret input", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-interaction-browser-otp-")); @@ -6951,7 +6965,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, // biome-ignore lint/suspicious/useAwait: localized test assertion preserves its explicit contract. onInteraction: async (message: unknown) => { const msg = message as InteractionMessage; @@ -7003,7 +7017,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, // biome-ignore lint/suspicious/useAwait: localized test assertion preserves its explicit contract. onInteraction: async (message: unknown) => { const msg = message as InteractionMessage; @@ -7434,7 +7448,7 @@ rl.on('line', (line) => { await t.test("runtime rejects connector output emitted while waiting for INTERACTION_RESPONSE", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-blocked-output-")); @@ -7481,7 +7495,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => new Promise(() => undefined), ownerToken, persistState: true, @@ -7533,7 +7547,7 @@ rl.on('line', (line) => { async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-blocked-state-")); @@ -7578,7 +7592,7 @@ rl.on('line', (line) => { collectionMode: "incremental", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => new Promise(() => undefined), ownerToken, persistState: true, @@ -7633,7 +7647,7 @@ rl.on('line', (line) => { async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-blocked-progress-")); @@ -7678,7 +7692,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => new Promise(() => undefined), ownerToken, persistState: true, @@ -7729,7 +7743,7 @@ rl.on('line', (line) => { async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-blocked-skip-result-")); @@ -7775,7 +7789,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => new Promise(() => undefined), ownerToken, persistState: true, @@ -7824,7 +7838,7 @@ rl.on('line', (line) => { await t.test("runtime rejects a second INTERACTION emitted while waiting for INTERACTION_RESPONSE", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-overlap-")); @@ -7872,7 +7886,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => new Promise(() => undefined), ownerToken, persistState: true, @@ -7925,7 +7939,7 @@ rl.on('line', (line) => { async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-blocked-done-")); @@ -7970,7 +7984,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => new Promise(() => undefined), ownerToken, persistState: true, @@ -8023,7 +8037,7 @@ rl.on('line', (line) => { async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-blocked-invalid-jsonl-")); @@ -8064,7 +8078,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => new Promise(() => undefined), ownerToken, persistState: true, @@ -8116,7 +8130,7 @@ rl.on('line', (line) => { await t.test("runtime returns INTERACTION timeout responses when the handler does not answer in time", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-timeout-")); @@ -8163,7 +8177,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, // biome-ignore lint/suspicious/useAwait: localized test assertion preserves its explicit contract. onInteraction: async (message: unknown) => { const msg = message as InteractionMessage; @@ -8202,7 +8216,7 @@ rl.on('line', (line) => { await t.test("runtime returns INTERACTION cancelled responses when the handler aborts", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-cancelled-")); @@ -8249,7 +8263,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, // biome-ignore lint/suspicious/useAwait: localized test assertion preserves its explicit contract. onInteraction: async (message: unknown) => { const msg = message as InteractionMessage; @@ -8290,7 +8304,7 @@ rl.on('line', (line) => { async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-invalid-response-")); @@ -8328,7 +8342,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, // biome-ignore lint/suspicious/useAwait: localized test assertion preserves its explicit contract. onInteraction: async (message: unknown) => { const msg = message as InteractionMessage; @@ -8690,7 +8704,7 @@ rl.on('line', (line) => { }); await t.test( - "runtime rejects INTERACTION when START.bindings omit interactive and records no interaction artifacts", + "runtime rejects INTERACTION whose kind the manifest does not declare and records no interaction artifacts", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; @@ -8742,7 +8756,10 @@ rl.on('line', (line) => { rejected = asRuntimeError(err); assert.equal(rejected.failure_reason, "connector_protocol_violation"); // biome-ignore lint/performance/useTopLevelRegex: localized test assertion preserves its explicit contract. - assert.match(rejected.message, /START\.bindings omitted interactive/); + assert.match( + rejected.message, + /INTERACTION\.kind not declared in capabilities\.human_interaction: credentials/ + ); assert.equal(checkpointSummaryOf(rejected).records_flushed, 0); assert.equal(checkpointSummaryOf(rejected).buffered_records_dropped, 0); assert.equal(checkpointSummaryOf(rejected).state_streams_staged, 0); @@ -8762,7 +8779,7 @@ rl.on('line', (line) => { const failedEvent = (runTimeline.data || []).find((event) => event.event_type === "run.failed"); assert.ok( failedEvent, - "run.failed should be emitted when INTERACTION arrives without an advertised interactive binding" + "run.failed should be emitted when INTERACTION names a kind the manifest does not declare" ); assert.equal(failedEvent.data.reason, "connector_protocol_violation"); assert.equal(failedEvent.data.records_flushed, 0); @@ -8912,7 +8929,10 @@ rl.on('line', (line) => { assert.equal(advancedEvents.length, 0); assert.deepEqual( new Set(stagedEvents.map((event) => eventCursorKey(event))), - new Set([cursorEventKey("items", "cursor_items_done_failed"), cursorEventKey("other_items", "cursor_other_done_failed")]) + new Set([ + cursorEventKey("items", "cursor_items_done_failed"), + cursorEventKey("other_items", "cursor_other_done_failed"), + ]) ); const failedEvent = (runTimeline.data || []).find((event) => event.event_type === "run.failed"); @@ -11191,25 +11211,26 @@ function testSourceIdForConnector(connectorId: string): string { } function withTestSourceDeclaration(manifest: ConnectorManifest): ConnectorManifest { - const streams = asArray(manifest.streams).map((stream) => { - const streamRecord = asRecord(stream); - return { - ...streamRecord, - selection: { - fields: true, - resources: true, + const declared = asRecord( + withFixtureSourceDeclaration( + { + ...manifest, + streams: asArray(manifest.streams).map((stream) => ({ + ...asRecord(stream), + selection: { fields: true, resources: true }, + })), }, - }; - }); + { + sourceId: testSourceIdForConnector(manifest.connector_id), + } + ) + ); return { ...manifest, source_declaration: { + ...asRecord(declared.source_declaration), declaration_version: `collection-profile-test:${manifest.connector_id}:${manifest.version}`, - display: { name: manifest.display_name }, - protocol_version: manifest.protocol_version, publisher: { id: "https://sources.example/publishers/collection-profile-tests" }, - source: { id: testSourceIdForConnector(manifest.connector_id), kind: "connector" }, - streams, }, }; } diff --git a/reference-implementation/test/composed-origin.test.ts b/reference-implementation/test/composed-origin.test.ts index 85a925f1a..27a2c4694 100644 --- a/reference-implementation/test/composed-origin.test.ts +++ b/reference-implementation/test/composed-origin.test.ts @@ -506,6 +506,7 @@ const COMPOSED_INGEST_STREAMS = [ name: "items", primary_key: ["id"], schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, semantics: "append_only", }, ]; @@ -639,7 +640,9 @@ test("composed browser origin carries metadata, owner session, console, device f await ensureConsoleBuild(); const webPort = await allocatePort(); const webOrigin = `http://127.0.0.1:${webPort}`; - const spotifyManifest = JSON.parse(await readFile(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8")); + const spotifyManifest = JSON.parse( + await readFile(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") + ); const server = await startServer({ asPort: 0, diff --git a/reference-implementation/test/connector-install-registry-integration.test.ts b/reference-implementation/test/connector-install-registry-integration.test.ts new file mode 100644 index 000000000..aeba5604c --- /dev/null +++ b/reference-implementation/test/connector-install-registry-integration.test.ts @@ -0,0 +1,216 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Install service to registry journey: the verified source declaration is published to the real registry + * (SQLite, and PostgreSQL 18 when PDPP_TEST_POSTGRES_URL is set) through the production registration + * wiring, and survives reuse and a restart with a new store and service instance. + * + * The OCI artifact seam is a trusted verified-layout fixture (`installArtifact`); signatures are not exercised. + * The install record's sidecar digest is the durable binding this journey checks. + */ + +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { getConnectorManifest } from "../server/auth.ts"; +import { + type ConnectorCatalogEntry, + createConnectorInstallService, + createFileConnectorInstallStore, + inspectActiveConnector, +} from "../server/connector-install/index.ts"; +import { installedConnectorManifestRegistration } from "../server/connector-install/manifest-registration.ts"; +import { closeDb, initDb } from "../server/db.ts"; +import { closePostgresStorage, initPostgresStorage } from "../server/postgres-storage.ts"; +import { snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest } from "../server/source-declaration-legacy-collection.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; +import { readCollectionProfileFixture } from "./helpers/installed-collection-profiles.ts"; +import { withTemporaryPostgresDatabase } from "./helpers/postgres-temp-database.ts"; + +const SLUG = "fixture-registry-journey"; +const CONFIG_DIGEST = `sha256:${"d".repeat(64)}`; +const OTHER_SOURCE = "https://sources.example/other/registry-journey"; +const ALREADY_BOUND = /already bound to source/; + +function profileFor(version: string): Record { + return { + ...readCollectionProfileFixture("github"), + connector_id: SLUG, + connector_key: SLUG, + manifest_uri: `https://registry.pdpp.dev/connectors/${SLUG}`, + version, + }; +} + +function digestOf(bytes: string): string { + return `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +} + +interface Version { + readonly entry: ConnectorCatalogEntry; + readonly profileBytes: string; + readonly sidecar: string; +} + +/** A version whose sidecar names `sourceId` as its Core source; the profile itself embeds nothing. */ +function versionWith(version: string, sourceId?: string): Version { + const profile = profileFor(version); + const declaration = + sourceId === undefined + ? withFixtureSourceDeclaration(profile).source_declaration + : snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest(profile, { + connectorImplementationId: `https://implementations.example/connectors/${SLUG}`, + publisherId: "https://publishers.example/registry-journey", + sourceId, + }); + const profileBytes = JSON.stringify(profile, null, 2); + return { + entry: { + config_digest: CONFIG_DIGEST, + connector_id: SLUG, + connector_key: SLUG, + digest: digestOf(profileBytes), + tier: "supported", + version, + }, + profileBytes, + sidecar: JSON.stringify(declaration, null, 2), + }; +} + +function writeLayout(root: string, version: Version): void { + mkdirSync(join(root, "profile"), { recursive: true }); + mkdirSync(join(root, "dist"), { recursive: true }); + writeFileSync(join(root, "profile", "collection-profile.json"), version.profileBytes); + writeFileSync(join(root, "dist", "collection-profile.mjs"), "export {};\n"); + writeFileSync(join(root, "provenance.json"), "{}\n"); + writeFileSync(join(root, "source-declaration.json"), version.sidecar); +} + +function retainedSourceId(stored: Record | null): string | undefined { + const declaration = stored?.source_declaration as { source?: { id?: unknown } } | undefined; + return typeof declaration?.source?.id === "string" ? declaration.source.id : undefined; +} + +/** The production registration wiring, optionally failing once on a chosen call to exercise rollback. */ +function productionService(dataDir: string, versions: readonly Version[], failOnCall?: number) { + const store = createFileConnectorInstallStore(dataDir); + let calls = 0; + const service = createConnectorInstallService({ + catalogLoader: async () => versions.map((version) => version.entry), + dataDir, + installArtifact: (root, entry) => { + const version = versions.find((candidate) => candidate.entry.digest === entry.digest); + if (!version) { + throw new Error("no fixture version for the requested digest"); + } + writeLayout(root, version); + }, + registerManifest: async (manifest) => { + calls += 1; + if (failOnCall === calls) { + throw new Error("registration refused (test)"); + } + return await installedConnectorManifestRegistration.registerManifest(manifest); + }, + repairManifestDerivedState: installedConnectorManifestRegistration.repairManifestDerivedState, + store, + }); + return { service, store }; +} + +async function journey(): Promise { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-install-registry-")); + try { + const v1 = versionWith("1.0.0"); + const v2 = versionWith("1.1.0"); + const first = productionService(dataDir, [v1, v2]); + const installed = await first.service.install(SLUG, v1.entry.digest); + const sourceAfterInstall = retainedSourceId((await getConnectorManifest(SLUG)) as Record | null); + assert.ok(sourceAfterInstall, "the registry retains a source for the installed connector"); + assert.equal( + installed.manifest.source_declaration !== undefined, + true, + "the install record carries the verified declaration" + ); + + // Same-digest reuse: the registry keeps the source and the record keeps its sidecar digest. + const reused = await first.service.install(SLUG, v1.entry.digest); + assert.equal(reused.sourceDeclarationSha256, installed.sourceDeclarationSha256); + assert.equal( + retainedSourceId((await getConnectorManifest(SLUG)) as Record | null), + sourceAfterInstall + ); + + // Restart: a new store and a new service read the same record and republish the same source. + const restarted = productionService(dataDir, [v1, v2]); + const inspected = await inspectActiveConnector(restarted.store, SLUG); + assert.equal(inspected.status, "active"); + await restarted.service.install(SLUG, v1.entry.digest); + assert.equal( + retainedSourceId((await getConnectorManifest(SLUG)) as Record | null), + sourceAfterInstall + ); + + // A new version whose declaration rebinds the retained source is refused by the registry, and the + // previous install stays active with its own source. + const rebinding = versionWith("1.2.0", OTHER_SOURCE); + const third = productionService(dataDir, [v1, rebinding]); + await assert.rejects(() => third.service.install(SLUG, rebinding.entry.digest), ALREADY_BOUND); + const active = await third.store.getActive(SLUG); + assert.equal(active?.digest, v1.entry.digest, "the previous install stays active"); + assert.equal( + retainedSourceId((await getConnectorManifest(SLUG)) as Record | null), + sourceAfterInstall + ); + + // Tampered bytes while stopped: refused on read, and the registry keeps the verified source. + writeFileSync(join(installed.root, "source-declaration.json"), '{"source":"tampered"}\n'); + const tampered = await inspectActiveConnector(createFileConnectorInstallStore(dataDir), SLUG); + assert.equal(tampered.status, "invalid"); + assert.equal( + retainedSourceId((await getConnectorManifest(SLUG)) as Record | null), + sourceAfterInstall + ); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +} + +test("SQLite: the verified declaration reaches the registry and survives reuse, restart and tamper", async () => { + initDb(":memory:"); + try { + await journey(); + } finally { + closeDb(); + } +}); + +test("PostgreSQL: the verified declaration reaches the registry and survives reuse, restart and tamper (skipped: PDPP_TEST_POSTGRES_URL unset)", { + skip: !process.env.PDPP_TEST_POSTGRES_URL, +}, async () => { + const url = process.env.PDPP_TEST_POSTGRES_URL; + assert.ok(url); + await withTemporaryPostgresDatabase( + { + closeConnections: closePostgresStorage, + connectionString: url, + databaseName: `pdpp_test_install_registry_${Date.now().toString(36)}_${Math.random().toString(16).slice(2, 10)}_1`, + templateName: null, + }, + async (databaseUrl) => { + initDb(":memory:"); + try { + await initPostgresStorage({ backend: "postgres", databaseUrl }); + await journey(); + } finally { + await closePostgresStorage(); + closeDb(); + } + } + ); +}); diff --git a/reference-implementation/test/connector-install-source-declaration.test.ts b/reference-implementation/test/connector-install-source-declaration.test.ts new file mode 100644 index 000000000..b3139d189 --- /dev/null +++ b/reference-implementation/test/connector-install-source-declaration.test.ts @@ -0,0 +1,577 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * The verified Core source declaration reaches registration through the install record. + * + * `normalizeCoreInstallLayout` moves the verified `source-declaration.json` into the install root. The + * install record must bind that file's digest, and registration must read the declaration from the + * verified record, not from the profile alone. These tests drive the real install service with a + * trusted verified-layout artifact through its installation seam (`installArtifact`, which stands in for + * the pinned OCI verifier); they do not exercise signatures. Registration here is a collector; the real + * registry and activation journey is covered by connector-install-registry-integration.test.ts. + */ + +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import fs, { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { + type ConnectorCatalogEntry, + type ConnectorInstallStore, + createConnectorInstallService, + createFileConnectorInstallStore, + inspectActiveConnector, +} from "../server/connector-install/index.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; +import { readCollectionProfileFixture } from "./helpers/installed-collection-profiles.ts"; + +const CONFIG_DIGEST = `sha256:${"c".repeat(64)}`; +const SLUG = "fixture-oci-custom"; +const INTEGRITY_REFUSAL = /source declaration bytes failed integrity verification/; +const SIDECAR_REFUSAL = /source declaration|Invalid SourceDeclaration/i; +const CONFLICT_REFUSAL = /conflicts with the verified source declaration/; +const SIDECAR_INVALID = /Verified source declaration is invalid/; +const PROFILE_TOO_LARGE = /Connector profile exceeds the 1 MiB RI limit/; +const REGISTRATION_REFUSED = /registration refused/; +const PINNED_SIDECAR_MISSING = /Verified source declaration is missing/; +const PINNED_SIDECAR_MISMATCH = /does not match installer-core verification/; +const MIB = 1024 * 1024; + +interface Layout { + readonly entry: ConnectorCatalogEntry; + readonly install?: (root: string) => void; + /** + * Set for a pinned install: the source-declaration bytes the pinned installer core verified. The installer + * reports their digest, and the sidecar on disk (`sidecar`) may then differ or be absent. + */ + readonly pinnedDeclaration?: string; + readonly profileBytes: string; + readonly sidecar: string | null; +} + +/** A signed-shape profile (streams and capabilities from the github fixture), re-keyed to a custom slug. */ +function profileFixture(version = "1.0.0"): Record { + return { + ...readCollectionProfileFixture("github"), + connector_id: SLUG, + connector_key: SLUG, + manifest_uri: `https://registry.pdpp.dev/connectors/${SLUG}`, + version, + }; +} + +function declarationOf(manifest: Record): Record { + const declared = withFixtureSourceDeclaration(manifest) as Record; + return declared.source_declaration as Record; +} + +function entryFor(bytes: string): ConnectorCatalogEntry { + return { + config_digest: CONFIG_DIGEST, + connector_id: SLUG, + connector_key: SLUG, + digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}`, + tier: "supported", + version: "1.0.0", + }; +} + +/** A layout whose catalog entry digests the profile bytes it installs, so the two can never disagree. */ +function layoutOf( + profile: Record, + sidecar: string | null = JSON.stringify(declarationOf(profile), null, 2) +): Layout { + const profileBytes = JSON.stringify(profile, null, 2); + return { entry: entryFor(profileBytes), profileBytes, sidecar }; +} + +/** Writes the layout that `normalizeCoreInstallLayout` produces: profile, dist, provenance, sidecar. */ +function writeVerifiedLayout(root: string, profileBytes: string, sidecarBytes: string | null): void { + mkdirSync(join(root, "profile"), { recursive: true }); + mkdirSync(join(root, "dist"), { recursive: true }); + writeFileSync(join(root, "profile", "collection-profile.json"), profileBytes); + writeFileSync(join(root, "dist", "collection-profile.mjs"), "export {};\n"); + writeFileSync(join(root, "provenance.json"), "{}\n"); + if (sidecarBytes !== null) { + writeFileSync(join(root, "source-declaration.json"), sidecarBytes); + } +} + +/** The real install service over a durable store, with a collector in place of the registry. */ +function serviceOver( + dataDir: string, + store: ConnectorInstallStore, + layouts: readonly Layout[], + options: { readonly failRegistrationOn?: number } = {} +) { + const registered: Record[] = []; + const service = createConnectorInstallService({ + catalogLoader: async () => layouts.map((layout) => layout.entry), + dataDir, + installArtifact: (root, entry) => { + const layout = layouts.find((candidate) => candidate.entry.digest === entry.digest); + if (!layout) { + throw new Error("no fixture layout for the requested digest"); + } + if (layout.install) { + layout.install(root); + } else { + writeVerifiedLayout(root, layout.profileBytes, layout.sidecar); + } + return layout.pinnedDeclaration === undefined + ? undefined + : { configDigest: CONFIG_DIGEST, sourceDeclarationSha256: digestOfText(layout.pinnedDeclaration) }; + }, + registerManifest: (manifest) => { + registered.push(manifest); + if (options.failRegistrationOn === registered.length) { + return Promise.reject(new Error("registration refused (test)")); + } + return Promise.resolve(); + }, + store, + }); + return { registered, service }; +} + +function harness(dataDir: string, layout: Layout) { + const store = createFileConnectorInstallStore(dataDir); + const { registered, service } = serviceOver(dataDir, store, [layout]); + return { entry: layout.entry, registered, service, store }; +} + +function sidecarOf(declaration: Record): string { + return JSON.stringify(declaration, null, 2); +} + +function digestOfText(bytes: string): string { + return `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +} + +/** + * Fails only the first lstat of the staged `source-declaration.json` with EIO. Every other filesystem call + * is the real one. The builtin export is swapped through the CJS object and resynced; callers restore it. + */ +function injectFirstSidecarStatEio(): { readonly faults: () => number; readonly restore: () => void } { + const realLstat = fs.lstatSync; + let faults = 0; + fs.lstatSync = ((path: fs.PathLike, options?: fs.StatOptions) => { + if (faults === 0 && String(path).endsWith("/next/source-declaration.json")) { + faults += 1; + throw Object.assign(new Error(`EIO: i/o error, lstat '${String(path)}'`), { + code: "EIO", + errno: -5, + path: String(path), + syscall: "lstat", + }); + } + return realLstat(path, options as never); + }) as typeof fs.lstatSync; + syncBuiltinESMExports(); + return { + faults: () => faults, + restore: () => { + fs.lstatSync = realLstat; + syncBuiltinESMExports(); + }, + }; +} + +test("the verified source declaration reaches registration from the install record", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-")); + try { + const profile = profileFixture(); + const declaration = declarationOf(profile); + const { entry, registered, service, store } = harness(dataDir, layoutOf(profile, sidecarOf(declaration))); + + const record = await service.install(entry.connector_id, entry.digest); + + assert.equal(registered.length, 1); + assert.notEqual(registered[0]?.source_declaration, undefined, "registration received no source declaration"); + assert.deepEqual(registered[0]?.source_declaration, declaration); + assert.deepEqual(record.manifest.source_declaration, declaration); + const inspected = await inspectActiveConnector(store, SLUG); + assert.equal(inspected.status, "active"); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a modified source declaration sidecar is refused on read verification", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-tamper-")); + try { + const profile = profileFixture(); + const { entry, service, store } = harness(dataDir, layoutOf(profile, sidecarOf(declarationOf(profile)))); + const record = await service.install(entry.connector_id, entry.digest); + + writeFileSync(join(record.root, "source-declaration.json"), '{"source":"changed"}\n'); + + const inspected = await inspectActiveConnector(store, SLUG); + assert.ok(inspected.status === "invalid"); + assert.match(inspected.reason, INTEGRITY_REFUSAL); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a missing source declaration sidecar is refused on read verification", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-missing-")); + try { + const profile = profileFixture(); + const { entry, service, store } = harness(dataDir, layoutOf(profile, sidecarOf(declarationOf(profile)))); + const record = await service.install(entry.connector_id, entry.digest); + + rmSync(join(record.root, "source-declaration.json")); + + const inspected = await inspectActiveConnector(store, SLUG); + assert.ok(inspected.status === "invalid"); + assert.match(inspected.reason, INTEGRITY_REFUSAL); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a symlinked source declaration sidecar is refused before publication", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-symlink-")); + const outside = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-outside-")); + try { + const profile = profileFixture(); + const profileBytes = JSON.stringify(profile, null, 2); + const declarationPath = join(outside, "declaration.json"); + writeFileSync(declarationPath, sidecarOf(declarationOf(profile))); + const layout: Layout = { + entry: entryFor(profileBytes), + install: (root) => { + writeVerifiedLayout(root, profileBytes, null); + symlinkSync(declarationPath, join(root, "source-declaration.json")); + }, + profileBytes, + sidecar: null, + }; + const { entry, registered, service, store } = harness(dataDir, layout); + + await assert.rejects(() => service.install(entry.connector_id, entry.digest), SIDECAR_REFUSAL); + assert.equal(registered.length, 0); + assert.equal((await store.getActive(SLUG)) ?? null, null); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + rmSync(outside, { force: true, recursive: true }); + } +}); + +test("a profile that embeds a conflicting source declaration is refused before publication", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-conflict-")); + try { + // The catalog digests the embedded profile that is actually installed (version 2.0.0), so the refusal + // is the intended conflict between the embedded declaration and the sidecar, not a digest mismatch. + const sidecarDeclaration = declarationOf(profileFixture()); + const embedded = withFixtureSourceDeclaration({ ...profileFixture("2.0.0"), version: "2.0.0" }); + const layout = layoutOf(embedded, sidecarOf(sidecarDeclaration)); + const { entry, registered, service, store } = harness(dataDir, layout); + + await assert.rejects(() => service.install(entry.connector_id, entry.digest), CONFLICT_REFUSAL); + assert.equal(registered.length, 0); + assert.equal((await store.getActive(SLUG)) ?? null, null); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("same-digest reuse re-registers the verified declaration and keeps the recorded sidecar digest", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-reuse-")); + try { + const profile = profileFixture(); + const declaration = declarationOf(profile); + const { entry, registered, service } = harness(dataDir, layoutOf(profile, sidecarOf(declaration))); + const first = await service.install(entry.connector_id, entry.digest); + assert.equal(typeof first.sourceDeclarationSha256, "string", "the install record binds the sidecar digest"); + + const second = await service.install(entry.connector_id, entry.digest); + + assert.equal(registered.length, 2, "reuse re-registers the verified manifest"); + assert.deepEqual(registered[1]?.source_declaration, declaration); + assert.equal(second.sourceDeclarationSha256, first.sourceDeclarationSha256); + assert.equal(second.manifestSha256, first.manifestSha256, "the raw profile bytes and hash are unchanged"); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("restart: a new store instance and a new service read the same declaration from the record", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-restart-")); + try { + const profile = profileFixture(); + const declaration = declarationOf(profile); + const layout = layoutOf(profile, sidecarOf(declaration)); + const first = harness(dataDir, layout); + const installed = await first.service.install(SLUG, layout.entry.digest); + + const restartedStore = createFileConnectorInstallStore(dataDir); + const inspected = await inspectActiveConnector(restartedStore, SLUG); + assert.equal(inspected.status, "active"); + assert.equal(inspected.record.sourceDeclarationSha256, installed.sourceDeclarationSha256); + assert.deepEqual(inspected.record.manifest.source_declaration, declaration); + + const restarted = serviceOver(dataDir, restartedStore, [layout]); + const reinstalled = await restarted.service.install(SLUG, layout.entry.digest); + assert.deepEqual(restarted.registered[0]?.source_declaration, declaration); + assert.equal(reinstalled.sourceDeclarationSha256, installed.sourceDeclarationSha256); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("restart: a sidecar tampered while stopped is refused and no tampered bytes are registered", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-restart-tamper-")); + try { + const profile = profileFixture(); + const declaration = declarationOf(profile); + const layout = layoutOf(profile, sidecarOf(declaration)); + const first = harness(dataDir, layout); + const installed = await first.service.install(SLUG, layout.entry.digest); + writeFileSync(join(installed.root, "source-declaration.json"), '{"source":"tampered"}\n'); + + const restartedStore = createFileConnectorInstallStore(dataDir); + const inspected = await inspectActiveConnector(restartedStore, SLUG); + assert.equal(inspected.status, "invalid"); + assert.match(inspected.reason, INTEGRITY_REFUSAL); + + const restarted = serviceOver(dataDir, restartedStore, [layout]); + await restarted.service.install(SLUG, layout.entry.digest); + assert.ok(restarted.registered.length >= 1); + for (const manifest of restarted.registered) { + assert.deepEqual(manifest.source_declaration, declaration, "only the verified declaration is registered"); + } + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("restart: a missing sidecar is refused by a new store instance", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-restart-missing-")); + try { + const profile = profileFixture(); + const layout = layoutOf(profile, sidecarOf(declarationOf(profile))); + const installed = await harness(dataDir, layout).service.install(SLUG, layout.entry.digest); + rmSync(join(installed.root, "source-declaration.json")); + + const inspected = await inspectActiveConnector(createFileConnectorInstallStore(dataDir), SLUG); + assert.equal(inspected.status, "invalid"); + assert.match(inspected.reason, INTEGRITY_REFUSAL); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a legacy record without the sidecar digest does not trust a newly supplied sidecar", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-legacy-")); + try { + const profile = profileFixture(); + const layout = layoutOf(profile, null); + const first = harness(dataDir, layout); + const installed = await first.service.install(SLUG, layout.entry.digest); + assert.equal(installed.sourceDeclarationSha256, undefined); + + // Simulate a record written before the sidecar digest existed, then supply a sidecar afterwards. + const statePath = join(dataDir, "connector-install-state.json"); + const state = JSON.parse(readFileSync(statePath, "utf8")) as Record>; + const record = state[SLUG]; + assert.ok(record, "the install state holds the installed connector"); + Reflect.deleteProperty(record, "sourceDeclarationSha256"); + writeFileSync(statePath, `${JSON.stringify(state, null, 2)}\n`); + writeFileSync(join(installed.root, "source-declaration.json"), sidecarOf(declarationOf(profileFixture()))); + + const inspected = await inspectActiveConnector(createFileConnectorInstallStore(dataDir), SLUG); + assert.equal(inspected.status, "active"); + assert.equal(inspected.record.manifest.source_declaration, undefined, "the unrecorded sidecar is not trusted"); + assert.equal(inspected.record.manifestSha256, installed.manifestSha256); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("an oversize sidecar is refused before publication and nothing is registered", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-oversize-")); + try { + const profile = profileFixture(); + const padded = `${sidecarOf(declarationOf(profile))}${" ".repeat(MIB)}`; + const { entry, registered, service, store } = harness(dataDir, layoutOf(profile, padded)); + + await assert.rejects(() => service.install(entry.connector_id, entry.digest), SIDECAR_INVALID); + assert.equal(registered.length, 0); + assert.equal((await store.getActive(SLUG)) ?? null, null); + assert.equal(existsSync(join(dataDir, "connectors", SLUG, entry.digest)), false, "the staged root is removed"); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("an oversize profile is refused before publication and nothing is registered", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-profile-oversize-")); + try { + const profile = { ...profileFixture(), padding: "x".repeat(MIB) }; + const { entry, registered, service, store } = harness( + dataDir, + layoutOf(profile, sidecarOf(declarationOf(profile))) + ); + + await assert.rejects(() => service.install(entry.connector_id, entry.digest), PROFILE_TOO_LARGE); + assert.equal(registered.length, 0); + assert.equal((await store.getActive(SLUG)) ?? null, null); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a sidecar that is valid JSON but not a valid Core declaration is refused and nothing is registered", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-shape-")); + try { + const profile = profileFixture(); + const invalid = { ...declarationOf(profile), protocol_version: "9.9.9" }; + const { entry, registered, service, store } = harness(dataDir, layoutOf(profile, sidecarOf(invalid))); + + await assert.rejects(() => service.install(entry.connector_id, entry.digest), SIDECAR_REFUSAL); + assert.equal(registered.length, 0); + assert.equal((await store.getActive(SLUG)) ?? null, null); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a failed registration of a new version rolls back to the previous good install", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-rollback-")); + try { + const v1 = layoutOf(profileFixture("1.0.0")); + const v2 = layoutOf(profileFixture("1.1.0")); + const store = createFileConnectorInstallStore(dataDir); + const good = serviceOver(dataDir, store, [v1]); + const previous = await good.service.install(SLUG, v1.entry.digest); + + const failing = serviceOver(dataDir, store, [v1, v2], { failRegistrationOn: 1 }); + await assert.rejects(() => failing.service.install(SLUG, v2.entry.digest), REGISTRATION_REFUSED); + + const active = await store.getActive(SLUG); + assert.equal(active?.digest, previous.digest, "the previous good install stays active"); + assert.equal(active?.sourceDeclarationSha256, previous.sourceDeclarationSha256); + assert.equal(existsSync(join(dataDir, "connectors", SLUG, v2.entry.digest)), false, "the failed root is removed"); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("an unreadable sidecar fails the install instead of being recorded as absent", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-eio-")); + try { + const v1 = layoutOf(profileFixture("1.0.0")); + const v2 = layoutOf(profileFixture("1.1.0")); + const store = createFileConnectorInstallStore(dataDir); + const previous = await serviceOver(dataDir, store, [v1]).service.install(SLUG, v1.entry.digest); + + const failing = serviceOver(dataDir, store, [v1, v2]); + const fault = injectFirstSidecarStatEio(); + try { + await assert.rejects(() => failing.service.install(SLUG, v2.entry.digest), { code: "EIO" }); + } finally { + fault.restore(); + } + + assert.equal(fault.faults(), 1, "the fault fired on the staged sidecar"); + assert.equal(failing.registered.length, 0, "nothing is registered from the unreadable install"); + const active = await store.getActive(SLUG); + assert.equal(active?.digest, previous.digest, "the previous good install stays active"); + assert.equal(active?.sourceDeclarationSha256, previous.sourceDeclarationSha256); + assert.equal(existsSync(join(dataDir, "connectors", SLUG, v2.entry.digest)), false, "the staged root is removed"); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a pinned install records the Core-verified sidecar digest and registers the verified declaration", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-pinned-")); + try { + const profile = profileFixture(); + const sidecar = sidecarOf(declarationOf(profile)); + const layout: Layout = { ...layoutOf(profile, sidecar), pinnedDeclaration: sidecar }; + const { entry, registered, service } = harness(dataDir, layout); + + const record = await service.install(SLUG, entry.digest); + + assert.equal(record.sourceDeclarationSha256, digestOfText(sidecar)); + assert.deepEqual(registered[0]?.source_declaration, declarationOf(profile)); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a pinned install whose verified sidecar is missing after normalization is refused, not recorded as legacy", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-pinned-missing-")); + try { + const profile = profileFixture(); + const verified = sidecarOf(declarationOf(profile)); + const layout: Layout = { ...layoutOf(profile, null), pinnedDeclaration: verified }; + const { entry, registered, service, store } = harness(dataDir, layout); + + await assert.rejects(() => service.install(SLUG, entry.digest), PINNED_SIDECAR_MISSING); + assert.equal(registered.length, 0); + assert.equal((await store.getActive(SLUG)) ?? null, null); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a pinned install whose sidecar bytes differ from the verified digest is refused", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-pinned-mismatch-")); + try { + const profile = profileFixture(); + const verified = sidecarOf(declarationOf(profile)); + const layout: Layout = { ...layoutOf(profile, '{"source":"other"}\n'), pinnedDeclaration: verified }; + const { entry, registered, service, store } = harness(dataDir, layout); + + await assert.rejects(() => service.install(SLUG, entry.digest), PINNED_SIDECAR_MISMATCH); + assert.equal(registered.length, 0); + assert.equal((await store.getActive(SLUG)) ?? null, null); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a transient EIO on a pinned install's sidecar refuses the install and registers nothing", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-pinned-eio-")); + try { + const profile = profileFixture(); + const sidecar = sidecarOf(declarationOf(profile)); + const layout: Layout = { ...layoutOf(profile, sidecar), pinnedDeclaration: sidecar }; + const { entry, registered, service, store } = harness(dataDir, layout); + const fault = injectFirstSidecarStatEio(); + try { + await assert.rejects(() => service.install(SLUG, entry.digest), { code: "EIO" }); + } finally { + fault.restore(); + } + + assert.equal(fault.faults(), 1); + assert.equal(registered.length, 0); + assert.equal((await store.getActive(SLUG)) ?? null, null); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a legacy install with no sidecar stays legacy and records no sidecar digest", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-legacy-absent-")); + try { + const profile = profileFixture(); + const { entry, registered, service } = harness(dataDir, layoutOf(profile, null)); + + const record = await service.install(SLUG, entry.digest); + + assert.equal(record.sourceDeclarationSha256, undefined); + assert.equal(registered[0]?.source_declaration, undefined); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); diff --git a/reference-implementation/test/connector-install-sql-persistence.test.ts b/reference-implementation/test/connector-install-sql-persistence.test.ts new file mode 100644 index 000000000..df7ddccda --- /dev/null +++ b/reference-implementation/test/connector-install-sql-persistence.test.ts @@ -0,0 +1,284 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * The digest-bearing install record through the production SQL store. + * + * `createConnectorInstallStore()` (no preload directory) persists `connector_installs.record_json`. This journey + * installs a pinned fixture through the production registration wiring, reads the record back through fresh store + * instances, and reads the activation through the lazy path (`getConnectorActivation`, which imports the legacy + * install row into `connector_activations`). It runs on SQLite always and on PostgreSQL 18 when + * PDPP_TEST_POSTGRES_URL is set. The OCI artifact is a trusted fixture that reports a verified source declaration + * digest; signatures are not exercised. Compare connector-install-registry-integration.test.ts, which uses the + * file install store. + */ + +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { getConnectorManifest } from "../server/auth.ts"; +import { getConnectorActivation } from "../server/connector-install/activation-authority.ts"; +import { + type ConnectorCatalogEntry, + type ConnectorInstallRecord, + createConnectorInstallService, + createConnectorInstallStore, + inspectActiveConnector, +} from "../server/connector-install/index.ts"; +import { installedConnectorManifestRegistration } from "../server/connector-install/manifest-registration.ts"; +import { closeDb, getDb, initDb } from "../server/db.ts"; +import { + closePostgresStorage, + initPostgresStorage, + isPostgresStorageBackend, + postgresQuery, +} from "../server/postgres-storage.ts"; +import { snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest } from "../server/source-declaration-legacy-collection.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; +import { readCollectionProfileFixture } from "./helpers/installed-collection-profiles.ts"; +import { withTemporaryPostgresDatabase } from "./helpers/postgres-temp-database.ts"; + +const SLUG = "fixture-sql-persistence"; +const CONFIG_DIGEST = `sha256:${"e".repeat(64)}`; +const OTHER_SOURCE = "https://sources.example/other/sql-persistence"; +const INTEGRITY_REFUSAL = /source declaration bytes failed integrity verification/; +const CONFLICT_REFUSAL = /conflicts with the verified source declaration/; + +function digestOf(bytes: string): string { + return `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +} + +function profileFor(version: string): Record { + return { + ...readCollectionProfileFixture("github"), + connector_id: SLUG, + connector_key: SLUG, + manifest_uri: `https://registry.pdpp.dev/connectors/${SLUG}`, + version, + }; +} + +function declarationOf(profile: Record): Record { + return withFixtureSourceDeclaration(profile).source_declaration as Record; +} + +interface Version { + readonly entry: ConnectorCatalogEntry; + readonly profileBytes: string; + /** The verified sidecar bytes the pinned installer core reports, and the install writes. */ + readonly sidecar: string; +} + +/** + * A version. `embedded` is written into the profile itself; it is absent unless a test needs a profile that + * carries a declaration (equal, or conflicting with the sidecar). + */ +function versionOf(version: string, embedded?: unknown): Version { + const base = profileFor(version); + const profile = embedded === undefined ? base : { ...base, source_declaration: embedded }; + const profileBytes = JSON.stringify(profile, null, 2); + return { + entry: { + config_digest: CONFIG_DIGEST, + connector_id: SLUG, + connector_key: SLUG, + digest: digestOf(profileBytes), + tier: "supported", + version, + }, + profileBytes, + sidecar: JSON.stringify(declarationOf(base), null, 2), + }; +} + +function writeLayout(root: string, version: Version): void { + mkdirSync(join(root, "profile"), { recursive: true }); + mkdirSync(join(root, "dist"), { recursive: true }); + writeFileSync(join(root, "profile", "collection-profile.json"), version.profileBytes); + writeFileSync(join(root, "dist", "collection-profile.mjs"), "export {};\n"); + writeFileSync(join(root, "provenance.json"), "{}\n"); + writeFileSync(join(root, "source-declaration.json"), version.sidecar); +} + +/** The production store and registration wiring over the owned data directory. */ +function serviceOver(dataDir: string, versions: readonly Version[]) { + const registered: Record[] = []; + const service = createConnectorInstallService({ + catalogLoader: async () => versions.map((version) => version.entry), + dataDir, + installArtifact: (root, entry) => { + const version = versions.find((candidate) => candidate.entry.digest === entry.digest); + if (!version) { + throw new Error("no fixture version for the requested digest"); + } + writeLayout(root, version); + return { + configDigest: CONFIG_DIGEST, + sourceDeclarationSha256: digestOf(version.sidecar), + }; + }, + registerManifest: async (manifest) => { + registered.push(manifest); + return await installedConnectorManifestRegistration.registerManifest(manifest); + }, + repairManifestDerivedState: installedConnectorManifestRegistration.repairManifestDerivedState, + store: createConnectorInstallStore(), + }); + return { registered, service }; +} + +async function storedRecord(): Promise { + const json = isPostgresStorageBackend() + ? ( + await postgresQuery<{ record_json: string }>( + "SELECT record_json FROM connector_installs WHERE connector_id=$1", + [SLUG] + ) + ).rows[0]?.record_json + : getDb() + .prepare("SELECT record_json FROM connector_installs WHERE connector_id=?") + .get<{ record_json: string }>(SLUG)?.record_json; + return json === undefined ? null : (JSON.parse(json) as ConnectorInstallRecord); +} + +async function retainedSourceId(): Promise { + const declaration = ( + (await getConnectorManifest(SLUG)) as { + source_declaration?: { source?: { id?: unknown } }; + } | null + )?.source_declaration; + return declaration?.source?.id; +} + +/** Runs `run` with PDPP_DATA_DIR owned by the test and no preload directory, restoring both afterwards. */ +async function withOwnedDataDir(run: (dataDir: string) => Promise): Promise { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-install-sql-")); + const previousDataDir = process.env.PDPP_DATA_DIR; + const previousPreload = process.env.PDPP_CONNECTOR_PRELOAD_DIR; + process.env.PDPP_DATA_DIR = dataDir; + delete process.env.PDPP_CONNECTOR_PRELOAD_DIR; + try { + await run(dataDir); + } finally { + if (previousDataDir === undefined) { + delete process.env.PDPP_DATA_DIR; + } else { + process.env.PDPP_DATA_DIR = previousDataDir; + } + if (previousPreload === undefined) { + delete process.env.PDPP_CONNECTOR_PRELOAD_DIR; + } else { + process.env.PDPP_CONNECTOR_PRELOAD_DIR = previousPreload; + } + rmSync(dataDir, { force: true, recursive: true }); + } +} + +async function journey(dataDir: string): Promise { + const v1 = versionOf("1.0.0"); + const declaration = declarationOf(profileFor("1.0.0")); + const first = serviceOver(dataDir, [v1]); + + // Install through the production store; the record carries the Core-verified sidecar digest. + const installed = await first.service.install(SLUG, v1.entry.digest); + assert.equal(installed.sourceDeclarationSha256, digestOf(v1.sidecar)); + + // A fresh read from SQL: the digest, the raw profile hash and the verified declaration survive the round trip. + const persisted = await storedRecord(); + assert.ok(persisted, "the SQL install store holds the record"); + assert.equal(persisted.sourceDeclarationSha256, digestOf(v1.sidecar), "the stored record keeps the sidecar digest"); + assert.equal(persisted.manifestSha256, digestOf(v1.profileBytes), "the raw profile hash is unchanged"); + assert.deepEqual(persisted.manifest.source_declaration, declaration); + const inspected = await inspectActiveConnector(createConnectorInstallStore(), SLUG); + assert.equal(inspected.status, "active"); + assert.equal(inspected.record.sourceDeclarationSha256, digestOf(v1.sidecar)); + + // Lazy activation: the first read imports the install row into connector_activations. + const activation = await getConnectorActivation(SLUG); + assert.ok(activation, "the activation is imported on first read"); + assert.equal(activation.state, "active", "the registry and the installed artifact agree"); + assert.equal(activation.record.sourceDeclarationSha256, digestOf(v1.sidecar)); + assert.deepEqual(activation.record.manifest.source_declaration, declaration); + assert.equal((await getConnectorActivation(SLUG))?.activationId, activation.activationId, "the import is stable"); + + // Same-digest reuse keeps the record's digest and the retained registry source. + const sourceAfterInstall = await retainedSourceId(); + assert.ok(sourceAfterInstall, "the registry retains a source"); + const reused = await first.service.install(SLUG, v1.entry.digest); + assert.equal(reused.sourceDeclarationSha256, digestOf(v1.sidecar)); + assert.equal((await storedRecord())?.sourceDeclarationSha256, digestOf(v1.sidecar)); + assert.equal(await retainedSourceId(), sourceAfterInstall); + + // Tampered while stopped: the read is refused, and a repair registers only the verified declaration. + const root = join(dataDir, "connectors", SLUG, v1.entry.digest); + writeFileSync(join(root, "source-declaration.json"), '{"source":"tampered"}\n'); + const tampered = await inspectActiveConnector(createConnectorInstallStore(), SLUG); + assert.equal(tampered.status, "invalid"); + assert.match(tampered.reason, INTEGRITY_REFUSAL); + + const repaired = serviceOver(dataDir, [v1]); + const repairedRecord = await repaired.service.install(SLUG, v1.entry.digest); + assert.equal(repaired.registered.length, 1, "the repair registers once"); + for (const manifest of repaired.registered) { + assert.deepEqual(manifest.source_declaration, declaration, "only the verified declaration is registered"); + } + assert.equal(repairedRecord.sourceDeclarationSha256, digestOf(v1.sidecar)); + assert.equal(await retainedSourceId(), sourceAfterInstall, "the registry keeps the verified source"); + + // A profile whose embedded declaration conflicts with the sidecar is refused and changes nothing. + const conflicting = versionOf( + "1.1.0", + snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest(profileFor("1.1.0"), { + connectorImplementationId: `https://implementations.example/connectors/${SLUG}`, + publisherId: "https://publishers.example/sql-persistence", + sourceId: OTHER_SOURCE, + }) + ); + const rejecting = serviceOver(dataDir, [v1, conflicting]); + await assert.rejects(() => rejecting.service.install(SLUG, conflicting.entry.digest), CONFLICT_REFUSAL); + assert.equal(rejecting.registered.length, 0, "the conflicting profile is never registered"); + assert.equal((await storedRecord())?.digest, v1.entry.digest, "the previous install stays active"); + + // A profile that embeds an equal declaration is accepted and records its own sidecar digest. + const equal = versionOf("1.2.0", declarationOf(profileFor("1.2.0"))); + const accepted = await serviceOver(dataDir, [equal]).service.install(SLUG, equal.entry.digest); + assert.equal(accepted.sourceDeclarationSha256, digestOf(equal.sidecar)); + assert.equal((await storedRecord())?.digest, equal.entry.digest); +} + +test("SQLite: a pinned install's source declaration digest survives the SQL store, lazy activation and repair", async () => { + initDb(":memory:"); + try { + await withOwnedDataDir(journey); + } finally { + closeDb(); + } +}); + +test("PostgreSQL: a pinned install's source declaration digest survives the SQL store, lazy activation and repair (skipped: PDPP_TEST_POSTGRES_URL unset)", { + skip: !process.env.PDPP_TEST_POSTGRES_URL, +}, async () => { + const url = process.env.PDPP_TEST_POSTGRES_URL; + assert.ok(url); + await withTemporaryPostgresDatabase( + { + closeConnections: closePostgresStorage, + connectionString: url, + databaseName: `pdpp_test_install_sql_${Date.now().toString(36)}_${Math.random().toString(16).slice(2, 10)}_1`, + templateName: null, + }, + async (databaseUrl) => { + initDb(":memory:"); + try { + await initPostgresStorage({ backend: "postgres", databaseUrl }); + await withOwnedDataDir(journey); + } finally { + await closePostgresStorage(); + closeDb(); + } + } + ); +}); diff --git a/reference-implementation/test/connector-install.test.ts b/reference-implementation/test/connector-install.test.ts index 97e44e2a6..5c76c5a91 100644 --- a/reference-implementation/test/connector-install.test.ts +++ b/reference-implementation/test/connector-install.test.ts @@ -21,6 +21,7 @@ import { dirname, join, resolve } from "node:path"; import { createRequire } from "node:module"; import { pathToFileURL } from "node:url"; import test from "node:test"; +import { validateConnectorManifest } from "../server/connector-manifest-validation.ts"; import { type ConnectorCatalogEntry, @@ -62,6 +63,7 @@ const RE_SYMBOLIC_LINK = /symbolic-link/; const RE_UPDATE_FAILED = /crashed while staging update/; const RE_UPDATE_TARGET_MISSING = /no verified update target/; const RE_REGISTRATION_FAILED = /manifest registration failed/; +const RE_LOCAL_EXPAND_INVALID = /query\.expand.*must match a same-stream relationships\[\] entry/; const require = createRequire(import.meta.url); async function waitFor(condition: () => boolean, timeoutMs = 5000): Promise { const started = Date.now(); @@ -86,6 +88,43 @@ const entry: ConnectorCatalogEntry = { version: "1.0.0", }; +test("adding a developer source rejects invalid query.expand before publishing local source state", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-local-profile-validation-")); + const sourceRoot = join(dataDir, "candidate"); + try { + const profile = JSON.parse( + readFileSync(new URL("./fixtures/collection-profiles/github.json", import.meta.url), "utf8") + ) as Record; + // The authoritative validator accepts the unmodified profile. This test + // changes only the semantic contract that shallow local checks miss. + validateConnectorManifest(profile); + assert.ok(Array.isArray(profile.streams)); + const firstStream = profile.streams[0] as Record; + firstStream.query = { expand: [{ name: "missing_relationship" }] }; + assert.throws(() => validateConnectorManifest(profile), RE_LOCAL_EXPAND_INVALID); + + mkdirSync(join(sourceRoot, "profile"), { recursive: true }); + mkdirSync(join(sourceRoot, "dist")); + writeFileSync(join(sourceRoot, "profile", "collection-profile.json"), JSON.stringify(profile)); + writeFileSync(join(sourceRoot, "dist", "collection-profile.mjs"), "export {};\n"); + const localSourceStore = createFileLocalConnectorSourceStore(dataDir); + const service = createConnectorInstallService({ + dataDir, + localSourceStore, + registerManifest: async (manifest) => { + validateConnectorManifest(manifest); + }, + store: createFileConnectorInstallStore(dataDir), + }); + const { addLocalSource } = service; + assert.ok(addLocalSource); + await assert.rejects(() => addLocalSource(sourceRoot), RE_LOCAL_EXPAND_INVALID); + assert.deepEqual(await localSourceStore.list(), [], "a rejected profile must not be published as a local source"); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + test("pinned catalog schema accepts binding features and rejects unknown binding fields", () => { const catalog = { catalog_version: "1.0", @@ -148,16 +187,16 @@ test("owner local-source state lives in PDPP_DATA_DIR even when PDPP_CONNECTOR_P try { mkdirSync(join(sourceRoot, "profile")); mkdirSync(join(sourceRoot, "dist")); + const profile = JSON.parse( + readFileSync(new URL("./fixtures/collection-profiles/github.json", import.meta.url), "utf8") + ) as Record; + profile.connector_id = "https://vendor.example/connectors/preload-split-fixture"; + profile.manifest_uri = profile.connector_id; + profile.connector_key = "preload-split-fixture"; + profile.version = "0.1.0"; writeFileSync( join(sourceRoot, "profile", "collection-profile.json"), - JSON.stringify({ - connector_id: "https://registry.pdpp.dev/connectors/preload-split-fixture", - connector_key: "preload-split-fixture", - display_name: "Preload split fixture", - runtime_requirements: { bindings: { network: { required: true } } }, - streams: [{ name: "items" }], - version: "0.1.0", - }) + JSON.stringify(profile) ); writeFileSync(join(sourceRoot, "dist", "collection-profile.mjs"), "export default {};\n"); diff --git a/reference-implementation/test/connector-manifest-validation.test.ts b/reference-implementation/test/connector-manifest-validation.test.ts index 5684de7d6..43195cdd1 100644 --- a/reference-implementation/test/connector-manifest-validation.test.ts +++ b/reference-implementation/test/connector-manifest-validation.test.ts @@ -39,7 +39,9 @@ const TOP_LEVEL_REGEX_27 = /capabilities\.proven\.local_collector=true requires runtime_requirements\.bindings\.filesystem/; const TOP_LEVEL_REGEX_28 = /credential_capture field 'password' is a secret field .* with no label/; const TOP_LEVEL_REGEX_29 = /credential_capture field 'password' is a secret field with zero env aliases/; -const PROFILE_VERSION_PATTERN = /protocol_version must be one of 0\.1\.0, 0\.2\.0, received 0\.3\.0/; +// The unknown-version message is a fixed rule. It never echoes the manifest's own value. +const PROFILE_VERSION_PATTERN = /protocol_version must be one of 0\.1\.0, 0\.2\.0$/; +const PROFILE_VERSION_MESSAGE = "protocol_version must be one of 0.1.0, 0.2.0"; const PROFILE_0_2_0_CAPABILITY_PATTERN = /protocol_capabilities DETAIL_COVERAGE_KEY_SETS require protocol_version 0\.2\.0/; const UNKNOWN_CAPABILITY_PATTERN = /protocol_capabilities must be an array of known capability names/; @@ -227,6 +229,11 @@ test("validateConnectorManifest enforces the Collection Profile declaration at r validateConnectorManifest({ ...manifestWithStream(), ...keySets, protocol_version: "0.2.0" }) ); assert.doesNotThrow(() => validateConnectorManifest({ ...manifestWithStream(), protocol_capabilities: ["BLOB"] })); + assert.throws( + () => validateConnectorManifest({ ...manifestWithStream(), protocol_version: "0.3.0" }), + (error: unknown) => (error as Error).message === PROFILE_VERSION_MESSAGE, + "an unknown version is rejected with the fixed rule, not an echo of the manifest value" + ); const rejected: [Record, RegExp][] = [ [{ protocol_version: "0.3.0" }, PROFILE_VERSION_PATTERN], [{ ...keySets, protocol_version: "0.1.0" }, PROFILE_0_2_0_CAPABILITY_PATTERN], diff --git a/reference-implementation/test/connector-registry-manifest-derivation.test.ts b/reference-implementation/test/connector-registry-manifest-derivation.test.ts index 540fb3f7c..cb309dc4e 100644 --- a/reference-implementation/test/connector-registry-manifest-derivation.test.ts +++ b/reference-implementation/test/connector-registry-manifest-derivation.test.ts @@ -69,6 +69,7 @@ import { readPolyfillManifests } from "@pdpp/polyfill-connectors/manifests"; import { seedableConnectorsFromManifests } from "../cli/commands/seed.ts"; import { SUPPORTED_SEED_CONNECTOR_KEYS } from "../connectors/seed/index.ts"; +import { validateLegacySourceBindings } from "../scripts/legacy-source-binding-validation.ts"; import type { ConnectorManifestLike } from "../server/connection-setup-plan.ts"; import { buildConnectionSetupPlan, @@ -79,6 +80,7 @@ import { canonicalConnectorKey, isConnectorKey } from "../server/connector-key.t import { BROWSER_BOUND_KEYS, FIRST_PARTY_CONNECTOR_KEYS, + LEGACY_FIRST_PARTY_SOURCE_ENTRIES, LEGACY_LOCAL_ALIASES, LOCAL_COLLECTOR_PROVEN_KEYS, NATIVE_CONNECTOR_KEYS, @@ -86,6 +88,12 @@ import { STATIC_SECRET_LIVE_PROVEN_KEYS, } from "../server/generated/connector-registry.generated.ts"; +const GENERATED_BINDINGS_LINE = /^export const LEGACY_FIRST_PARTY_SOURCE_ENTRIES:[^\n]*$/m; +const MAPPING_CHANGED = /Pinned historical source mapping changed/; +const DUPLICATE_BINDINGS = /duplicate keys or source IDs/; +const INVALID_BINDING_KEY = /Invalid pinned connector_key/; +const INVALID_BINDING_SOURCE = /Invalid pinned legacy connector_id/; + const trackedRegistry = { BROWSER_BOUND_KEYS, FIRST_PARTY_CONNECTOR_KEYS, @@ -173,6 +181,98 @@ test("the generator throws (does not silently drop the claim) when a manifest de } }); +test("historical source bindings stay pinned to real package identities, independent of fixture overrides", () => { + const scratchDir = mkdtempSync(join(tmpdir(), "connector-registry-binding-pin-")); + try { + for (const { file, manifest } of readPolyfillManifests()) { + writeFileSync( + join(scratchDir, file), + JSON.stringify({ ...(manifest as Record), manifest_uri: "https://provenance.example/changed" }) + ); + } + const scratchPath = join(scratchDir, "generated.ts"); + execFileSync("node", ["--experimental-strip-types", generatorScript, scratchPath], { + cwd: riRoot, + env: { ...process.env, PDPP_POLYFILL_MANIFESTS_DIR: scratchDir }, + stdio: "pipe", + }); + assert.equal(readFileSync(scratchPath, "utf8"), readFileSync(trackedRegistryPath, "utf8")); + + const firstManifestFile = readdirSync(scratchDir).find((file) => file.endsWith(".json")); + assert.ok(firstManifestFile, "the override fixture contains a manifest"); + const firstManifestPath = join(scratchDir, firstManifestFile); + const changed = JSON.parse(readFileSync(firstManifestPath, "utf8")) as Record; + changed.connector_id = "https://registry.pdpp.dev/connectors/repointed"; + writeFileSync(firstManifestPath, JSON.stringify(changed)); + execFileSync("node", ["--experimental-strip-types", generatorScript, scratchPath], { + cwd: riRoot, + env: { ...process.env, PDPP_POLYFILL_MANIFESTS_DIR: scratchDir }, + stdio: "pipe", + }); + const overridden = readFileSync(scratchPath, "utf8"); + const generatedBindings = (source: string): string => { + const match = source.match(GENERATED_BINDINGS_LINE); + assert.ok(match, "the generator emits historical binding data"); + return match[0]; + }; + assert.equal(generatedBindings(overridden), generatedBindings(readFileSync(trackedRegistryPath, "utf8"))); + + const overrideKey = "fixture-override-probe"; + writeFileSync( + join(scratchDir, `${overrideKey}.json`), + JSON.stringify({ + connector_id: "https://fixture.example/connectors/fixture-override-probe", + connector_key: overrideKey, + manifest_uri: "https://provenance.example/override-probe", + runtime_requirements: { bindings: { browser: { required: true } } }, + }) + ); + execFileSync("node", ["--experimental-strip-types", generatorScript, scratchPath], { + cwd: riRoot, + env: { ...process.env, PDPP_POLYFILL_MANIFESTS_DIR: scratchDir }, + stdio: "pipe", + }); + const withOverrideProbe = readFileSync(scratchPath, "utf8"); + assert.match(withOverrideProbe, new RegExp(`BROWSER_BOUND_KEYS:[^\\n]*${overrideKey}`)); + assert.equal(generatedBindings(withOverrideProbe), generatedBindings(readFileSync(trackedRegistryPath, "utf8"))); + } finally { + rmSync(scratchDir, { force: true, recursive: true }); + } +}); + +test("historical binding validation rejects reassignment, additions, duplicates, and malformed identities", () => { + const approved = LEGACY_FIRST_PARTY_SOURCE_ENTRIES; + const [first] = approved; + const last = approved.at(-1); + assert.ok(first); + assert.ok(last); + const replaceFirst = (entry: readonly [string, string]): readonly (readonly [string, string])[] => [ + entry, + ...approved.slice(1), + ]; + assert.throws( + () => validateLegacySourceBindings(replaceFirst(["amazon", "https://other.example/source"])), + MAPPING_CHANGED + ); + assert.throws( + () => validateLegacySourceBindings([...approved, ["new-connector", "https://other.example/source"] as const]), + MAPPING_CHANGED + ); + assert.throws(() => validateLegacySourceBindings([...approved, first]), DUPLICATE_BINDINGS); + assert.throws( + () => validateLegacySourceBindings([...approved, ["other-key", first[1]] as const]), + DUPLICATE_BINDINGS + ); + assert.throws( + () => validateLegacySourceBindings([...approved.slice(0, -1), ["bad/key", last[1]] as const]), + INVALID_BINDING_KEY + ); + assert.throws( + () => validateLegacySourceBindings([...approved.slice(0, -1), [last[0], "https://"] as const]), + INVALID_BINDING_SOURCE + ); +}); + test("every generated registry entry is a real, non-empty connector key with no duplicates", () => { for (const [name, value] of Object.entries(trackedRegistry)) { if (Array.isArray(value)) { diff --git a/reference-implementation/test/connector-source-identity-boundary.test.ts b/reference-implementation/test/connector-source-identity-boundary.test.ts new file mode 100644 index 000000000..371777fc4 --- /dev/null +++ b/reference-implementation/test/connector-source-identity-boundary.test.ts @@ -0,0 +1,1079 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Connector source identity at the registration boundary. + * + * A connector's Core source identity is the `source.id` of the SourceDeclaration + * the runtime retains for its connector key. Resolution order, on both write + * paths (`registerConnector` and the installer's `publishConnectorActivation`): + * + * 1. a retained declaration keeps its `source.id`; provenance cannot change it; + * 2. otherwise an explicit declaration is validated and used; + * 3. otherwise a first-party key takes the reviewed legacy source identity from + * the runtime-owned map (connector-legacy-source-map.ts), independent of the + * incoming provenance (`manifest_uri`) and alias (`connector_id`); + * 4. otherwise the registration is refused before any write. + * + * Covers both write paths. + */ + +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import test from "node:test"; +import { executeAsPolyfillConnectorRegister } from "../operations/as-polyfill-connector-register/index.ts"; +import { + __setRegisterConnectorPhaseHookForTest, + connectorDeclarationDigest, + getAcceptedDeclarationDigest, + getConnectorManifest, + normalizeConnectorManifestForStorage, + registerConnector, +} from "../server/auth.ts"; +import { + getConnectorActivation, + publishConnectorActivation, +} from "../server/connector-install/activation-authority.ts"; +import type { ConnectorInstallRecord } from "../server/connector-install/index.ts"; +import { + LEGACY_FIRST_PARTY_ENTRIES, + LEGACY_FIRST_PARTY_KEYS, + legacyFirstPartySourceIdFor, +} from "../server/connector-legacy-source-map.ts"; +import { validateConnectorManifest } from "../server/connector-manifest-validation.ts"; +import { closeDb, initDb } from "../server/db.ts"; +import { readPolyfillManifests } from "../server/polyfill-connectors-runtime.ts"; +import { closePostgresStorage, initPostgresStorage } from "../server/postgres-storage.ts"; +import { snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest } from "../server/source-declaration-legacy-collection.ts"; +import { withTemporaryPostgresDatabase } from "./helpers/postgres-temp-database.ts"; + +const PUBLISHER = "https://publishers.example/accepted"; +const DOC_ONE = "https://docs.example/manifests/one.json"; +const DOC_TWO = "https://docs.example/manifests/two.json"; +const EXPLICIT_SOURCE = "https://sources.example/explicit/identity-boundary"; +const OTHER_SOURCE = "https://sources.example/other/identity-boundary"; + +const INVALID_REQUEST = "invalid_request"; +const ALREADY_BOUND = /already bound to source/; +const CANONICAL_KEY_MESSAGE = /canonical key/; +const INVALID_SOURCE_DECLARATION = /Invalid SourceDeclaration/; +const KEY_MATCH_MESSAGE = /connector_id must match connector_key/; +const NO_SOURCE_MESSAGE = /no source declaration/; +const URL_KEY_MESSAGE = /is a URL with no connector_key/; + +/** The reviewed historical source identity of a shipped first-party connector. */ +function firstPartySource(slug: string): string { + return `https://registry.pdpp.dev/connectors/${slug}`; +} + +function streamFor(name: string) { + return { + name, + primary_key: ["id"], + schema: { + properties: { id: { type: "string" } }, + required: ["id"], + type: "object", + }, + selection: { fields: true, resources: true }, + semantics: "append_only", + }; +} + +function legacyManifest(overrides: Record): Record { + return { + capabilities: { human_interaction: [] }, + display_name: "Identity boundary", + protocol_version: "0.1.0", + streams: [streamFor("items")], + version: "1.0.0", + ...overrides, + }; +} + +/** A fully valid Core declaration, produced by the existing legacy projection. */ +function explicitDeclaration(connectorKey: string, sourceId: string, streams = [streamFor("items")]) { + return snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest( + legacyManifest({ + connector_id: connectorKey, + connector_key: connectorKey, + streams, + }), + { + connectorImplementationId: `https://implementations.example/connectors/${connectorKey}`, + publisherId: PUBLISHER, + sourceId, + } + ); +} + +function retainedSourceId(stored: Record | null): string | undefined { + const declaration = stored?.source_declaration as { source?: { id?: unknown } } | undefined; + return typeof declaration?.source?.id === "string" ? declaration.source.id : undefined; +} + +async function storedFor(connectorKey: string): Promise | null> { + return ((await getConnectorManifest(connectorKey)) as Record | null) ?? null; +} + +async function assertRejectedWithoutWrite( + connectorKey: string, + attempt: () => Promise, + messagePattern: RegExp +): Promise { + await assert.rejects(attempt, (error: unknown) => { + const err = error as { code?: unknown; message?: unknown }; + assert.equal(err.code, INVALID_REQUEST); + assert.match(String(err.message), messagePattern); + return true; + }); + assert.equal(await storedFor(connectorKey), null, "a rejected registration must not persist a manifest"); +} + +const SCENARIOS: Array<{ name: string; run: () => Promise | void }> = []; + +function scenario(name: string, run: () => Promise | void): void { + SCENARIOS.push({ name, run }); +} + +const PROVENANCE_VARIANTS: [string, Record][] = [ + ["provenance U1", { manifest_uri: DOC_ONE }], + ["provenance U2", { manifest_uri: DOC_TWO }], + ["no provenance", {}], +]; + +// Fresh first registration, unmapped custom key: provenance must not choose the source. +for (const [label, provenance] of PROVENANCE_VARIANTS) { + scenario(`a fresh custom key without a declaration is refused before any write (${label})`, async () => { + const key = "srcid-fresh-custom"; + await assertRejectedWithoutWrite( + key, + () => + registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + ...provenance, + }), + { + skipDerivedRepair: true, + } + ), + NO_SOURCE_MESSAGE + ); + }); +} + +// Fresh key-only registration (no connector_id), unmapped custom key: refused, no fallback. +scenario("a fresh key-only custom manifest without a declaration is refused before any write", async () => { + const key = "srcid-fresh-keyonly"; + await assertRejectedWithoutWrite( + key, + () => registerConnector(legacyManifest({ connector_key: key, manifest_uri: DOC_ONE }), { skipDerivedRepair: true }), + NO_SOURCE_MESSAGE + ); +}); + +// Unmapped registry-URL alias: a URL connector_id must not become the operational key or the source. +scenario("an unmapped registry URL connector_id is refused and never becomes a source", async () => { + const url = "https://registry.pdpp.dev/connectors/srcid-fresh-url"; + await assertRejectedWithoutWrite( + url, + () => registerConnector(legacyManifest({ connector_id: url, manifest_uri: DOC_ONE }), { skipDerivedRepair: true }), + URL_KEY_MESSAGE + ); +}); + +// Causal guard: an explicit, valid declaration does not make an unknown URL an operational key. +scenario("a URL-only connector_id with an explicit valid declaration is refused before any write", async () => { + const url = "https://sources.example/connectors/srcid-explicit-url"; + const key = "srcid-explicit-url"; + await assertRejectedWithoutWrite( + url, + () => + registerConnector( + legacyManifest({ + connector_id: url, + manifest_uri: DOC_ONE, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ), + URL_KEY_MESSAGE + ); + assert.equal(await storedFor(key), null, "the declared slug must not be registered either"); +}); + +// Known first-party key: the reviewed source identity wins over any incoming provenance. +for (const [label, provenance] of PROVENANCE_VARIANTS) { + scenario(`a first-party key takes its reviewed source identity regardless of provenance (${label})`, async () => { + const key = "amazon"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + ...provenance, + }), + { + skipDerivedRepair: true, + } + ); + assert.equal(retainedSourceId(await storedFor(key)), firstPartySource("amazon")); + }); +} + +// Known registry-URL alias with no connector_key resolves to its key and the same reviewed source. +scenario("a known registry URL alias resolves to its key and the reviewed source", async () => { + await registerConnector( + legacyManifest({ + connector_id: firstPartySource("gmail"), + manifest_uri: DOC_ONE, + }), + { skipDerivedRepair: true } + ); + const stored = await storedFor("gmail"); + assert.equal(stored?.connector_key, "gmail"); + assert.equal(retainedSourceId(stored), firstPartySource("gmail")); +}); + +// Known legacy local alias with its canonical key and no declaration resolves to the reviewed source. +scenario("a known legacy local alias with its canonical key takes the reviewed source", async () => { + await registerConnector( + legacyManifest({ + connector_id: "claude_code", + connector_key: "claude-code", + }), + { + skipDerivedRepair: true, + } + ); + assert.equal(retainedSourceId(await storedFor("claude-code")), firstPartySource("claude-code")); +}); + +// Control: an explicit validated declaration is the supported route for a first-party key too. +scenario("an explicit declaration wins over the reviewed mapping on a first registration", async () => { + const key = "amazon"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); +}); + +// Retained identity: a provenance-only update keeps the retained source (no fresh synthesis involved). +scenario("a provenance-only update keeps the retained source identity", async () => { + const key = "srcid-prov"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_TWO, + version: "1.1.0", + }), + { skipDerivedRepair: true } + ); + assert.equal( + retainedSourceId(await storedFor(key)), + EXPLICIT_SOURCE, + "manifest_uri must not replace the retained source" + ); +}); + +// Retained identity: a key-only update keeps the retained declaration. +scenario("a key-only update keeps the retained declaration", async () => { + const key = "srcid-keyonly"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + await registerConnector(legacyManifest({ connector_key: key, version: "1.1.0" }), { skipDerivedRepair: true }); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); +}); + +// Retained identity: an explicit different source identity cannot replace the retained one. +scenario("an explicit different source identity cannot replace the retained one", async () => { + const key = "srcid-replace"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + await assert.rejects( + () => + registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + source_declaration: explicitDeclaration(key, OTHER_SOURCE), + version: "1.1.0", + }), + { skipDerivedRepair: true } + ), + (error: unknown) => { + const err = error as { code?: unknown; message?: unknown }; + assert.equal(err.code, INVALID_REQUEST); + assert.match(String(err.message), ALREADY_BOUND); + return true; + } + ); + assert.equal( + retainedSourceId(await storedFor(key)), + EXPLICIT_SOURCE, + "the retained source must survive the rejected update" + ); +}); + +// Fail-before (explicit invalid): an explicit declaration is validated before it is retained. +scenario("an invalid explicit SourceDeclaration is rejected before persistence", async () => { + const key = "srcid-invalid"; + const invalid = { + ...explicitDeclaration(key, EXPLICIT_SOURCE), + protocol_version: "9.9.9", + }; + await assertRejectedWithoutWrite( + key, + () => + registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + source_declaration: invalid, + }), + { skipDerivedRepair: true } + ), + INVALID_SOURCE_DECLARATION + ); +}); + +// Control: a valid explicit declaration keeps its identity when provenance changes. +scenario("a valid explicit declaration keeps its source identity when provenance changes", async () => { + const key = "srcid-explicit"; + const declaration = explicitDeclaration(key, EXPLICIT_SOURCE); + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + source_declaration: declaration, + }), + { skipDerivedRepair: true } + ); + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_TWO, + source_declaration: declaration, + version: "1.1.0", + }), + { skipDerivedRepair: true } + ); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); +}); + +// Control: a legitimate schema update keeps the source and yields a new declaration version. +scenario("a schema update keeps the source identity and yields a new declaration version", async () => { + const key = "srcid-schema"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + const first = (await storedFor(key))?.source_declaration as { + declaration_version: string; + }; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + streams: [streamFor("items"), streamFor("more")], + version: "1.1.0", + }), + { skipDerivedRepair: true } + ); + const second = (await storedFor(key))?.source_declaration as { + declaration_version: string; + }; + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); + assert.notEqual(second.declaration_version, first.declaration_version); +}); + +// Control: the normalizer keeps a retained identity when given explicit context. +scenario("the normalizer keeps a retained identity when given it as context", () => { + const key = "srcid-context"; + const retained = explicitDeclaration(key, EXPLICIT_SOURCE); + const { storedManifest } = normalizeConnectorManifestForStorage( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_TWO, + }), + { retainedSourceDeclaration: retained } + ); + assert.equal(retainedSourceId(storedManifest), EXPLICIT_SOURCE); +}); + +// Control: an alias with a different explicit key is rejected (existing validator guard). +scenario("a known alias with a different explicit key is rejected", async () => { + await assertRejectedWithoutWrite( + "codex", + () => + registerConnector( + legacyManifest({ + connector_id: "claude_code", + connector_key: "codex", + }), + { + skipDerivedRepair: true, + } + ), + CANONICAL_KEY_MESSAGE + ); +}); + +// Control: an explicit key with an unmapped, unequal custom URI is rejected (existing validator guard). +scenario("an explicit key with an unmapped unequal custom URI is rejected", async () => { + await assertRejectedWithoutWrite( + "srcid-custom", + () => + registerConnector( + legacyManifest({ + connector_id: "https://example.org/manifest.json", + connector_key: "srcid-custom", + }), + { + skipDerivedRepair: true, + } + ), + KEY_MATCH_MESSAGE + ); +}); + +// Control: a known alias with its canonical explicit key is accepted when the source comes from an explicit declaration. +scenario("a known alias with its canonical explicit key is accepted", async () => { + await registerConnector( + legacyManifest({ + connector_id: "claude_code", + connector_key: "claude-code", + source_declaration: explicitDeclaration("claude-code", EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + const stored = await storedFor("claude-code"); + assert.equal(stored?.connector_key, "claude-code"); + assert.equal(retainedSourceId(stored), EXPLICIT_SOURCE); +}); + +// Control: an idempotent re-registration of the same content keeps everything. +scenario("an idempotent re-registration keeps the retained identity", async () => { + const key = "srcid-idem"; + const manifest = legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }); + await registerConnector(manifest, { skipDerivedRepair: true }); + await registerConnector(manifest, { skipDerivedRepair: true }); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); +}); + +// Concurrency: two registrations that name different explicit sources cannot both commit. +scenario("concurrent registrations of different explicit sources commit exactly one", async () => { + const key = "srcid-race"; + const results = await Promise.allSettled([ + registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ), + registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, OTHER_SOURCE), + }), + { skipDerivedRepair: true } + ), + ]); + const fulfilled = results.filter((result) => result.status === "fulfilled"); + assert.equal(fulfilled.length, 1, "exactly one registration may commit"); + const winner = retainedSourceId(await storedFor(key)); + assert.ok(winner === EXPLICIT_SOURCE || winner === OTHER_SOURCE); +}); + +interface Deferred { + readonly promise: Promise; + readonly resolve: () => void; +} + +function deferred(): Deferred { + let resolve: () => void = () => undefined; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +interface ParkedWriter { + readonly arrived: Deferred; + readonly released: Deferred; +} + +/** + * Parks each registration of `connectorKey` at the stale-read point (after it observed the retained identity, + * before it persists) until the test releases it. Parking is keyed by the source the writer intends. + */ +function parkRegistrationsAtStaleRead( + connectorKey: string, + sourceIds: readonly string[] +): { + readonly gate: (sourceId: string) => ParkedWriter; + readonly release: (sourceId: string) => void; + readonly reset: () => void; +} { + const gates = new Map( + sourceIds.map((sourceId) => [sourceId, { arrived: deferred(), released: deferred() }]) + ); + const gate = (sourceId: string): ParkedWriter => { + const found = gates.get(sourceId); + if (!found) { + throw new Error(`no parked writer for ${sourceId}`); + } + return found; + }; + __setRegisterConnectorPhaseHookForTest(async (point, context) => { + if (point !== "before-manifest-persisted" || context.connectorId !== connectorKey) { + return; + } + const sourceId = retainedSourceId(context.manifest as Record); + const writer = sourceId === undefined ? undefined : gates.get(sourceId); + if (!writer) { + return; + } + writer.arrived.resolve(); + await writer.released.promise; + }); + return { + gate, + release: (sourceId) => gate(sourceId).released.resolve(), + reset: () => { + __setRegisterConnectorPhaseHookForTest(null); + for (const writer of gates.values()) { + writer.released.resolve(); + } + }, + }; +} + +/** + * Resolves when the writer reaches its stale-read point. Fails if the writer settles first: then the + * interleaving was not replayed. Racing (not awaiting) keeps a late settlement from becoming an unhandled rejection. + */ +function parkedBefore(arrived: Promise, writer: Promise): Promise { + return Promise.race([ + arrived, + writer.then(() => { + throw new Error("a writer settled before the stale-read point: the interleaving was not replayed"); + }), + ]); +} + +function explicitRegistration(key: string, sourceId: string, extra: Record = {}) { + return registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, sourceId), + ...extra, + }), + { skipDerivedRepair: true } + ); +} + +// Causal replay of the unlocked interleaving (source-identity-boundary-r5 model, Locked = FALSE): both writers read +// "no retained identity" before either persists. The commit-time check must keep the first commit and refuse the +// second, leaving the registry row byte-identical. +scenario( + "two writers that both read no retained identity: the commit-time check keeps the first and refuses the second", + async () => { + const key = "srcid-race-replay"; + const park = parkRegistrationsAtStaleRead(key, [EXPLICIT_SOURCE, OTHER_SOURCE]); + const writerA = explicitRegistration(key, EXPLICIT_SOURCE); + const writerB = explicitRegistration(key, OTHER_SOURCE); + try { + await Promise.all([ + parkedBefore(park.gate(EXPLICIT_SOURCE).arrived.promise, writerA), + parkedBefore(park.gate(OTHER_SOURCE).arrived.promise, writerB), + ]); + park.release(EXPLICIT_SOURCE); + await writerA; + const afterA = JSON.stringify(await storedFor(key)); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); + + park.release(OTHER_SOURCE); + await assert.rejects(writerB, (error: unknown) => { + const err = error as { code?: unknown; message?: unknown }; + assert.equal(err.code, INVALID_REQUEST); + assert.match(String(err.message), ALREADY_BOUND); + return true; + }); + assert.equal(JSON.stringify(await storedFor(key)), afterA, "the refused writer must not change the registry row"); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE, "the first writer's source survives"); + assert.equal(await getConnectorActivation(key), null, "no activation is written by a registration"); + } finally { + park.reset(); + await Promise.allSettled([writerA, writerB]); + } + } +); + +// Installer publish races a registration that read no identity. The registration resumes last and must be refused. +scenario( + "a registration that read no identity before an installer publish is refused and the installer source survives", + async () => { + const key = "srcid-race-install"; + const park = parkRegistrationsAtStaleRead(key, [EXPLICIT_SOURCE]); + const registration = explicitRegistration(key, EXPLICIT_SOURCE); + try { + await parkedBefore(park.gate(EXPLICIT_SOURCE).arrived.promise, registration); + await publishConnectorActivation( + installRecord( + key, + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, OTHER_SOURCE), + }) + ) + ); + park.release(EXPLICIT_SOURCE); + await assert.rejects(registration, (error: unknown) => { + const err = error as { code?: unknown; message?: unknown }; + assert.equal(err.code, INVALID_REQUEST); + assert.match(String(err.message), ALREADY_BOUND); + return true; + }); + assert.equal(retainedSourceId(await storedFor(key)), OTHER_SOURCE, "the installer's source is retained"); + const activation = await getConnectorActivation(key); + assert.equal(retainedSourceId(activation?.canonicalManifest ?? null), OTHER_SOURCE); + } finally { + park.reset(); + await Promise.allSettled([registration]); + } + } +); + +// A seeded retained identity that differs from the reviewed map must survive a schema/provenance update through the writer. +scenario("a retained custom source for a mapped key survives a schema update without a declaration", async () => { + const key = "amazon"; + await explicitRegistration(key, EXPLICIT_SOURCE); + const before = (await storedFor(key))?.source_declaration as { declaration_version: string } | undefined; + assert.notEqual(EXPLICIT_SOURCE, firstPartySource("amazon"), "the seed must differ from the reviewed map"); + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_TWO, + streams: [streamFor("items"), streamFor("more")], + version: "1.1.0", + }), + { skipDerivedRepair: true } + ); + const stored = await storedFor(key); + const after = stored?.source_declaration as { + declaration_version: string; + publisher: { id: string }; + streams: { name: string }[]; + }; + assert.equal(retainedSourceId(stored), EXPLICIT_SOURCE, "the retained custom source, not the map, is kept"); + assert.equal(after.publisher.id, PUBLISHER, "the retained publisher is kept"); + assert.deepEqual( + after.streams.map((stream) => stream.name), + ["items", "more"], + "the new schema is the one stored" + ); + assert.equal(stored?.version, "1.1.0"); + assert.notEqual(after.declaration_version, before?.declaration_version); + assert.equal(await getConnectorActivation(key), null); +}); + +// Pure lookup: prototype names and unmapped names never resolve, and no key can be repointed through the exports. +scenario("prototype and unmapped names never resolve to a reviewed source", async () => { + for (const name of ["constructor", "__proto__", "toString", "hasOwnProperty", "valueOf", "srcid-not-mapped"]) { + assert.equal(legacyFirstPartySourceIdFor(name), undefined, `${name} must not resolve`); + } + await assertRejectedWithoutWrite( + "constructor", + () => + registerConnector(legacyManifest({ connector_id: "constructor", connector_key: "constructor" }), { + skipDerivedRepair: true, + }), + NO_SOURCE_MESSAGE + ); +}); + +// Mutation control: the exported map views are frozen; a mutation attempt is refused and the lookup is unchanged. +scenario("exported key, entry and tuple views cannot be mutated to repoint a lookup", () => { + assert.equal(Object.isFrozen(LEGACY_FIRST_PARTY_KEYS), true); + assert.equal(Object.isFrozen(LEGACY_FIRST_PARTY_ENTRIES), true); + const [firstKey, firstSource] = LEGACY_FIRST_PARTY_ENTRIES[0] ?? []; + assert.ok(firstKey && firstSource); + const tuple = LEGACY_FIRST_PARTY_ENTRIES[0] as unknown as object; + assert.equal(Reflect.set(tuple, 1, OTHER_SOURCE), false, "a frozen entry tuple refuses assignment"); + assert.equal(Reflect.set(LEGACY_FIRST_PARTY_ENTRIES as unknown as object, 0, [firstKey, OTHER_SOURCE]), false); + assert.equal(Reflect.set(LEGACY_FIRST_PARTY_KEYS as unknown as object, 0, "constructor"), false); + assert.equal(legacyFirstPartySourceIdFor(firstKey), firstSource, "the lookup is unchanged"); + assert.equal(legacyFirstPartySourceIdFor("amazon"), firstPartySource("amazon")); +}); + +// Null and undefined mean "unspecified" (documented normalizer behavior), never a request for a map fallback. +async function assertAbsenceDeclaration(declaration: null | undefined): Promise { + await registerConnector( + legacyManifest({ connector_id: "amazon", connector_key: "amazon", source_declaration: declaration }), + { skipDerivedRepair: true } + ); + assert.equal(retainedSourceId(await storedFor("amazon")), firstPartySource("amazon")); + await assertRejectedWithoutWrite( + "srcid-null-unmapped", + () => + registerConnector( + legacyManifest({ + connector_id: "srcid-null-unmapped", + connector_key: "srcid-null-unmapped", + source_declaration: declaration, + }), + { skipDerivedRepair: true } + ), + NO_SOURCE_MESSAGE + ); +} + +scenario( + "a null or undefined declaration is absence: mapped keys take the map, unmapped keys refuse, retained keys keep", + async () => { + await assertAbsenceDeclaration(null); + await assertAbsenceDeclaration(undefined); + const key = "srcid-null-retained"; + await explicitRegistration(key, EXPLICIT_SOURCE); + await registerConnector( + legacyManifest({ connector_id: key, connector_key: key, source_declaration: null, version: "1.1.0" }), + { skipDerivedRepair: true } + ); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); + } +); + +// Malformed nonnull declarations refuse before any write, and never fall back to the reviewed map or provenance. +for (const [label, malformed] of [ + ["an empty object", {}], + ["a string", "declaration"], + ["a number", 42], + ["a boolean", true], + ["an array", []], +] as const) { + scenario(`${label} as a declaration is refused before write on a mapped key (no map fallback)`, async () => { + await assertRejectedWithoutWrite( + "amazon", + () => + registerConnector( + legacyManifest({ connector_id: "amazon", connector_key: "amazon", source_declaration: malformed }), + { + skipDerivedRepair: true, + } + ), + INVALID_SOURCE_DECLARATION + ); + }); +} + +/** The POST /connectors operation, wired with the same dependencies the server mounts. */ +function postRegister(manifest: Record) { + return executeAsPolyfillConnectorRegister( + { manifest }, + { + declarationDigest: connectorDeclarationDigest, + getAcceptedDeclarationDigest, + getConnectorManifest, + normalizeManifestForStorage: normalizeConnectorManifestForStorage, + registerConnector: (stored: Record) => registerConnector(stored, { skipDerivedRepair: true }), + validateManifest: (stored: Record) => validateConnectorManifest(stored), + } + ); +} + +// POST control: an identical re-POST of a retained connector is idempotent and keeps its source. +scenario("an identical POST to a retained connector is idempotent and keeps its source", async () => { + const key = "srcid-post-idem"; + const manifest = legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }); + await registerConnector(manifest, { skipDerivedRepair: true }); + const outcome = await postRegister(manifest); + assert.equal(outcome.outcome, "success", JSON.stringify(outcome)); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); +}); + +// POST control: a first-party identical re-POST with no declaration is idempotent under the reviewed source. +scenario("an identical POST of a first-party connector without a declaration is idempotent", async () => { + const manifest = legacyManifest({ + connector_id: "amazon", + connector_key: "amazon", + manifest_uri: DOC_ONE, + }); + const first = await postRegister(manifest); + assert.equal(first.outcome, "success", JSON.stringify(first)); + const second = await postRegister(manifest); + assert.equal(second.outcome, "success", JSON.stringify(second)); + assert.equal(retainedSourceId(await storedFor("amazon")), firstPartySource("amazon")); +}); + +// POST: a version update without a declaration keeps the retained source, whatever provenance it names. +scenario("a POST version update without a declaration keeps the retained source", async () => { + const key = "srcid-post-update"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + const outcome = await postRegister( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_TWO, + version: "1.1.0", + }) + ); + assert.equal(outcome.outcome, "success", JSON.stringify(outcome)); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); +}); + +// POST by a known registry alias: an identical re-POST compares against the retained identity of its canonical key. +// A lookup under the raw alias misses the retained row, takes the reviewed fresh-path source, and reports +// equivocation for content the server already accepted. +scenario( + "an identical POST by a known registry alias to a retained connector is idempotent and keeps its source", + async () => { + const key = "amazon"; + const alias = `https://registry.pdpp.dev/connectors/${key}`; + // The retained declaration is the one a re-projection of this provenance produces (implementation id = manifest_uri), + // so an identical re-POST can be compared by digest. + const retained = snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest( + legacyManifest({ connector_id: key, connector_key: key }), + { connectorImplementationId: alias, publisherId: PUBLISHER, sourceId: EXPLICIT_SOURCE } + ); + await registerConnector( + legacyManifest({ connector_id: key, connector_key: key, manifest_uri: alias, source_declaration: retained }), + { skipDerivedRepair: true } + ); + const outcome = await postRegister(legacyManifest({ connector_id: alias, manifest_uri: alias })); + assert.equal(outcome.outcome, "success", JSON.stringify(outcome)); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); + } +); + +/** An installer record whose manifest is the given one; only the identity fields matter here. */ +function installRecord(connectorKey: string, manifest: Record): ConnectorInstallRecord { + const hex = (digit: string) => `sha256:${digit.repeat(64)}`; + return { + activatedAt: "2026-11-02T00:00:00.000Z", + bindings: {}, + configDigest: hex("0"), + connectorId: connectorKey, + digest: hex("1"), + entrypointPath: "/app/entrypoint.js", + entrypointSha256: hex("2"), + manifest, + manifestPath: "/app/manifest.json", + manifestSha256: hex("3"), + provenancePath: "/app/provenance.json", + provenanceSha256: hex("4"), + registry: "ghcr.io", + repository: `ghcr.io/example/connectors/${connectorKey}`, + root: `/var/lib/pdpp/${connectorKey}`, + tier: null, + version: "1.0.0", + }; +} + +// Installer: a retained source survives a provenance-only install. +scenario("an installer publish keeps the retained source identity across a provenance change", async () => { + const key = "srcid-install-prov"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + await publishConnectorActivation( + installRecord( + key, + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_TWO, + }) + ) + ); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE, "the registry keeps the retained source"); + const activation = await getConnectorActivation(key); + assert.equal( + retainedSourceId(activation?.canonicalManifest ?? null), + EXPLICIT_SOURCE, + "the activation records the same source" + ); +}); + +// Installer: an explicit different identity is rejected before any write. +scenario("an installer publish cannot rebind a retained source to an explicit different one", async () => { + const key = "srcid-install-replace"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + await assert.rejects( + () => + publishConnectorActivation( + installRecord( + key, + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + source_declaration: explicitDeclaration(key, OTHER_SOURCE), + }) + ) + ), + (error: unknown) => { + const err = error as { code?: unknown; message?: unknown }; + assert.equal(err.code, INVALID_REQUEST); + assert.match(String(err.message), ALREADY_BOUND); + return true; + } + ); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE, "the registry keeps the retained source"); + assert.equal(await getConnectorActivation(key), null, "no activation is written for a rejected publish"); +}); + +// Installer, fresh: an unmapped custom key with provenance is refused before any registry or activation write. +scenario("an installer publish refuses a fresh custom key without a declaration", async () => { + const key = "srcid-install-fresh"; + await assert.rejects( + () => + publishConnectorActivation( + installRecord( + key, + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + }) + ) + ), + (error: unknown) => { + const err = error as { code?: unknown; message?: unknown }; + assert.equal(err.code, INVALID_REQUEST); + assert.match(String(err.message), NO_SOURCE_MESSAGE); + return true; + } + ); + assert.equal(await storedFor(key), null, "no registry row is written"); + assert.equal(await getConnectorActivation(key), null, "no activation is written"); +}); + +// The reviewed map is the proof for fresh first-party registrations: it must equal the pinned package +// identities exactly. A package change that adds, removes or re-points a shipped key fails here, so the +// map is reviewed rather than extended silently. +test("the reviewed legacy source map equals the pinned first-party package identities", () => { + const pinned = readPolyfillManifests(); + const byKey = new Map(pinned.map((entry) => [entry.manifest.connector_key, entry.manifest])); + assert.equal(pinned.length, 45, "the pinned package ships 45 first-party manifests"); + assert.deepEqual([...LEGACY_FIRST_PARTY_KEYS].sort(), [...byKey.keys()].sort()); + const canonicalPairs = [...LEGACY_FIRST_PARTY_ENTRIES].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + const mappingPin = createHash("sha256").update(JSON.stringify(canonicalPairs), "utf8").digest("hex"); + assert.equal(mappingPin, "7e8f726725b7a14705571339c84a52d8f05888ffc0996b77cce3d4ae13dae9e6"); + for (const [key, sourceId] of LEGACY_FIRST_PARTY_ENTRIES) { + const manifest = byKey.get(key); + assert.ok(manifest, `pinned manifest for ${key}`); + assert.equal(manifest.connector_id, sourceId, `${key}: connector_id is the reviewed source`); + assert.equal(manifest.manifest_uri, sourceId, `${key}: manifest_uri is the reviewed source`); + assert.equal(manifest.source_declaration, undefined, `${key}: the pinned manifest declares no source`); + } +}); + +for (const entry of SCENARIOS) { + test(`SQLite: ${entry.name}`, async () => { + initDb(":memory:"); + try { + await entry.run(); + } finally { + closeDb(); + } + }); + + test(`PostgreSQL: ${entry.name} (skipped: PDPP_TEST_POSTGRES_URL unset)`, { + skip: !process.env.PDPP_TEST_POSTGRES_URL, + }, async () => { + const url = process.env.PDPP_TEST_POSTGRES_URL; + assert.ok(url); + await withTemporaryPostgresDatabase( + { + closeConnections: closePostgresStorage, + connectionString: url, + databaseName: `pdpp_test_srcid_${Date.now().toString(36)}_${Math.random().toString(16).slice(2, 10)}_1`, + templateName: null, + }, + async (databaseUrl) => { + initDb(":memory:"); + try { + await initPostgresStorage({ backend: "postgres", databaseUrl }); + await entry.run(); + } finally { + await closePostgresStorage(); + closeDb(); + } + } + ); + }); +} diff --git a/reference-implementation/test/control-actions.test.ts b/reference-implementation/test/control-actions.test.ts index 4283db924..1bb6fc190 100644 --- a/reference-implementation/test/control-actions.test.ts +++ b/reference-implementation/test/control-actions.test.ts @@ -392,7 +392,7 @@ async function emitSyntheticRun({ actor_id: connectorId, actor_type: "runtime", data: { - bindings: { filesystem: {}, interactive: {}, network: {} }, + bindings: { browser: {}, filesystem: {}, network: {} }, boot_epoch: _epoch.boot_epoch, collection_mode: "incremental", connector_instance_id: connectorInstanceId, @@ -478,7 +478,7 @@ async function emitSyntheticStartedRun({ actor_id: connectorId, actor_type: "runtime", data: { - bindings: { filesystem: {}, interactive: {}, network: {} }, + bindings: { browser: {}, filesystem: {}, network: {} }, boot_epoch: _epoch.boot_epoch, collection_mode: "incremental", connection_id: connectorInstanceId, @@ -1051,7 +1051,7 @@ test("controller startup reconciles abandoned controller-managed runs after rest actor_id: connectorId, actor_type: "runtime", data: { - bindings: { filesystem: {}, interactive: {}, network: {} }, + bindings: { browser: {}, filesystem: {}, network: {} }, // Synthetic "prior incarnation" stamp — this test seeds the // pre-crash state of an interrupted run. // diff --git a/reference-implementation/test/detail-coverage-shortfall-severity.test.ts b/reference-implementation/test/detail-coverage-shortfall-severity.test.ts index ea19e9967..2b5c22572 100644 --- a/reference-implementation/test/detail-coverage-shortfall-severity.test.ts +++ b/reference-implementation/test/detail-coverage-shortfall-severity.test.ts @@ -43,12 +43,12 @@ import { join } from "node:path"; // carrying a durable DETAIL_GAP. import test from "node:test"; import { type RuntimeRunConnectorResult, runConnector } from "../runtime/index.ts"; - import { getDb } from "../server/db.ts"; import { startServer } from "../server/index.ts"; import { createRequestConnectorInstanceStore } from "../server/request-store-factories.ts"; import { getDefaultConnectorDetailGapStore } from "../server/stores/connector-detail-gap-store.ts"; import { admitOwnerRunConnection } from "../server/stores/connector-instance-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; // Hoisted: the coverage-shortfall explanation the runtime authors, and the // connection-scoped denominator the sibling-isolation oracle asserts on. @@ -188,49 +188,56 @@ async function issueOwnerToken(asUrl: string): Promise { // Connector-agnostic shape of the live failure: one parent state_stream // ("holdings") whose key fans out to two child detail streams. -const MANIFEST = { - connector_id: "coverage-shortfall-regression", - display_name: "Coverage Shortfall Regression", - manifest_uri: "https://registry.pdpp.dev/connectors/coverage-shortfall-regression", - protocol_version: "0.1.0", - streams: [ - { - name: "holdings", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - { - name: "activity", - primary_key: ["id"], - relationships: [{ cardinality: "has_one", foreign_key: "holding_id", name: "holding", stream: "holdings" }], - schema: { - properties: { holding_id: { type: "string" }, id: { type: "string" } }, - required: ["id"], - type: "object", +const MANIFEST = withFixtureSourceDeclaration( + { + connector_id: "coverage-shortfall-regression", + display_name: "Coverage Shortfall Regression", + manifest_uri: "https://registry.pdpp.dev/connectors/coverage-shortfall-regression", + protocol_version: "0.1.0", + streams: [ + { + name: "holdings", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - { - coverage_strategy: "parent_detail_accounting", - name: "valuations", - parent_streams: ["holdings", "activity"], - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", -}; + { + name: "activity", + primary_key: ["id"], + relationships: [{ cardinality: "has_one", foreign_key: "holding_id", name: "holding", stream: "holdings" }], + schema: { + properties: { holding_id: { type: "string" }, id: { type: "string" } }, + required: ["id"], + type: "object", + }, + selection: { fields: true, resources: true }, + semantics: "append_only", + }, + { + coverage_strategy: "parent_detail_accounting", + name: "valuations", + parent_streams: ["holdings", "activity"], + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "append_only", + }, + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/coverage-shortfall-regression" } +); /** - * The same connector, declaring DETAIL_COVERAGE_KEY_SETS. Only a connector that - * declares it gets evidence-based crediting of hydrated keys; MANIFEST keeps - * the crediting it had before the capability existed. The capability is new in - * Collection Profile 0.2.0, so the manifest declares that version (Section 3.1). + * The same connector, declaring DETAIL_COVERAGE_KEY_SETS. The manifest + * declaration alone makes a hydrated key need an accepted detail record for it: + * the runtime derives that requirement from the manifest, not from START. The + * canned connectors below never see the extension in START, so they follow the + * base hydrated_keys fallback. MANIFEST, which does not declare the extension, + * credits a hydrated key by assertion. The runtime's manifest validation + * requires protocol_version 0.2.0 for a manifest that declares the extension. + * That is a package rule, not canonical Collection Profile text. */ const KEY_SETS_MANIFEST = { ...MANIFEST, @@ -1273,8 +1280,9 @@ test("a 0.1.0 manifest keeps legacy crediting and cannot declare a 0.2.0 capabil // Pre-spawn gate (Sections 3.4 and 4): an unknown capability name fails closed, // and a known required capability this runtime does not implement fails before -// any connector code runs. DETAIL_COVERAGE_KEY_SETS is advisory and -// STREAM_EVIDENCE is implemented, so both pass. BLOB is covered separately. +// any connector code runs. DETAIL_COVERAGE_KEY_SETS is advisory, and OBSERVATION +// and STREAM_EVIDENCE are implemented, so all three pass. BLOB is required and +// unimplemented, so it is refused here before spawn. test("the pre-spawn gate rejects unknown and unsupported required capabilities and admits implemented ones", async (t) => { const server = await typedStartServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); t.after(() => closeServer(server)); @@ -1291,7 +1299,7 @@ test("the pre-spawn gate rejects unknown and unsupported required capabilities a ["DETAIL_COVERAGE_KEY_SETS", UNKNOWN_CAPABILITY_PATTERN, null], [null, UNKNOWN_CAPABILITY_PATTERN, null], [["OBSERVATION"], null, null], - [["DETAIL_COVERAGE_KEY_SETS", "OBSERVATION", "BLOB"], null, null], + [["DETAIL_COVERAGE_KEY_SETS", "OBSERVATION", "BLOB"], UNSUPPORTED_CAPABILITY_PATTERN, "BLOB"], [["DETAIL_COVERAGE_KEY_SETS", "STREAM_EVIDENCE"], null, null], ]; for (const [protocolCapabilities, rejection, unsupported] of cases) { @@ -1317,16 +1325,17 @@ test("the pre-spawn gate rejects unknown and unsupported required capabilities a } }); -// START.protocol_capabilities lists what this runtime implements (Section -// 5.1): it accepts empty_keys/unchanged_keys and it validates STREAM_EVIDENCE. -// It also accepts OBSERVATION (Section 5.10). It does not implement BLOB, so it -// does not list it. +// START.protocol_capabilities lists what this runtime implements (Section 5.1): +// OBSERVATION (Section 5.10) and STREAM_EVIDENCE (Section 5.7). It does not list +// the advisory DETAIL_COVERAGE_KEY_SETS, because it does not implement the +// extension's crediting, so a connector falls back to hydrated_keys. BLOB is +// unimplemented and never listed. test("START advertises the protocol capabilities this runtime implements, to every connector", async (t) => { const server = await typedStartServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); t.after(() => closeServer(server)); const ownerToken = await registerAndAuthorize(server); - for (const manifest of [MANIFEST, KEY_SETS_MANIFEST]) { - // biome-ignore lint/performance/noAwaitInLoops: each manifest is one isolated run. + await [MANIFEST, KEY_SETS_MANIFEST].reduce(async (previous, manifest) => { + await previous; const { start, thrown } = await runCapturingStart( server, ownerToken, @@ -1334,33 +1343,40 @@ test("START advertises the protocol capabilities this runtime implements, to eve holdingsRun([holdingsCoverage({ empty_keys: ["K"], required_keys: ["K"] })]) ); assert.equal(thrown, null); - assert.deepEqual(start?.protocol_capabilities, ["DETAIL_COVERAGE_KEY_SETS", "OBSERVATION", "STREAM_EVIDENCE"]); - } + assert.deepEqual(start?.protocol_capabilities, ["OBSERVATION", "STREAM_EVIDENCE"]); + }, Promise.resolve()); }); -// BLOB is a runtime-specific extension (Collection Profile Section 7) that this -// runtime does not implement. chatgpt and anthropic declare it but emit BLOB -// only for an item over the inline cap, so a declaring connector still starts -// here. A run that emits no BLOB completes; a run that emits one fails at that -// line, with the same error as before the capability gate existed. -test("a 0.1.0 manifest declaring BLOB starts; it completes without BLOB and fails at a BLOB line", async (t) => { +// BLOB is required by a manifest that declares it, and this runtime does not +// implement it. The gate refuses such a manifest before spawn, in 0.1.0 and 0.2.0, +// whether or not the connector would emit a BLOB line. This replaces the former +// spawn waiver. A connector that did not declare BLOB and emits one is rejected +// at that line, as an unknown message type. +const BLOB_REFUSED_PATTERN = /requires protocol capabilities this runtime does not support: BLOB$/; + +test("a manifest declaring BLOB is refused before spawn at 0.1.0 and 0.2.0, even when no BLOB would be emitted", async (t) => { const server = await typedStartServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); t.after(() => closeServer(server)); const ownerToken = await registerAndAuthorize(server); - const blobManifest = { ...MANIFEST, protocol_capabilities: ["BLOB"], version: "1.2.0" }; - assert.equal(isAdmitted(await registerManifest(server, blobManifest)), true, "the BLOB manifest is admitted"); + const blobManifest010 = { ...MANIFEST, protocol_capabilities: ["BLOB"], version: "1.2.0" }; + const blobManifest020 = { ...KEY_SETS_MANIFEST, protocol_capabilities: ["BLOB"], version: "1.2.1" }; + await [blobManifest010, blobManifest020].reduce(async (previous, blobManifest) => { + await previous; + const refused = await runCapturingStart( + server, + ownerToken, + blobManifest, + holdingsRun([holdingsCoverage({ empty_keys: ["K"], required_keys: ["K"] })]) + ); + assert.equal(refused.start, null, `${blobManifest.protocol_version} BLOB manifest never reaches the connector`); + assert.match(String((refused.thrown as Error | null)?.message), BLOB_REFUSED_PATTERN); + }, Promise.resolve()); +}); - const clean = await runCapturingStart( - server, - ownerToken, - blobManifest, - holdingsRun([holdingsCoverage({ empty_keys: ["K"], required_keys: ["K"] })]) - ); - assert.equal(clean.thrown, null); - assert.ok(clean.start, "the connector received START"); - assert.equal(clean.start?.protocol_capabilities instanceof Array, true); - assert.equal((clean.start?.protocol_capabilities as string[]).includes("BLOB"), false, "START does not offer BLOB"); - assert.equal(clean.result?.status, "succeeded"); +test("a connector that emits BLOB without declaring it is rejected at that line as an unknown message", async (t) => { + const server = await typedStartServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); + t.after(() => closeServer(server)); + const ownerToken = await registerAndAuthorize(server); const blobLine = { file: "blob-1.json", @@ -1371,7 +1387,7 @@ test("a 0.1.0 manifest declaring BLOB starts; it completes without BLOB and fail stream: "holdings", type: "BLOB", }; - const withBlob = await runCapturingStart(server, ownerToken, blobManifest, [blobLine, ...holdingsRun([])]); + const withBlob = await runCapturingStart(server, ownerToken, MANIFEST, [blobLine, ...holdingsRun([])]); assert.ok(withBlob.start, "the connector received START"); const failure = withBlob.thrown instanceof Error ? withBlob.thrown.message : JSON.stringify(withBlob.result); assert.match(failure, UNKNOWN_BLOB_MESSAGE_PATTERN); diff --git a/reference-implementation/test/event-spine.test.ts b/reference-implementation/test/event-spine.test.ts index 40e60dd04..373b7cf49 100644 --- a/reference-implementation/test/event-spine.test.ts +++ b/reference-implementation/test/event-spine.test.ts @@ -30,6 +30,7 @@ import { startServer as startServerUntyped } from "../server/index.ts"; import { ingestRecord as ingestRecordUntyped } from "../server/records.ts"; import { createRequestConnectorInstanceStore } from "../server/request-store-factories.ts"; import { admitOwnerRunConnection } from "../server/stores/connector-instance-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; // Real ingest resolves the acting owner subject from the request's bearer // token (`getOwnerTokenSubjectId` in server/index.ts), independent of @@ -2096,7 +2097,6 @@ test("event spine", async (t) => { assert.deepEqual(startedEvent.data.bindings, { browser: {}, filesystem: {}, - interactive: {}, network: {}, }); assert.deepEqual(startedEvent.data.scope, { @@ -2190,8 +2190,11 @@ rl.on('line', (line) => { }); const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; - const manifest = { + const manifest = withFixtureSourceDeclaration({ connector_id: "event-spine-multi-stream-checkpoint-test", + connector_key: "event-spine-multi-stream-checkpoint-test", + display_name: "Event Spine Multi-Stream Checkpoint Test", + protocol_version: "0.1.0", streams: [ { name: "items", @@ -2204,6 +2207,8 @@ rl.on('line', (line) => { required: ["id"], type: "object", }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", }, { name: "other_items", @@ -2216,22 +2221,11 @@ rl.on('line', (line) => { required: ["id"], type: "object", }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", }, ], version: "0.1.0", - }; - Object.assign(manifest, { - source_declaration: { - declaration_version: "event-spine-multi-stream-checkpoint-test.v1", - display: { name: "Event Spine Multi-Stream Checkpoint Test" }, - protocol_version: "0.1.0", - publisher: { id: "https://pdpp.dev/reference-implementation/tests" }, - source: { - id: "https://registry.pdpp.dev/connectors/event-spine-multi-stream-checkpoint-test", - kind: "connector", - }, - streams: manifest.streams, - }, }); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-event-spine-multi-stream-commit-")); @@ -2333,8 +2327,10 @@ rl.on('line', (line) => { rsPort: 0, }); const asUrl = `http://localhost:${server.asPort}`; - const manifest = { - connector_id: "https://registry.pdpp.dev/connectors/event-spine-partial-checkpoint-failure-test", + const manifest = withFixtureSourceDeclaration({ + connector_id: "event-spine-partial-checkpoint-failure-test", + connector_key: "event-spine-partial-checkpoint-failure-test", + display_name: "Event Spine Partial Checkpoint Failure Test", protocol_version: "0.1.0", streams: [ { @@ -2348,6 +2344,8 @@ rl.on('line', (line) => { required: ["id"], type: "object", }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", }, { name: "other_items", @@ -2360,22 +2358,11 @@ rl.on('line', (line) => { required: ["id"], type: "object", }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", }, ], version: "0.1.0", - }; - Object.assign(manifest, { - source_declaration: { - declaration_version: "event-spine-partial-checkpoint-failure-test.v1", - display: { name: "Event Spine Partial Checkpoint Failure Test" }, - protocol_version: "0.1.0", - publisher: { id: "https://pdpp.dev/reference-implementation/tests" }, - source: { - id: manifest.connector_id, - kind: "connector", - }, - streams: manifest.streams, - }, }); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-event-spine-partial-checkpoint-failure-")); @@ -4730,7 +4717,15 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId: spotifyManifest.connector_id, connectorPath, - manifest: spotifyManifest, + // The connector emits an otp INTERACTION. The seed declares only + // `credentials`, so this run's manifest also declares `otp`. + manifest: { + ...spotifyManifest, + capabilities: { + ...((spotifyManifest.capabilities as Record | undefined) ?? {}), + human_interaction: ["credentials", "otp"], + }, + }, onInteraction: async () => new Promise(() => { /* intentionally waits for the page to advance */ diff --git a/reference-implementation/test/helpers/checkpoint-dependency-multi-parent-scenario.ts b/reference-implementation/test/helpers/checkpoint-dependency-multi-parent-scenario.ts index a82acddca..0838692e8 100644 --- a/reference-implementation/test/helpers/checkpoint-dependency-multi-parent-scenario.ts +++ b/reference-implementation/test/helpers/checkpoint-dependency-multi-parent-scenario.ts @@ -18,6 +18,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { runConnector as RunConnectorType } from "../../runtime/index.ts"; +import { withFixtureSourceDeclaration } from "./fixture-source-declaration.ts"; type RunConnectorFn = typeof RunConnectorType; @@ -91,39 +92,42 @@ export function fakeAdmitRunConnection( } export function multiParentManifest(connectorId: string) { - return { - capabilities: { human_interaction: [] }, - connector_id: connectorId, - display_name: connectorId, - manifest_uri: `https://sources.example/${connectorId}`, - protocol_version: "0.1.0", - streams: [ - { - name: "holdings", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - name: "activity", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - coverage_strategy: "parent_detail_accounting", - name: "valuations", - parent_streams: ["holdings", "activity"], - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "0.1.0", - }; + return withFixtureSourceDeclaration( + { + capabilities: { human_interaction: [] }, + connector_id: connectorId, + display_name: connectorId, + manifest_uri: `https://sources.example/${connectorId}`, + protocol_version: "0.1.0", + streams: [ + { + name: "holdings", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + name: "activity", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + coverage_strategy: "parent_detail_accounting", + name: "valuations", + parent_streams: ["holdings", "activity"], + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "0.1.0", + }, + { sourceId: `https://sources.example/${connectorId}` } + ); } function writeConnectorStub(tmpDir: string, script: string): string { @@ -175,7 +179,7 @@ export async function runMultiParentScenario({ process.stdout.write(JSON.stringify({ type: 'RECORD', stream: 'holdings', key: 'H', data: { id: 'H' }, emitted_at: new Date().toISOString() }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'RECORD', stream: 'activity', key: 'A', data: { id: 'A' }, emitted_at: new Date().toISOString() }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'DETAIL_COVERAGE', reference_only: true, stream: 'valuations', state_stream: 'holdings', required_keys: ['H'], hydrated_keys: [] }) + '\\n'); - process.stdout.write(JSON.stringify({ type: 'DETAIL_COVERAGE', reference_only: true, stream: 'valuations', state_stream: 'activity', required_keys: ['A'], hydrated_keys: [], empty_keys: ['A'] }) + '\\n'); + process.stdout.write(JSON.stringify({ type: 'DETAIL_COVERAGE', reference_only: true, stream: 'valuations', state_stream: 'activity', required_keys: ['A'], hydrated_keys: ['A'] }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'STATE', stream: 'holdings', cursor: { page: 'holdings-1' } }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'STATE', stream: 'activity', cursor: { page: 'activity-1' } }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'DONE', status: 'succeeded', records_emitted: 2 }) + '\\n'); @@ -229,13 +233,19 @@ export async function runMultiParentScenario({ } /** - * Hydrated-key crediting depends on the connector's own manifest declaration. - * The connector lists `H` as hydrated for the holdings boundary but emits no - * valuations record for it, and reports `A` as verified empty for activity. - * With DETAIL_COVERAGE_KEY_SETS declared, the bare `H` listing is not evidence - * and holdings stays put. Without it, the connector keeps the crediting it had - * before the capability existed and both cursors commit. Shared by the SQLite - * and Postgres conformance files so both backends run one scenario. + * DETAIL_COVERAGE_KEY_SETS credit depends on the manifest declaration, and the + * runtime omits the capability from START, so every connector here falls back + * to base-profile `hydrated_keys` on the wire. The two modes differ only in the + * manifest declaration: + * - Declaring connector: a hydrated key credits only with an accepted detail + * record for it. Activity `A` has one (`valuations` id `A`, the primary key + * of its own parent boundary), so activity commits. Holdings `H` has no + * record, so its hydration credits nothing and holdings stays put. + * - Nondeclaring connector: hydration credits by assertion, so `H` credits and + * both parents commit. + * Raw `empty_keys` and `unchanged_keys` are never emitted, so neither mode + * credits a key through them. Both backend suites run the same wire exchange + * and assert the same committed state. */ export async function runHydratedKeyCreditingScenario({ asUrl, @@ -254,24 +264,30 @@ export async function runHydratedKeyCreditingScenario({ connectorId: string; declaresKeySets: boolean; }): Promise { - // DETAIL_COVERAGE_KEY_SETS is new in Collection Profile 0.2.0, so a manifest - // that declares it declares that version (profile Section 3.1). - const manifest = { - ...multiParentManifest(connectorId), - protocol_capabilities: declaresKeySets ? ["DETAIL_COVERAGE_KEY_SETS"] : [], - protocol_version: declaresKeySets ? "0.2.0" : "0.1.0", - }; + // DETAIL_COVERAGE_KEY_SETS is an extension of the connector-protocol package, + // not a canonical profile capability. The runtime's manifest validation + // requires protocol_version 0.2.0 for a manifest that declares it, so a + // declaring manifest uses that version. + const manifest = withFixtureSourceDeclaration( + { + ...multiParentManifest(connectorId), + protocol_capabilities: declaresKeySets ? ["DETAIL_COVERAGE_KEY_SETS"] : [], + protocol_version: declaresKeySets ? "0.2.0" : "0.1.0", + }, + { sourceId: `https://sources.example/${connectorId}` } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-checkpoint-profile-hydrated-credit-")); const connectorPath = writeConnectorStub( tmpDir, ` process.stdout.write(JSON.stringify({ type: 'RECORD', stream: 'holdings', key: 'H', data: { id: 'H' }, emitted_at: new Date().toISOString() }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'RECORD', stream: 'activity', key: 'A', data: { id: 'A' }, emitted_at: new Date().toISOString() }) + '\\n'); + process.stdout.write(JSON.stringify({ type: 'RECORD', stream: 'valuations', key: 'A', data: { id: 'A' }, emitted_at: new Date().toISOString() }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'DETAIL_COVERAGE', reference_only: true, stream: 'valuations', state_stream: 'holdings', required_keys: ['H'], hydrated_keys: ['H'] }) + '\\n'); - process.stdout.write(JSON.stringify({ type: 'DETAIL_COVERAGE', reference_only: true, stream: 'valuations', state_stream: 'activity', required_keys: ['A'], hydrated_keys: [], empty_keys: ['A'] }) + '\\n'); + process.stdout.write(JSON.stringify({ type: 'DETAIL_COVERAGE', reference_only: true, stream: 'valuations', state_stream: 'activity', required_keys: ['A'], hydrated_keys: ['A'] }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'STATE', stream: 'holdings', cursor: { page: 'holdings-1' } }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'STATE', stream: 'activity', cursor: { page: 'activity-1' } }) + '\\n'); - process.stdout.write(JSON.stringify({ type: 'DONE', status: 'succeeded', records_emitted: 2 }) + '\\n'); + process.stdout.write(JSON.stringify({ type: 'DONE', status: 'succeeded', records_emitted: 3 }) + '\\n'); rl.close(); process.exit(0); ` @@ -301,7 +317,17 @@ export async function runHydratedKeyCreditingScenario({ assert.equal(result.status, "succeeded"); assert.equal(result.checkpoint_summary?.state_streams_staged, 2); - assert.equal(result.checkpoint_summary?.state_streams_committed, declaresKeySets ? 1 : 2); + // The runtime omits DETAIL_COVERAGE_KEY_SETS from START, so both modes send + // the same base-fallback hydration. Only the manifest declaration differs. + // A declaring connector keeps its accepted-record requirement: activity is + // proven by its valuations record, holdings' bare hydration is not. + assert.equal( + result.checkpoint_summary?.state_streams_committed, + declaresKeySets ? 1 : 2, + declaresKeySets + ? "a declaring connector commits only the parent whose hydrated key has an accepted record" + : "a nondeclaring connector's hydrated declarations credit by assertion, so both parents commit" + ); const stateResp = await fetch(`${rsUrl}/v1/state/${encodeURIComponent(manifest.connector_id)}`, { headers: { Authorization: `Bearer ${ownerToken}` }, @@ -328,39 +354,42 @@ export async function runHydratedKeyCreditingScenario({ // ----------------------------------------------------------------------- export function staticParentManifest(connectorId: string) { - return { - capabilities: { human_interaction: [] }, - connector_id: connectorId, - display_name: connectorId, - manifest_uri: `https://sources.example/${connectorId}`, - protocol_version: "0.1.0", - streams: [ - { - name: "messages", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - name: "unrelated_channel", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - coverage_strategy: "checkpoint_window", - name: "reactions", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - state_stream: "messages", - }, - ], - version: "0.1.0", - }; + return withFixtureSourceDeclaration( + { + capabilities: { human_interaction: [] }, + connector_id: connectorId, + display_name: connectorId, + manifest_uri: `https://sources.example/${connectorId}`, + protocol_version: "0.1.0", + streams: [ + { + name: "messages", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + name: "unrelated_channel", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + coverage_strategy: "checkpoint_window", + name: "reactions", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + state_stream: "messages", + }, + ], + version: "0.1.0", + }, + { sourceId: `https://sources.example/${connectorId}` } + ); } /** @@ -459,39 +488,42 @@ export async function runStaticParentEmitsCoverageScenario({ } export function undeclaredParentManifest(connectorId: string) { - return { - capabilities: { human_interaction: [] }, - connector_id: connectorId, - display_name: connectorId, - manifest_uri: `https://sources.example/${connectorId}`, - protocol_version: "0.1.0", - streams: [ - { - name: "messages", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - name: "unrelated_parent", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - coverage_strategy: "parent_detail_accounting", - name: "attachments", - parent_streams: ["messages"], - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "0.1.0", - }; + return withFixtureSourceDeclaration( + { + capabilities: { human_interaction: [] }, + connector_id: connectorId, + display_name: connectorId, + manifest_uri: `https://sources.example/${connectorId}`, + protocol_version: "0.1.0", + streams: [ + { + name: "messages", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + name: "unrelated_parent", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + coverage_strategy: "parent_detail_accounting", + name: "attachments", + parent_streams: ["messages"], + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "0.1.0", + }, + { sourceId: `https://sources.example/${connectorId}` } + ); } /** @@ -572,39 +604,42 @@ export async function runUndeclaredParentScenario({ } export function subsetParentManifest(connectorId: string) { - return { - capabilities: { human_interaction: [] }, - connector_id: connectorId, - display_name: connectorId, - manifest_uri: `https://sources.example/${connectorId}`, - protocol_version: "0.1.0", - streams: [ - { - name: "a", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - name: "b", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - coverage_strategy: "parent_detail_accounting", - name: "detail", - parent_streams: ["a", "b"], - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "0.1.0", - }; + return withFixtureSourceDeclaration( + { + capabilities: { human_interaction: [] }, + connector_id: connectorId, + display_name: connectorId, + manifest_uri: `https://sources.example/${connectorId}`, + protocol_version: "0.1.0", + streams: [ + { + name: "a", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + name: "b", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + coverage_strategy: "parent_detail_accounting", + name: "detail", + parent_streams: ["a", "b"], + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "0.1.0", + }, + { sourceId: `https://sources.example/${connectorId}` } + ); } /** @@ -637,7 +672,7 @@ export async function runSubsetParentCoverageScenario({ ` process.stdout.write(JSON.stringify({ type: 'RECORD', stream: 'a', key: 'a-1', data: { id: 'a-1' }, emitted_at: new Date().toISOString() }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'RECORD', stream: 'b', key: 'b-1', data: { id: 'b-1' }, emitted_at: new Date().toISOString() }) + '\\n'); - process.stdout.write(JSON.stringify({ type: 'DETAIL_COVERAGE', reference_only: true, stream: 'detail', state_stream: 'a', required_keys: ['a-1'], hydrated_keys: [], empty_keys: ['a-1'] }) + '\\n'); + process.stdout.write(JSON.stringify({ type: 'DETAIL_COVERAGE', reference_only: true, stream: 'detail', state_stream: 'a', required_keys: ['a-1'], hydrated_keys: ['a-1'] }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'STATE', stream: 'a', cursor: { page: 'a-1' } }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'STATE', stream: 'b', cursor: { page: 'b-1' } }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'DONE', status: 'succeeded', records_emitted: 2 }) + '\\n'); diff --git a/reference-implementation/test/helpers/connector-state-scheduler-conformance.ts b/reference-implementation/test/helpers/connector-state-scheduler-conformance.ts index de8f96c4c..7faa10e10 100644 --- a/reference-implementation/test/helpers/connector-state-scheduler-conformance.ts +++ b/reference-implementation/test/helpers/connector-state-scheduler-conformance.ts @@ -246,8 +246,8 @@ interface SchedulerDriver { type TestFn = (name: string, fn: () => Promise) => void; -export const CONNECTOR_A = "https://test.pdpp.dev/connectors/conformance-a"; -export const CONNECTOR_B = "https://test.pdpp.dev/connectors/conformance-b"; +export const CONNECTOR_A = "conformance-a"; +export const CONNECTOR_B = "conformance-b"; export const STREAM_X = "stream_x"; export const STREAM_Y = "stream_y"; diff --git a/reference-implementation/test/helpers/fixture-source-declaration.ts b/reference-implementation/test/helpers/fixture-source-declaration.ts index 98d9d12c7..aca0d97c4 100644 --- a/reference-implementation/test/helpers/fixture-source-declaration.ts +++ b/reference-implementation/test/helpers/fixture-source-declaration.ts @@ -7,11 +7,10 @@ const FIXTURE_PUBLISHER = "https://publishers.pdpp.test/fixtures"; /** * Test-only. Gives a fixture manifest an explicit, validated source declaration computed from its own - * final content, so registration uses the fixture's authored source identity instead of inferring one from - * the manifest's `connector_id` or `manifest_uri`. The default identity is the fixture's own - * (`https://fixtures.pdpp.test/sources/`), not a production identity; pass `sourceId` to pin another. - * Always recomputed from the content it is given, so a manifest derived by spreading another fixture never - * keeps a declaration for content it no longer has. + * final content, so a first registration of a fixture key with no reviewed legacy mapping goes through the + * Core route. The source identity is the fixture's own (`https://fixtures.pdpp.test/sources/`), not a + * production identity. Always recomputed from the content it is given, so a manifest derived by spreading + * another fixture never keeps a declaration for content it no longer has. */ export function withFixtureSourceDeclaration(manifest: T, options: { sourceId?: string } = {}): T { const content = Object.fromEntries( diff --git a/reference-implementation/test/helpers/legacy-upgrade.ts b/reference-implementation/test/helpers/legacy-upgrade.ts index 33882defe..dbf7eea5b 100644 --- a/reference-implementation/test/helpers/legacy-upgrade.ts +++ b/reference-implementation/test/helpers/legacy-upgrade.ts @@ -8,7 +8,7 @@ * `package/manifests/*.json` bundle entries. An upgraded database holds them * the way c82718d4's registerConnector stored them: the row id is the * manifest's `connector_key`, and the JSON went through - * `normalizeConnectorManifestForStorage`. Only the OCI fetch is replaced in + * the pre-retention declaration projection (`legacyStoredManifest`). Only the OCI fetch is replaced in * the scenarios; the install service, owner routes, registration, state, * ingest and read paths are the production ones. */ @@ -19,7 +19,7 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { normalizeConnectorManifestForStorage, registerConnector } from "../../server/auth.ts"; +import { registerConnector } from "../../server/auth.ts"; import { type ConnectorCatalogEntry, createConnectorInstallService, @@ -28,6 +28,7 @@ import { import { getDb } from "../../server/db.ts"; import { startServer } from "../../server/index.ts"; import { postgresQuery } from "../../server/postgres-storage.ts"; +import { snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest } from "../../server/source-declaration-legacy-collection.ts"; import { createPostgresConnectorInstanceStore, createSqliteConnectorInstanceStore, @@ -52,9 +53,28 @@ export function legacyManifest(key: string): Record { return JSON.parse(readFileSync(join(HERE, "..", "fixtures", "legacy-upgrade", `${key}.json`), "utf8")); } -/** The manifest JSON c82718d4 left in `connectors.manifest`. */ +/** Publisher the pre-retention projection stamped on every legacy declaration (server/auth.ts). */ +const LEGACY_PROJECTION_PUBLISHER_ID = "https://pdpp.dev/reference-implementation"; + +/** + * The manifest JSON c82718d4 left in `connectors.manifest`: the O-era manifest plus the + * declaration base projected from its provenance (`source.id = manifest_uri`). Built here + * rather than through the current normalizer, whose fresh-path rule refuses keys with no + * reviewed mapping, because an upgraded row was written by the earlier projection. + */ export function legacyStoredManifest(key: string): Record { - return normalizeConnectorManifestForStorage(legacyManifest(key)).storedManifest; + const manifest = legacyManifest(key); + // The O-era projection stored connector_id/connector_key as the canonical key and kept the URL as manifest_uri. + const stored: Record = { ...manifest, connector_id: key, connector_key: key }; + const provenance = typeof manifest.manifest_uri === "string" ? manifest.manifest_uri : String(manifest.connector_id); + return { + ...stored, + source_declaration: snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest(stored, { + connectorImplementationId: provenance, + publisherId: LEGACY_PROJECTION_PUBLISHER_ID, + sourceId: provenance, + }), + }; } type InstanceStore = ReturnType | ReturnType; diff --git a/reference-implementation/test/helpers/production-store-connector-state-scheduler-driver.ts b/reference-implementation/test/helpers/production-store-connector-state-scheduler-driver.ts index ddf73f4ad..0cb9c7597 100644 --- a/reference-implementation/test/helpers/production-store-connector-state-scheduler-driver.ts +++ b/reference-implementation/test/helpers/production-store-connector-state-scheduler-driver.ts @@ -83,8 +83,9 @@ interface ActiveRunRecord { const HARNESS_MANIFESTS = [ { connector_id: CONNECTOR_A, + connector_key: CONNECTOR_A, display_name: "Conformance Connector A", - manifest_uri: `https://sources.example/${CONNECTOR_A}`, + manifest_uri: "https://sources.example/https://test.pdpp.dev/connectors/conformance-a", protocol_version: "0.1.0", runtime_requirements: { bindings: { network: { required: true } } }, streams: [ @@ -107,8 +108,9 @@ const HARNESS_MANIFESTS = [ }, { connector_id: CONNECTOR_B, + connector_key: CONNECTOR_B, display_name: "Conformance Connector B", - manifest_uri: `https://sources.example/${CONNECTOR_B}`, + manifest_uri: "https://sources.example/https://test.pdpp.dev/connectors/conformance-b", protocol_version: "0.1.0", runtime_requirements: { bindings: { network: { required: true } } }, streams: [ @@ -274,7 +276,14 @@ export function createProductionStoreConnectorStateSchedulerDriver() { initDb(); for (const manifest of HARNESS_MANIFESTS) { // biome-ignore lint/performance/noAwaitInLoops: ordered test setup is intentionally sequential - await registerConnector(withFixtureSourceDeclaration(manifest, { sourceId: manifest.manifest_uri })); + await registerConnector( + withFixtureSourceDeclaration(manifest, { + sourceId: + manifest.connector_id === CONNECTOR_A + ? "https://sources.example/https://test.pdpp.dev/connectors/conformance-a" + : "https://sources.example/https://test.pdpp.dev/connectors/conformance-b", + }) + ); } stateStore = createSqliteConnectorStateStore(); schedulerStore = createSqliteSchedulerStore(); diff --git a/reference-implementation/test/helpers/ri-zero-connector-knowledge-data-load-scan.ts b/reference-implementation/test/helpers/ri-zero-connector-knowledge-data-load-scan.ts index 17c2a2ea3..c32f14e4a 100644 --- a/reference-implementation/test/helpers/ri-zero-connector-knowledge-data-load-scan.ts +++ b/reference-implementation/test/helpers/ri-zero-connector-knowledge-data-load-scan.ts @@ -150,7 +150,7 @@ const MANIFEST_ROOTS = [ * external state read at request time, never RI-committed source or * RI-committed JSON, so it was never a candidate for this allowlist either * (see `reference-implementation/server/version-disposition.ts`'s own module - * doc comment). The canonical Collection Profile observation schema is + * doc comment). The canonical Collection Profile observation schema is now * registered for its exact runtime loader. It describes protocol shapes, not * provider policy; the content check still rejects connector identity in it. * Additional entries require the same review as the generic read sites below. @@ -159,6 +159,9 @@ const SANCTIONED_POLICY_RESOURCES: ReadonlyMap> = ne ["reference-implementation/runtime/observations.ts", new Set([ "reference-implementation/runtime/observation.schema.json", ])], + ["reference-implementation/runtime/binding-declarations.ts", new Set([ + "reference-implementation/runtime/connector-manifest.schema.json", + ])], ]); const POLYFILL_MANIFEST_READ_SITE = 'reference-implementation/server/polyfill-manifest-reconcile.ts::readManifestJson::readFile(path, "utf8")'; diff --git a/reference-implementation/test/helpers/sqlite-connector-state-scheduler-driver.ts b/reference-implementation/test/helpers/sqlite-connector-state-scheduler-driver.ts index b49d73373..75f4aa7c6 100644 --- a/reference-implementation/test/helpers/sqlite-connector-state-scheduler-driver.ts +++ b/reference-implementation/test/helpers/sqlite-connector-state-scheduler-driver.ts @@ -79,8 +79,9 @@ interface ActiveDbRow { const HARNESS_MANIFESTS = [ { connector_id: CONNECTOR_A, + connector_key: CONNECTOR_A, display_name: "Conformance Connector A", - manifest_uri: `https://sources.example/${CONNECTOR_A}`, + manifest_uri: "https://sources.example/https://test.pdpp.dev/connectors/conformance-a", protocol_version: "0.1.0", runtime_requirements: { bindings: { network: { required: true } } }, streams: [ @@ -103,8 +104,9 @@ const HARNESS_MANIFESTS = [ }, { connector_id: CONNECTOR_B, + connector_key: CONNECTOR_B, display_name: "Conformance Connector B", - manifest_uri: `https://sources.example/${CONNECTOR_B}`, + manifest_uri: "https://sources.example/https://test.pdpp.dev/connectors/conformance-b", protocol_version: "0.1.0", runtime_requirements: { bindings: { network: { required: true } } }, streams: [ @@ -268,7 +270,14 @@ export function createSqliteConnectorStateSchedulerDriver() { initDb(); for (const manifest of HARNESS_MANIFESTS) { // biome-ignore lint/performance/noAwaitInLoops: ordered setup is intentionally sequential because each iteration advances shared test state. - await registerConnector(withFixtureSourceDeclaration(manifest, { sourceId: manifest.manifest_uri })); + await registerConnector( + withFixtureSourceDeclaration(manifest, { + sourceId: + manifest.connector_id === CONNECTOR_A + ? "https://sources.example/https://test.pdpp.dev/connectors/conformance-a" + : "https://sources.example/https://test.pdpp.dev/connectors/conformance-b", + }) + ); } // Establish this incarnation's boot epoch before any run is // announced: the spine refuses an unstamped `run.started`, and the diff --git a/reference-implementation/test/introspection-manifest-fail-closed.test.ts b/reference-implementation/test/introspection-manifest-fail-closed.test.ts index 73731d6fd..462e3b98e 100644 --- a/reference-implementation/test/introspection-manifest-fail-closed.test.ts +++ b/reference-implementation/test/introspection-manifest-fail-closed.test.ts @@ -40,6 +40,7 @@ const MANIFEST = { primary_key: ["id"], schema: { properties: { id: { type: "string" } }, type: "object" }, selection: { fields: true, resources: true }, + semantics: "mutable_state", }, ], }, @@ -49,6 +50,7 @@ const MANIFEST = { primary_key: ["id"], schema: { properties: { id: { type: "string" } }, type: "object" }, selection: { fields: true, resources: true }, + semantics: "mutable_state", }, ], version: "1.0.0", diff --git a/reference-implementation/test/legacy-upgrade-preservation.test.ts b/reference-implementation/test/legacy-upgrade-preservation.test.ts index 32e5f1f17..172ea2a42 100644 --- a/reference-implementation/test/legacy-upgrade-preservation.test.ts +++ b/reference-implementation/test/legacy-upgrade-preservation.test.ts @@ -64,11 +64,10 @@ test("a developer-local source preserves the underscore legacy id apple_contacts const dataDir = join(dir, "data"); mkdirSync(join(sourceRoot, "profile"), { recursive: true }); mkdirSync(join(sourceRoot, "dist"), { recursive: true }); - // The profile is the O-era manifest unchanged (connector_key apple_contacts). - writeFileSync( - join(sourceRoot, "profile", "collection-profile.json"), - JSON.stringify(legacyManifest("apple_contacts")) - ); + // The profile is the shipped O-era manifest, unmodified. This is an upgrade: the retained historical + // registration seeded below already carries the source identity, and add/select must not change it. + // A fresh, unmapped local source with the same key would still need an explicit source_declaration. + writeFileSync(join(sourceRoot, "profile", "collection-profile.json"), JSON.stringify(legacyManifest("apple_contacts"))); writeFileSync(join(sourceRoot, "dist", "collection-profile.mjs"), "export default {};\n"); const localStore = createFileLocalConnectorSourceStore(dataDir); const installStore = createFileConnectorInstallStore(dataDir); diff --git a/reference-implementation/test/manifest-runtime-requirements.test.ts b/reference-implementation/test/manifest-runtime-requirements.test.ts index 0c388c7be..e8167495d 100644 --- a/reference-implementation/test/manifest-runtime-requirements.test.ts +++ b/reference-implementation/test/manifest-runtime-requirements.test.ts @@ -11,7 +11,13 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { BindingDeclarationError } from "../runtime/binding-declarations.ts"; +import { getConnectorManifest } from "../server/auth.ts"; +import { getDb } from "../server/db.ts"; import { startServer } from "../server/index.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; + +const DUPLICATE_OBJECT_KEY = /repeats object key/; type StartedServer = Awaited>; @@ -76,8 +82,9 @@ async function registerConnectorManifest(asUrl: string, manifest: unknown): Prom } function makeManifest(runtimeRequirements: unknown) { - return { - connector_id: "https://registry.pdpp.test/connectors/runtime-requirements-fixture", + return withFixtureSourceDeclaration({ + connector_id: "runtime-requirements-fixture", + connector_key: "runtime-requirements-fixture", display_name: "Runtime requirements fixture", protocol_version: "0.1.0", runtime_requirements: runtimeRequirements, @@ -99,7 +106,7 @@ function makeManifest(runtimeRequirements: unknown) { }, ], version: "0.1.0", - }; + }); } test("valid browser runtime binding is accepted", async () => { @@ -122,7 +129,7 @@ test("unsupported runtime binding is rejected", async () => { assert.ok(hasErrorBody(body)); assert.equal(body.error.code, "invalid_request"); // biome-ignore lint/performance/useTopLevelRegex: localized test assertion preserves its explicit contract. - assert.match(body.error.message, /unsupported keys: toaster/u); + assert.match(body.error.message, /toaster must have required property 'kind'/u); }); }); @@ -136,6 +143,77 @@ test("runtime binding required flag must be boolean", async () => { assert.ok(hasErrorBody(body)); assert.equal(body.error.code, "invalid_request"); // biome-ignore lint/performance/useTopLevelRegex: localized test assertion preserves its explicit contract. - assert.match(body.error.message, /browser\.required must be a boolean/u); + assert.match(body.error.message, /browser\.required must be boolean/u); + }); +}); + +const RAW_KEY_CONNECTOR = "raw-binding-keys"; + +/** + * Valid registration for the raw-key tests: the operational key is a slug and + * the source identity is a distinct absolute URI. A URI-only `connector_id` + * registers with a source-declaration error (sourceId must be an absolute URI) + * or stores a URL connector_key that the read path rejects, so neither is a + * usable control. + */ +function makeRawBindingKeysManifest() { + return withFixtureSourceDeclaration({ + ...makeManifest({ bindings: { network: { required: false } } }), + connector_id: RAW_KEY_CONNECTOR, + connector_key: RAW_KEY_CONNECTOR, + manifest_uri: `https://registry.pdpp.test/connectors/${RAW_KEY_CONNECTOR}`, + }); +} + +test("registration rejects duplicate binding keys in the raw HTTP body before persistence", async () => { + await withHarness(async ({ asUrl }) => { + const manifest = makeRawBindingKeysManifest(); + const valid = JSON.stringify(manifest); + const duplicate = valid.replace( + '"network":{"required":false}', + '"network":{"required":true},"network":{"required":false}' + ); + assert.notEqual(duplicate, valid, "the request must retain both raw object keys"); + const response = await fetch(`${asUrl}/connectors`, { + body: duplicate, + headers: { "Content-Type": "application/json" }, + method: "POST", + }); + assert.equal(response.status, 400); + assert.match(await response.text(), DUPLICATE_OBJECT_KEY); + assert.equal(await getConnectorManifest(manifest.connector_id), null, "a rejected declaration is not persisted"); + const baseline = await registerConnectorManifest(asUrl, manifest); + assert.equal(baseline.status, 201, "the same declaration without duplicate keys must register"); + }); +}); + +test("stored manifest reads reject duplicate raw keys before JSON parsing loses them", async () => { + await withHarness(async ({ asUrl }) => { + const manifest = makeRawBindingKeysManifest(); + const registered = await registerConnectorManifest(asUrl, manifest); + assert.equal(registered.status, 201); + try { + assert.ok(await getConnectorManifest(manifest.connector_id)); + } catch (error: unknown) { + assert.fail( + `valid stored fixture must load: ${error instanceof Error ? String(error.cause ?? error) : String(error)}` + ); + } + const valid = JSON.stringify(manifest); + const duplicate = valid.replace( + '"network":{"required":false}', + '"network":{"required":true},"network":{"required":false}' + ); + assert.notEqual(duplicate, valid); + getDb().prepare("UPDATE connectors SET manifest = ? WHERE connector_id = ?").run(duplicate, manifest.connector_id); + await assert.rejects( + () => getConnectorManifest(manifest.connector_id), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.ok(error.cause instanceof BindingDeclarationError); + assert.equal(error.cause.code, "duplicate_json_key"); + return true; + } + ); }); }); diff --git a/reference-implementation/test/manifest-validation-runtime-refresh.test.ts b/reference-implementation/test/manifest-validation-runtime-refresh.test.ts index eb16a9c26..5e988f638 100644 --- a/reference-implementation/test/manifest-validation-runtime-refresh.test.ts +++ b/reference-implementation/test/manifest-validation-runtime-refresh.test.ts @@ -8,10 +8,8 @@ // bindings + external tools and its refresh policy; a loosened check silently // admits a malformed manifest at registration. // -// NOTE (pinned quirk): validateRuntimeRequirements returns EARLY when -// `runtime_requirements.bindings` is absent, so `external_tools` is only validated -// when `bindings` is also declared. These tests include `bindings` for the -// external-tools cases and pin the early-return behavior explicitly. +// Binding and external-tool declarations are validated independently, including +// external tools in a runtime_requirements object without a bindings member. // // Mutation surface: // validateRuntimeRequirements -- object-shape guards, RUNTIME_REQUIREMENT_BINDINGS @@ -54,7 +52,7 @@ test("validateRuntimeRequirements: absent runtime_requirements / bindings is a p assert.doesNotThrow(() => validateRuntimeRequirements({}, CODE)); assert.doesNotThrow( () => validateRuntimeRequirements({ runtime_requirements: {} }, CODE), - "no bindings -> early return pass" + "an absent bindings member does not add a binding requirement" ); }); @@ -67,14 +65,14 @@ test("validateRuntimeRequirements: a recognized binding with a boolean required test("validateRuntimeRequirements: an unsupported binding key is rejected", () => { expectReject( () => validateRuntimeRequirements({ runtime_requirements: { bindings: { teleport: {} } } }, CODE), - "unsupported keys: teleport" + "teleport must have required property 'required'" ); }); test("validateRuntimeRequirements: a non-boolean `required` is rejected", () => { expectReject( () => validateRuntimeRequirements({ runtime_requirements: { bindings: { network: { required: "yes" } } } }, CODE), - "required must be a boolean" + "network.required must be boolean" ); }); @@ -82,28 +80,37 @@ test("validateRuntimeRequirements: non-object runtime_requirements or bindings i expectReject(() => validateRuntimeRequirements({ runtime_requirements: [] }, CODE), "must be an object"); expectReject( () => validateRuntimeRequirements({ runtime_requirements: { bindings: [] } }, CODE), - "bindings must be an object" + "bindings must be object" ); }); // --------------------------------------------------------------------------- -// validateRuntimeRequirements — external_tools (requires bindings present) +// validateRuntimeRequirements — external_tools (independent of bindings) // --------------------------------------------------------------------------- const withBindings = { bindings: { network: { required: true } } }; -test("validateRuntimeRequirements: external_tools is only validated when bindings is declared (pinned quirk)", () => { - // WITHOUT bindings, a clearly-invalid external_tools entry is NOT validated - // (early return) — this pins the current behavior so a refactor that reorders - // the checks is caught. - assert.doesNotThrow( +test("validateRuntimeRequirements: external_tools is validated even when bindings is absent", () => { + // Formerly an early return skipped external_tools when bindings was absent. + // The coupling is removed: the invalid entry is rejected either way. + expectReject( () => validateRuntimeRequirements({ runtime_requirements: { external_tools: [{ bogus: 1 }] } }, CODE), - "no bindings -> external_tools not reached" + "has unsupported keys: bogus" ); - // WITH bindings, the same invalid entry IS rejected. expectReject(() => validateRuntimeRequirements({ runtime_requirements: { ...withBindings, external_tools: [{ bogus: 1 }] } }, CODE) ); + // Control: a valid external_tools entry without bindings still passes. + assert.doesNotThrow(() => + validateRuntimeRequirements( + { + runtime_requirements: { + external_tools: [{ license: "MIT", name: "git", purpose: "vcs" }], + }, + }, + CODE + ) + ); }); test("validateRuntimeRequirements: external_tools requires name/license/purpose non-empty strings", () => { diff --git a/reference-implementation/test/owner-connection-intent.test.ts b/reference-implementation/test/owner-connection-intent.test.ts index a112c509e..483e9b152 100644 --- a/reference-implementation/test/owner-connection-intent.test.ts +++ b/reference-implementation/test/owner-connection-intent.test.ts @@ -44,6 +44,7 @@ import { COLLECTOR_PROTOCOL_VERSION } from "../server/collector-protocol.ts"; import { type ConnectorManifestLike, classifyConnectorIntentModality } from "../server/connection-setup-plan.ts"; import { canonicalConnectorKey } from "../server/connector-key.ts"; import { startServer } from "../server/index.ts"; +import { snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest } from "../server/source-declaration-legacy-collection.ts"; import { createSqliteConnectorInstanceStore } from "../server/stores/connector-instance-store.ts"; import { TEST_PRE_REGISTERED_PUBLIC_CLIENTS } from "./fixtures/demo-clients.ts"; @@ -285,27 +286,23 @@ function loadPackageManifest(name: string): PackageManifest { return entry.manifest as PackageManifest; } +/** + * Test-only explicit attribution. The declaration is the production legacy projection of the manifest's + * own content (streams keep only the Core members), not a copy of connector-only stream fields, which + * Core refuses. Source and publisher are fixture identities, never a production source. + */ function withExplicitTestSourceDeclaration(manifest: PackageManifest): PackageManifest { if (manifest.source_declaration && typeof manifest.source_declaration === "object") { return manifest; } const connectorKey = canonicalConnectorKey(manifest.connector_id) ?? manifest.connector_id; - const streams = Array.isArray(manifest.streams) - ? manifest.streams.map((stream) => ({ - ...stream, - ...(stream.semantics === "append" ? { semantics: "append_only" } : {}), - })) - : []; return { ...manifest, - source_declaration: { - declaration_version: `owner-connection-intent-test:${connectorKey}:v1`, - display: { name: typeof manifest.display_name === "string" ? manifest.display_name : connectorKey }, - protocol_version: manifest.protocol_version, - publisher: { id: "https://pdpp.dev/reference-implementation/tests" }, - source: { id: `https://sources.example/connectors/${encodeURIComponent(connectorKey)}`, kind: "connector" }, - streams, - }, + source_declaration: snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest(manifest, { + connectorImplementationId: `https://implementations.example/connectors/${encodeURIComponent(connectorKey)}`, + publisherId: "https://pdpp.dev/reference-implementation/tests", + sourceId: `https://sources.example/connectors/${encodeURIComponent(connectorKey)}`, + }), }; } diff --git a/reference-implementation/test/owner-connector-templates.test.ts b/reference-implementation/test/owner-connector-templates.test.ts index 641ad5bad..0da2fa29a 100644 --- a/reference-implementation/test/owner-connector-templates.test.ts +++ b/reference-implementation/test/owner-connector-templates.test.ts @@ -19,6 +19,7 @@ import { canonicalConnectorKey } from "../server/connector-key.ts"; import { startServer } from "../server/index.ts"; import { createSqliteConnectorInstanceStore } from "../server/stores/connector-instance-store.ts"; import { TEST_PRE_REGISTERED_PUBLIC_CLIENTS } from "./fixtures/demo-clients.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; const OWNER_SUBJECT_ID = "owner_local"; const NOW = "2026-06-01T00:00:00.000Z"; @@ -379,7 +380,7 @@ test("owner-template setup plan offers a catalog-installed browser connector by acmeManifest.display_name = "Acme Shop"; acmeManifest.setup = undefined; acmeManifest.capabilities = { ...asRecord(acmeManifest.capabilities), public_listing: { tier: "preview" } }; - await registerConnector(asUrl, acmeManifest); + await registerConnector(asUrl, withFixtureSourceDeclaration(acmeManifest)); const ownerToken = await issueOwnerToken(asUrl); const { status, body } = await fetchJson(`${rsUrl}/v1/owner/connector-templates`, { diff --git a/reference-implementation/test/ri-zero-connector-knowledge-conformance.test.ts b/reference-implementation/test/ri-zero-connector-knowledge-conformance.test.ts index 9217b0814..4245d27da 100644 --- a/reference-implementation/test/ri-zero-connector-knowledge-conformance.test.ts +++ b/reference-implementation/test/ri-zero-connector-knowledge-conformance.test.ts @@ -99,6 +99,37 @@ test("the pinned observation schema has one exact loader and cannot carry connec } }); +test("the pinned connector manifest schema has one exact loader and cannot carry connector identity", () => { + const fixtureRoot = mkdtempSync(join(tmpdir(), "ri-manifest-schema-resource-")); + const sourceRel = "reference-implementation/runtime/binding-declarations.ts"; + const sourceFile = join(fixtureRoot, sourceRel); + const schemaFile = join(dirname(sourceFile), "connector-manifest.schema.json"); + const source = + 'import manifestSchema from "./connector-manifest.schema.json" with { type: "json" };\nexport default manifestSchema;\n'; + try { + mkdirSync(dirname(sourceFile), { recursive: true }); + writeFileSync(sourceFile, source); + writeFileSync(schemaFile, JSON.stringify({ enum: ["browser", "network", "filesystem"] })); + assert.deepEqual(scanFileDataLoads(sourceFile, sourceRel, fixtureRoot), [], + "the canonical manifest schema is a generic resource, not connector-specific policy"); + + writeFileSync(schemaFile, JSON.stringify({ nested: { enum: ["synthetic_provider_identity"] } })); + const injected = scanFileDataLoads(sourceFile, sourceRel, fixtureRoot, new Set(["synthetic_provider_identity"])); + assert.ok(injected.some((violation) => violation.rule === "hardcoded-connector-literal-in-ri-owned-json"), + "sanctioning the manifest schema path must retain the provider-content check"); + + writeFileSync(schemaFile, JSON.stringify({ enum: ["browser"] })); + const otherRel = "reference-implementation/runtime/other-manifest-loader.ts"; + const otherFile = join(fixtureRoot, otherRel); + writeFileSync(otherFile, source); + assert.ok(scanFileDataLoads(otherFile, otherRel, fixtureRoot) + .some((violation) => violation.rule === "unsanctioned-policy-resource-path"), + "a different loader does not inherit the manifest schema exception"); + } finally { + rmSync(fixtureRoot, { force: true, recursive: true }); + } +}); + test("manifest-derived connector identity set is non-trivial (sanity check on the derivation itself)", () => { const keys = manifestDerivedConnectorKeys({ repoRoot }); // A hardcoded lower-bound count would itself be exactly the kind of diff --git a/reference-implementation/test/run-interaction-control.test.ts b/reference-implementation/test/run-interaction-control.test.ts index 886d86c96..e22e6b14e 100644 --- a/reference-implementation/test/run-interaction-control.test.ts +++ b/reference-implementation/test/run-interaction-control.test.ts @@ -28,6 +28,19 @@ import { canonicalConnectorKey } from "../server/connector-key.ts"; import { startServer } from "../server/index.ts"; import { resolveCredentialFreeFixtureRunEnv } from "./helpers/credential-free-run-fixture.ts"; +// Test scaffolding only. The shipped seed declares `credentials` alone, while these +// tests drive connectors that also emit otp and manual_action. Declaring them here +// keeps the shipped seed unchanged and the undeclared-kind refusal intact elsewhere. +function withTestHumanInteraction>(manifest: T): T { + return { + ...manifest, + capabilities: { + ...((manifest.capabilities as Record | undefined) ?? {}), + human_interaction: ["credentials", "otp", "manual_action"], + }, + } as T; +} + const REGEXP_1 = /text\/html/; const REGEXP_2 = /Pending interaction/; const REGEXP_3 = /Send success/; @@ -264,8 +277,8 @@ async function withHarness( rsPort: 0, })) as ClosableServer; const asUrl = `http://localhost:${server.asPort}`; - const spotifyManifest: SpotifyManifest = JSON.parse( - readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") + const spotifyManifest: SpotifyManifest = withTestHumanInteraction( + JSON.parse(readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8")) ); try { await registerConnector(asUrl, spotifyManifest); @@ -286,8 +299,8 @@ async function withCustomHarness(connectorPath: string, fn: (ctx: HarnessContext rsPort: 0, })) as ClosableServer; const asUrl = `http://localhost:${server.asPort}`; - const spotifyManifest: SpotifyManifest = JSON.parse( - readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") + const spotifyManifest: SpotifyManifest = withTestHumanInteraction( + JSON.parse(readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8")) ); try { await registerConnector(asUrl, spotifyManifest); diff --git a/reference-implementation/test/run-interaction-stream-routes.test.ts b/reference-implementation/test/run-interaction-stream-routes.test.ts index 36cf67755..697310d12 100644 --- a/reference-implementation/test/run-interaction-stream-routes.test.ts +++ b/reference-implementation/test/run-interaction-stream-routes.test.ts @@ -48,6 +48,20 @@ import { normalizeReferenceWireViewportPayload } from "../server/streaming/proto import { createStreamingSessionStore } from "../server/streaming/sessions.ts"; import { resolveCredentialFreeFixtureRunEnv } from "./helpers/credential-free-run-fixture.ts"; +// Test scaffolding only. The shipped seed declares `credentials` alone, while these +// tests drive connectors that also emit otp and manual_action. Declaring them here +// keeps the shipped seed unchanged and the undeclared-kind refusal intact elsewhere. +function withTestHumanInteraction>(manifest: T): T { + return { + ...manifest, + capabilities: { + ...((manifest.capabilities as Record | undefined) ?? {}), + human_interaction: ["credentials", "otp", "manual_action"], + }, + } as T; +} + + const REGEXP_1 = /retry/i; const REGEXP_2 = /retry/i; const REGEXP_3 = /text\/event-stream/; @@ -895,9 +909,7 @@ async function withHarness(options: HarnessOptions | null, fn: (ctx: HarnessCont }); const asUrl = `http://localhost:${server.asPort}`; const manifestName = harnessOptions.manifestName || "spotify"; - const spotifyManifest: SpotifyManifest = JSON.parse( - readFileSync(join(REFERENCE_IMPL_DIR, `fixtures/seed-manifests/${manifestName}.json`), "utf8") - ); + const spotifyManifest: SpotifyManifest = withTestHumanInteraction(JSON.parse(readFileSync(join(REFERENCE_IMPL_DIR, `fixtures/seed-manifests/${manifestName}.json`), "utf8"))); try { await registerConnector(asUrl, spotifyManifest); await fn({ asUrl, companions, server, spotifyManifest }); @@ -3536,9 +3548,7 @@ test("SSE delivers multiple frames and acks each, even when ack rejects", async }); try { const asUrl = `http://localhost:${server.asPort}`; - const spotifyManifest: SpotifyManifest = JSON.parse( - readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") - ); + const spotifyManifest: SpotifyManifest = withTestHumanInteraction(JSON.parse(readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8"))); await registerConnector(asUrl, spotifyManifest); const started = await startRun(asUrl, spotifyManifest.connector_id); const pending = await waitForPendingInteraction(asUrl, started.run_id); @@ -3644,7 +3654,7 @@ test("mint fails closed with 503 streaming_companion_unavailable when no compani }); try { const asUrl = `http://localhost:${server.asPort}`; - const spotifyManifest = JSON.parse(readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8")); + const spotifyManifest = withTestHumanInteraction(JSON.parse(readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8"))); await registerConnector(asUrl, spotifyManifest); const started = await startRun(asUrl, spotifyManifest.connector_id); const pending = await waitForPendingInteraction(asUrl, started.run_id); diff --git a/reference-implementation/test/runtime-requirements-validation.test.ts b/reference-implementation/test/runtime-requirements-validation.test.ts index 653ae3669..722cecee4 100644 --- a/reference-implementation/test/runtime-requirements-validation.test.ts +++ b/reference-implementation/test/runtime-requirements-validation.test.ts @@ -74,7 +74,7 @@ test("validateRuntimeRequirements: returns when runtime_requirements is absent", test("validateRuntimeRequirements: accepts valid bindings", () => { assert.equal( validateRuntimeRequirements( - { runtime_requirements: { bindings: { browser: { required: true }, network: {} } } }, + { runtime_requirements: { bindings: { browser: { required: true }, network: { required: false } } } }, CODE ), undefined @@ -95,18 +95,18 @@ test("validateRuntimeRequirements: accepts a valid external tool (with bindings test("validateRuntimeRequirements: rejects non-object requirements or bindings", () => { assertRejects({ runtime_requirements: "x" }, "runtime_requirements must be an object"); - assertRejects({ runtime_requirements: { bindings: "x" } }, "runtime_requirements.bindings must be an object"); + assertRejects({ runtime_requirements: { bindings: "x" } }, "runtime_requirements.bindings must be object"); }); test("validateRuntimeRequirements: rejects an unsupported binding key", () => { - assertRejects({ runtime_requirements: { bindings: { gpu: {} } } }, "bindings has unsupported keys: gpu"); + assertRejects({ runtime_requirements: { bindings: { gpu: {} } } }, "gpu must have required property 'required'"); }); test("validateRuntimeRequirements: rejects a non-object binding value and a non-boolean required", () => { - assertRejects({ runtime_requirements: { bindings: { browser: "x" } } }, "bindings.browser must be an object"); + assertRejects({ runtime_requirements: { bindings: { browser: "x" } } }, "bindings.browser must be object"); assertRejects( { runtime_requirements: { bindings: { browser: { required: "yes" } } } }, - "bindings.browser.required must be a boolean" + "bindings.browser.required must be boolean" ); }); @@ -269,13 +269,11 @@ test("validateRuntimeRequirements: rejects an unsupported local_paths.paths entr // --- short-circuit contract ------------------------------------------------- -test("validateRuntimeRequirements: external_tools is NOT validated when bindings is absent (early return)", () => { - // No bindings => the function returns before ever reaching external_tools, so - // an otherwise-invalid external_tools value is not rejected here. - assert.equal( - validateRuntimeRequirements({ runtime_requirements: { external_tools: "not-an-array" } }, CODE), - undefined, - "absent bindings short-circuits external_tools validation" +test("validateRuntimeRequirements: external_tools is validated when bindings is absent", () => { + // Formerly an early return skipped external_tools when bindings was absent. + assertRejects( + { runtime_requirements: { external_tools: "not-an-array" } }, + "runtime_requirements.external_tools must be an array" ); }); diff --git a/reference-implementation/test/runtime-stream-collection-failed-commit.test.ts b/reference-implementation/test/runtime-stream-collection-failed-commit.test.ts index d9ef97859..e23b55beb 100644 --- a/reference-implementation/test/runtime-stream-collection-failed-commit.test.ts +++ b/reference-implementation/test/runtime-stream-collection-failed-commit.test.ts @@ -12,6 +12,7 @@ import { runConnector } from "../runtime/index.ts"; import { startServer as startServerUntyped } from "../server/index.ts"; import { createRequestConnectorInstanceStore } from "../server/request-store-factories.ts"; import { admitOwnerRunConnection } from "../server/stores/connector-instance-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; // Regression coverage for the live runtime path (reference-implementation/runtime/index.ts) // used by every real connector run, as distinct from the standalone @@ -141,29 +142,32 @@ function fakeAdmitRunConnection( // shape that discriminates it: one sibling that fully succeeds, one stream // that reports its own certified failure. function testManifest(connectorId: string) { - return { - connector_id: connectorId, - display_name: "Runtime stream isolation test", - manifest_uri: `https://registry.pdpp.dev/connectors/${connectorId}`, - protocol_version: "0.1.0", - streams: [ - { - name: "sibling_a", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - name: "sibling_b", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "1.0.0", - }; + return withFixtureSourceDeclaration( + { + connector_id: connectorId, + display_name: "Runtime stream isolation test", + manifest_uri: `https://registry.pdpp.dev/connectors/${connectorId}`, + protocol_version: "0.1.0", + streams: [ + { + name: "sibling_a", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + name: "sibling_b", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "1.0.0", + }, + { sourceId: `https://registry.pdpp.dev/connectors/${connectorId}` } + ); } type RuntimeManifest = NonNullable[0]["manifest"]> & { @@ -173,18 +177,21 @@ type RuntimeManifest = NonNullable[0]["manifest" type RuntimeManifestStream = NonNullable[number]; function completeManifest(connectorId: string, streams: RuntimeManifestStream[]): RuntimeManifest { - return { - connector_id: connectorId, - display_name: "Runtime stream isolation test", - manifest_uri: `https://registry.pdpp.dev/connectors/${connectorId}`, - protocol_version: "0.1.0", - streams: streams.map((stream) => ({ - selection: { fields: true, resources: true }, - semantics: "mutable_state", - ...stream, - })), - version: "1.0.0", - }; + return withFixtureSourceDeclaration( + { + connector_id: connectorId, + display_name: "Runtime stream isolation test", + manifest_uri: `https://registry.pdpp.dev/connectors/${connectorId}`, + protocol_version: "0.1.0", + streams: streams.map((stream) => ({ + selection: { fields: true, resources: true }, + semantics: "mutable_state", + ...stream, + })), + version: "1.0.0", + }, + { sourceId: `https://registry.pdpp.dev/connectors/${connectorId}` } + ); } function writeConnectorStub(tmpDir: string, script: string): string { @@ -871,10 +878,9 @@ rl.on('line', (line) => { type: string; }; assert.equal(replayStart.type, "START"); - assert.ok( - replayStart.protocol_capabilities?.includes("DETAIL_COVERAGE_KEY_SETS"), - "START must advertise support before connectors emit empty_keys/unchanged_keys" - ); + // START lists exactly what this runtime implements. DETAIL_COVERAGE_KEY_SETS is + // advisory and not advertised, so this connector reports hydrated keys only. + assert.deepEqual(replayStart.protocol_capabilities, ["OBSERVATION", "STREAM_EVIDENCE"]); const replayStartState = replayStart.state; assert.ok( !("sibling_b" in (replayStartState ?? {})), diff --git a/reference-implementation/test/scheduler-static-secret-injection.test.ts b/reference-implementation/test/scheduler-static-secret-injection.test.ts index 4b7166945..2c15737b7 100644 --- a/reference-implementation/test/scheduler-static-secret-injection.test.ts +++ b/reference-implementation/test/scheduler-static-secret-injection.test.ts @@ -61,6 +61,7 @@ import type { RunRecord } from "../runtime/scheduler-domain-types.ts"; import { closeDb, getDb } from "../server/db.ts"; import { startServer } from "../server/index.ts"; import { createSqliteConnectorInstanceStore } from "../server/stores/connector-instance-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; const BACKGROUND_SAFE_MANIFEST = { capabilities: { @@ -246,33 +247,36 @@ async function registerFixtureConnectorInstance(options: { } function minimalNestedAuthManifest(connectorKey: string): Record { - return { - capabilities: { - auth: { kind: "custom", required: [["NESTED_PRIMARY_TOKEN", "NESTED_SECONDARY_TOKEN"]] }, - refresh_policy: { - background_safe: true, - interaction_posture: "none", - rationale: "Test fixture for connector environment policy coverage.", - recommended_interval_seconds: 60, - recommended_mode: "automatic", + return withFixtureSourceDeclaration( + { + capabilities: { + auth: { kind: "custom", required: [["NESTED_PRIMARY_TOKEN", "NESTED_SECONDARY_TOKEN"]] }, + refresh_policy: { + background_safe: true, + interaction_posture: "none", + rationale: "Test fixture for connector environment policy coverage.", + recommended_interval_seconds: 60, + recommended_mode: "automatic", + }, }, + connector_id: connectorKey, + connector_key: connectorKey, + display_name: connectorKey, + manifest_uri: `https://registry.pdpp.dev/connectors/${connectorKey}`, + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "0.1.0", }, - connector_id: connectorKey, - connector_key: connectorKey, - display_name: connectorKey, - manifest_uri: `https://registry.pdpp.dev/connectors/${connectorKey}`, - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "0.1.0", - }; + { sourceId: `https://registry.pdpp.dev/connectors/${connectorKey}` } + ); } test("connector platform env keeps cross-host execution inputs without ambient secrets", () => { @@ -1024,33 +1028,39 @@ test("connector stderr progress and retained diagnostics redact secret and proxy }); try { - await registerManifest(`http://localhost:${server.asPort}`, { - ...BACKGROUND_SAFE_MANIFEST, - capabilities: { - refresh_policy: { - background_safe: true, - interaction_posture: "none", - rationale: "Test fixture for stderr redaction coverage.", - recommended_interval_seconds: 60, - recommended_mode: "automatic", - }, - }, - connector_id: "stderr-server-test", - connector_key: "stderr-server-test", - display_name: "Stderr server test", - manifest_uri: "https://registry.pdpp.dev/connectors/stderr-server-test", - protocol_version: "0.1.0", - streams: [ + await registerManifest( + `http://localhost:${server.asPort}`, + withFixtureSourceDeclaration( { - name: "items", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", + ...BACKGROUND_SAFE_MANIFEST, + capabilities: { + refresh_policy: { + background_safe: true, + interaction_posture: "none", + rationale: "Test fixture for stderr redaction coverage.", + recommended_interval_seconds: 60, + recommended_mode: "automatic", + }, + }, + connector_id: "stderr-server-test", + connector_key: "stderr-server-test", + display_name: "Stderr server test", + manifest_uri: "https://registry.pdpp.dev/connectors/stderr-server-test", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "0.1.0", }, - ], - version: "0.1.0", - }); + { sourceId: "https://registry.pdpp.dev/connectors/stderr-server-test" } + ) + ); await registerFixtureConnectorInstance({ connectorId: "stderr-server-test", connectorInstanceId: "cin_stderr_server", @@ -1271,7 +1281,13 @@ test("a store row suppresses credentials_required on the scheduled path (and its connectorInstanceId: "cin_github_test", connectorPath, intervalMs: 60_000, - manifest: BACKGROUND_SAFE_MANIFEST, + manifest: { + ...BACKGROUND_SAFE_MANIFEST, + capabilities: { + ...BACKGROUND_SAFE_MANIFEST.capabilities, + human_interaction: ["credentials"], + }, + }, ownerSubjectId: "scheduler_creds_required_user", ownerToken, }, diff --git a/reference-implementation/test/scheduler.test.ts b/reference-implementation/test/scheduler.test.ts index fae254c4d..9fa4ebdbd 100644 --- a/reference-implementation/test/scheduler.test.ts +++ b/reference-implementation/test/scheduler.test.ts @@ -42,6 +42,7 @@ import { import type { ActiveRunRecord, SchedulerRunHistoryRecord } from "../server/stores/scheduler-store.ts"; import { getDefaultSchedulerStore } from "../server/stores/scheduler-store.ts"; import { resolveCredentialFreeFixtureRunEnv } from "./helpers/credential-free-run-fixture.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; const __dirname = dirname(fileURLToPath(import.meta.url)); const REFERENCE_IMPL_DIR = join(__dirname, ".."); @@ -55,27 +56,30 @@ function readServerSchedulerFixtureManifest(): Record & { const base = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ) as Record & { capabilities?: Record }; - return { - ...base, - capabilities: { - ...base.capabilities, - refresh_policy: { - background_safe: true, - bot_detection_sensitivity: "low", - interaction_posture: "none", - maximum_staleness_seconds: 3600, - minimum_interval_seconds: 1, - rate_limit_sensitivity: "low", - rationale: "Deterministic scheduler integration fixture.", - recommended_interval_seconds: 60, - recommended_mode: "automatic", + return withFixtureSourceDeclaration( + { + ...base, + capabilities: { + ...base.capabilities, + refresh_policy: { + background_safe: true, + bot_detection_sensitivity: "low", + interaction_posture: "none", + maximum_staleness_seconds: 3600, + minimum_interval_seconds: 1, + rate_limit_sensitivity: "low", + rationale: "Deterministic scheduler integration fixture.", + recommended_interval_seconds: 60, + recommended_mode: "automatic", + }, }, + connector_id: "scheduler-fixture", + connector_key: "scheduler-fixture", + display_name: "Scheduler fixture", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-fixture", }, - connector_id: "scheduler-fixture", - connector_key: "scheduler-fixture", - display_name: "Scheduler fixture", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-fixture", - }; + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-fixture" } + ); } interface ClosableHttpServer { @@ -1262,29 +1266,32 @@ test("scheduler progress watchdog allows long direct runs that keep reporting pr }); test("scheduler preserves failure reasons and checkpoint summaries from failed runConnector results", async () => { - const manifest = { - connector_id: "scheduler-failure-test", - display_name: "Scheduler Failure Test Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-failure-test", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, - value: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-failure-test", + display_name: "Scheduler Failure Test Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-failure-test", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + value: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-failure-test" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-failure-")); const connectorPath = join(tmpDir, "connector.mjs"); writeFileSync( @@ -1395,43 +1402,46 @@ rl.on('line', (line) => { }); test("scheduler preserves partial checkpoint commit summaries from state persistence failures after DONE(succeeded)", async () => { - const manifest = { - connector_id: "scheduler-partial-checkpoint-test", - display_name: "Scheduler Partial Checkpoint Test Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-partial-checkpoint-test", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, - value: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-partial-checkpoint-test", + display_name: "Scheduler Partial Checkpoint Test Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-partial-checkpoint-test", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + value: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - { - name: "other_items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, - value: { type: "string" }, + { + name: "other_items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + value: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-partial-checkpoint-test" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-partial-checkpoint-")); const connectorPath = join(tmpDir, "connector.mjs"); writeFileSync( @@ -1601,29 +1611,32 @@ rl.on('line', (line) => { }); test("scheduler preserves terminal counter mismatch failures from runConnector results", async () => { - const manifest = { - connector_id: "scheduler-terminal-counter-mismatch-test", - display_name: "Scheduler Terminal Counter Mismatch Test Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-terminal-counter-mismatch-test", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, - value: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-terminal-counter-mismatch-test", + display_name: "Scheduler Terminal Counter Mismatch Test Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-terminal-counter-mismatch-test", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + value: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-terminal-counter-mismatch-test" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-terminal-counter-mismatch-")); const connectorPath = join(tmpDir, "connector.mjs"); writeFileSync( @@ -1749,29 +1762,32 @@ rl.on('line', (line) => { }); test("scheduler preserves connector-declared terminal error details from failed runs", async () => { - const manifest = { - connector_id: "scheduler-terminal-error-test", - display_name: "Scheduler Terminal Error Test Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-terminal-error-test", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, - value: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-terminal-error-test", + display_name: "Scheduler Terminal Error Test Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-terminal-error-test", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + value: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-terminal-error-test" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-terminal-error-")); const connectorPath = join(tmpDir, "connector.mjs"); writeFileSync( @@ -1885,28 +1901,31 @@ rl.on('line', (line) => { }); test("scheduler preserves known gaps from partial connector runs", async () => { - const manifest = { - connector_id: "scheduler-known-gap-test", - display_name: "Scheduler Known Gap Test Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-known-gap-test", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-known-gap-test", + display_name: "Scheduler Known Gap Test Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-known-gap-test", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-known-gap-test" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-known-gap-")); const connectorPath = join(tmpDir, "connector.mjs"); writeFileSync( @@ -1995,29 +2014,32 @@ rl.on('line', (line) => { }); test("scheduler preserves connector-declared terminal error details from cancelled runs", async () => { - const manifest = { - connector_id: "scheduler-cancelled-terminal-error-test", - display_name: "Scheduler Cancelled Terminal Error Test Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-cancelled-terminal-error-test", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, - value: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-cancelled-terminal-error-test", + display_name: "Scheduler Cancelled Terminal Error Test Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-cancelled-terminal-error-test", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + value: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-cancelled-terminal-error-test" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-cancelled-terminal-error-")); const connectorPath = join(tmpDir, "connector.mjs"); writeFileSync( @@ -2131,28 +2153,31 @@ rl.on('line', (line) => { }); test("scheduler does not retry deterministic connector protocol violations", async () => { - const manifest = { - connector_id: "scheduler-no-retry-protocol-violation", - display_name: "Scheduler No Retry Protocol Violation Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-no-retry-protocol-violation", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-no-retry-protocol-violation", + display_name: "Scheduler No Retry Protocol Violation Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-no-retry-protocol-violation", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-no-retry-protocol-violation" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-no-retry-protocol-")); const connectorPath = join(tmpDir, "connector.mjs"); const attemptsPath = join(tmpDir, "attempts.log"); @@ -2237,29 +2262,32 @@ rl.on('line', (line) => { }); test("scheduler retries connector-declared retryable failures and records the succeeding attempt", async () => { - const manifest = { - connector_id: "scheduler-retryable-terminal-error", - display_name: "Scheduler Retryable Terminal Error Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-retryable-terminal-error", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, - value: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-retryable-terminal-error", + display_name: "Scheduler Retryable Terminal Error Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-retryable-terminal-error", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + value: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-retryable-terminal-error" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-retryable-terminal-error-")); const connectorPath = join(tmpDir, "connector.mjs"); const attemptsPath = join(tmpDir, "attempts.log"); @@ -2369,29 +2397,32 @@ rl.on('line', (line) => { }); test("scheduler does not retry connector-declared non-retryable failures", async () => { - const manifest = { - connector_id: "scheduler-nonretryable-terminal-error", - display_name: "Scheduler Nonretryable Terminal Error Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-nonretryable-terminal-error", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, - value: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-nonretryable-terminal-error", + display_name: "Scheduler Nonretryable Terminal Error Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-nonretryable-terminal-error", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + value: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-nonretryable-terminal-error" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-nonretryable-terminal-error-")); const connectorPath = join(tmpDir, "connector.mjs"); const attemptsPath = join(tmpDir, "attempts.log"); @@ -3945,11 +3976,11 @@ test("scheduler records one not-ready skip for automatic runs when runtime prere const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-not-ready-test", connector_key: "scheduler-not-ready-test", - }; + }); const server = await startServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; @@ -4014,11 +4045,11 @@ test("scheduler emits a fresh not-ready skip when readiness reason changes", asy const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-not-ready-changing-test", connector_key: "scheduler-not-ready-changing-test", - }; + }); const server = await startServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; @@ -4081,7 +4112,7 @@ test("scheduler default readiness checker skips missing manifest-declared extern const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-missing-tool-test", connector_key: "scheduler-missing-tool-test", @@ -4097,7 +4128,7 @@ test("scheduler default readiness checker skips missing manifest-declared extern }, ], }, - }; + }); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-missing-tool-")); const { attemptsPath, connectorPath } = writeLoggingConnector(tmpDir); const server = await startServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); @@ -4156,7 +4187,7 @@ test("scheduler default readiness checker probes a manifest-declared executable_ const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-slackdump-bin-test", connector_key: "scheduler-slackdump-bin-test", @@ -4177,7 +4208,7 @@ test("scheduler default readiness checker probes a manifest-declared executable_ }, ], }, - }; + }); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-slackdump-bin-")); const fakeSlackdumpPath = join(tmpDir, "mounted-slackdump"); writeFileSync(fakeSlackdumpPath, '#!/bin/sh\n[ "$1" = "version" ] || exit 2\nexit 0\n', "utf8"); @@ -4318,14 +4349,14 @@ test("scheduler default readiness checker does not treat browser bindings as rea const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-browser-not-ready-test", connector_key: "scheduler-browser-not-ready-test", runtime_requirements: { bindings: { browser: { required: true }, network: { required: true } }, }, - }; + }); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-browser-not-ready-")); const { attemptsPath, connectorPath } = writeLoggingConnector(tmpDir); const previousRemoteCdp = process.env.PDPP_BROWSER_SURFACE_REMOTE_CDP_URL; @@ -4428,14 +4459,14 @@ test("scheduler default readiness checker treats PDPP_NEKO_CDP_HTTP_URL as manag const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-browser-neko-cdp-ready-test", connector_key: "scheduler-browser-neko-cdp-ready-test", runtime_requirements: { bindings: { browser: { required: true }, network: { required: true } }, }, - }; + }); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-browser-neko-cdp-")); const { attemptsPath, connectorPath } = writeLoggingConnector(tmpDir); const previousRemoteCdp = process.env.PDPP_BROWSER_SURFACE_REMOTE_CDP_URL; @@ -4526,14 +4557,14 @@ test("scheduler default readiness checker treats PDPP_NEKO_MANAGED_CONNECTORS as const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-browser-neko-managed-ready-test", connector_key: "scheduler-browser-neko-managed-ready-test", runtime_requirements: { bindings: { browser: { required: true }, network: { required: true } }, }, - }; + }); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-browser-neko-managed-")); const { attemptsPath, connectorPath } = writeLoggingConnector(tmpDir); const previousRemoteCdp = process.env.PDPP_BROWSER_SURFACE_REMOTE_CDP_URL; @@ -4620,22 +4651,26 @@ test("scheduler default readiness checker treats PDPP_NEKO_MANAGED_CONNECTORS as }); test("scheduler marks connector as needs-human when automatic run triggers interaction", async () => { - const manifest = { - connector_id: "scheduler-interaction-test", - display_name: "Interaction Test Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-interaction-test", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + const manifest = withFixtureSourceDeclaration( + { + capabilities: { human_interaction: ["otp"] }, + connector_id: "scheduler-interaction-test", + display_name: "Interaction Test Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-interaction-test", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "append_only", + }, + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-interaction-test" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-interaction-")); const connectorPath = join(tmpDir, "connector.mjs"); @@ -4745,11 +4780,11 @@ test("scheduler backoff skip derives next_attempt_at from history when last_run_ const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-backoff-1970-regression", connector_key: "scheduler-backoff-1970-regression", - }; + }); const server = await startServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; @@ -4868,11 +4903,11 @@ test("scheduler backoff skip uses gave_up phrasing once health-state crosses blo const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-backoff-blocked-msg", connector_key: "scheduler-backoff-blocked-msg", - }; + }); const server = await startServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; @@ -4970,11 +5005,11 @@ test("scheduler does not re-emit persisted backoff transition markers on restart const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-backoff-restart-noise", connector_key: "scheduler-backoff-restart-noise", - }; + }); const server = await startServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; From 3da781b9db3c1123410bd7ddf71afd09367087c6 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Thu, 5 Nov 2026 15:46:12 -0600 Subject: [PATCH 2/5] fix(connector-install): adopt a published root left by a crash before activation The installer publishes a verified digest root with a rename and then writes the active record. A crash between the two left a root that verified against the catalog entry but had no record, and every later install of that digest threw "root already exists without a matching active record". The owner could not recover without deleting the root by hand. The install now activates and registers a root that re-verifies against the entry, the same way it reuses a root that has an active record. A root that does not verify is still removed. A symlinked root is still refused. A named crash point after the rename lets the kill-and-restart test reach that window. Assisted-by: AI Signed-off-by: Tim Nunamaker --- .../server/connector-install/index.ts | 20 +++++++++---------- ...nnector-install-registration-crash.test.ts | 18 +++++++++++++++++ 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/reference-implementation/server/connector-install/index.ts b/reference-implementation/server/connector-install/index.ts index 17f6b7321..92bec0625 100644 --- a/reference-implementation/server/connector-install/index.ts +++ b/reference-implementation/server/connector-install/index.ts @@ -852,6 +852,7 @@ async function installStagedEntry(options: { const record = readVerifiedRecord(staged, verifiedEntry, installed.sourceDeclarationSha256); renameSync(staged, options.root); published = true; + crashPoint("connector-install:after-publish"); const active = { ...record, root: options.root }; const verifiedActive = verifyStoredRecord(options.root, active, options.dataDir); await options.store.activate(verifiedActive); @@ -1012,18 +1013,17 @@ export function createConnectorInstallService(options: { ); } else { // A crash can leave the published directory between rename and the - // active-record commit. Do not delete a valid retained root only - // because its active record is missing. + // active-record commit. A root that still verifies against the catalog + // entry is adopted: activate and register it as an install would. + // Anything else is an invalid leftover and is removed. + let adoptable = true; try { readVerifiedRecord(root, entry); - throw new Error("Connector digest root already exists without a matching active record."); - } catch (error) { - if ( - error instanceof Error && - error.message === "Connector digest root already exists without a matching active record." - ) { - throw error; - } + } catch { + adoptable = false; + } + if (adoptable) { + return await reuseExistingInstall(root, entry, null, null, dataDir, options.registerManifest, store, signal); } removePublishedRootSafely(root, dataDir, connectorId, entry.digest); if (existsSync(root)) { diff --git a/reference-implementation/test/connector-install-registration-crash.test.ts b/reference-implementation/test/connector-install-registration-crash.test.ts index d2ac80252..080aa2c1e 100644 --- a/reference-implementation/test/connector-install-registration-crash.test.ts +++ b/reference-implementation/test/connector-install-registration-crash.test.ts @@ -463,3 +463,21 @@ test("a retried install after a kill between pointer and registry also restores assert.equal((await agreement()).agree, true); }); }); + +test("killed after the root is published and before the pointer write: a retried install adopts the root", async () => { + await withSandbox(async ({ dataDir, dbPath }) => { + const env = { PDPP_DATA_DIR: dataDir }; + await killChildAt(CHILD, "connector-install:after-publish", { + args: [dbPath, dataDir, "install"], + env, + }); + await initDb(dbPath); + assert.equal(existsSync(rootOf(dataDir, GMAIL_INSTALL_DIGEST)), true, "the crash left a published root"); + assert.equal(await activeRecord(), null, "no pointer was written"); + + const adopted = await installService(dataDir).install("gmail", GMAIL_INSTALL_DIGEST); + + assert.equal(adopted.digest, GMAIL_INSTALL_DIGEST); + assert.equal((await agreement()).agree, true, "the adopted root is active and registered"); + }); +}); From 4d5b31389cbbfa6dbc6dc4664e160549925e962d Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Thu, 5 Nov 2026 16:08:22 -0600 Subject: [PATCH 3/5] test: declare the source of three fixtures the registration gate refused Three test files register a manifest with no `source_declaration` and a key that has no reviewed source mapping. The registration gate in this branch refuses that, so the files failed at setup, not at the behavior they test: stream-evidence, the startup semantic backfill crash test and the search-index repair test on PostgreSQL. Each fixture now declares its source with the shared helper, the same way the other fixtures do. Assisted-by: AI Signed-off-by: Tim Nunamaker --- ...-index-repair-correctness.postgres.test.ts | 8 +- ...ill-catches-crash-abandoned-record.test.ts | 109 ++++++++++-------- .../test/stream-evidence.test.ts | 74 ++++++------ 3 files changed, 106 insertions(+), 85 deletions(-) diff --git a/reference-implementation/test/search-index-repair-correctness.postgres.test.ts b/reference-implementation/test/search-index-repair-correctness.postgres.test.ts index 1947e131f..077571d03 100644 --- a/reference-implementation/test/search-index-repair-correctness.postgres.test.ts +++ b/reference-implementation/test/search-index-repair-correctness.postgres.test.ts @@ -22,10 +22,12 @@ import { __setLexicalBackfillPhaseHookForTest } from "../server/search.ts"; import { reconcileSearchIndexDirtyScope } from "../server/search-index-reconcile.ts"; import { configureSemanticBackend } from "../server/search-semantic.ts"; import { recordSearchIndexDirtyFailure } from "../server/stores/search-index-dirty-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; const POSTGRES_URL = process.env.PDPP_TEST_POSTGRES_URL; -const manifestFor = (connectorId: string) => ({ +const manifestFor = (connectorId: string) => { + const manifest = { capabilities: { human_interaction: [] }, connector_id: connectorId, display_name: connectorId, @@ -50,7 +52,9 @@ const manifestFor = (connectorId: string) => ({ }, ], version: "0.1.0", -}); + }; + return withFixtureSourceDeclaration(manifest, { sourceId: manifest.manifest_uri }); +}; const target = (connectorId: string, connectorInstanceId: string) => ({ connector_id: connectorId, diff --git a/reference-implementation/test/semantic-startup-backfill-catches-crash-abandoned-record.test.ts b/reference-implementation/test/semantic-startup-backfill-catches-crash-abandoned-record.test.ts index 9467cde37..915e41e3e 100644 --- a/reference-implementation/test/semantic-startup-backfill-catches-crash-abandoned-record.test.ts +++ b/reference-implementation/test/semantic-startup-backfill-catches-crash-abandoned-record.test.ts @@ -40,6 +40,7 @@ import { semanticIndexUpsert, } from "../server/search-semantic.ts"; import { listDirtySearchIndexScopes } from "../server/stores/search-index-dirty-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; function target(connectorInstanceId: string) { return { @@ -57,36 +58,42 @@ function record(stream: string, key: string, subject: string) { }; } -const baseManifest = { - capabilities: { human_interaction: [] }, - connector_id: "crash-gap", - display_name: "Crash gap probe", - manifest_uri: "https://registry.pdpp.dev/connectors/crash-gap", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - query: { search: {} }, - schema: { - properties: { id: { type: "string" }, subject: { type: "string" } }, - required: ["id", "subject"], - type: "object", +const baseManifest = withFixtureSourceDeclaration( + { + capabilities: { human_interaction: [] }, + connector_id: "crash-gap", + display_name: "Crash gap probe", + manifest_uri: "https://registry.pdpp.dev/connectors/crash-gap", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + query: { search: {} }, + schema: { + properties: { id: { type: "string" }, subject: { type: "string" } }, + required: ["id", "subject"], + type: "object", + }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "1.0.0", -}; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/crash-gap" } +); -const semanticManifest = { - ...baseManifest, - streams: baseManifest.streams.map((stream) => ({ - ...stream, - query: { search: { semantic_fields: ["subject"] } }, - })), -}; +const semanticManifest = withFixtureSourceDeclaration( + { + ...baseManifest, + streams: baseManifest.streams.map((stream) => ({ + ...stream, + query: { search: { semantic_fields: ["subject"] } }, + })), + }, + { sourceId: baseManifest.manifest_uri } +); /** * A real publisher in a separate Node process, therefore a separate @@ -279,17 +286,20 @@ test("a 501-field manifest publishes bounded batches and restart embeds only the initDb(":memory:"); const fields = Array.from({ length: 501 }, (_, index) => `field_${index}`); const properties = Object.fromEntries(fields.map((field) => [field, { type: "string" }])); - const manifest = { - ...semanticManifest, - connector_id: "semantic-field-batches", - streams: [ - { - ...semanticManifest.streams[0], - query: { search: { semantic_fields: fields } }, - schema: { properties: { id: { type: "string" }, ...properties }, type: "object" }, - }, - ], - }; + const manifest = withFixtureSourceDeclaration( + { + ...semanticManifest, + connector_id: "semantic-field-batches", + streams: [ + { + ...semanticManifest.streams[0], + query: { search: { semantic_fields: fields } }, + schema: { properties: { id: { type: "string" }, ...properties }, type: "object" }, + }, + ], + }, + { sourceId: baseManifest.manifest_uri } + ); let calls = 0; const stub = makeStubBackend({ dimensions: 8 }); configureSemanticBackend({ @@ -389,15 +399,18 @@ for (const kind of ["blob-flat", "sqlite-vec", "postgres"] as const) { }); const instance = `cin_identity_${kind.replaceAll("-", "_")}_${runId}_${++sequence}`; const connector = `identity-${kind}-${runId}-${sequence}`; - const manifest = { - ...semanticManifest, - connector_id: connector, - display_name: connector, - streams: ["items", "item_", "item%", "itemX"].map((name) => ({ - ...required(semanticManifest.streams[0]), - name, - })), - }; + const manifest = withFixtureSourceDeclaration( + { + ...semanticManifest, + connector_id: connector, + display_name: connector, + streams: ["items", "item_", "item%", "itemX"].map((name) => ({ + ...required(semanticManifest.streams[0]), + name, + })), + }, + { sourceId: baseManifest.manifest_uri } + ); const inputs: string[] = []; const stub = makeStubBackend({ dimensions: 8 }); const embedding = { diff --git a/reference-implementation/test/stream-evidence.test.ts b/reference-implementation/test/stream-evidence.test.ts index aeca2cecd..236e5482d 100644 --- a/reference-implementation/test/stream-evidence.test.ts +++ b/reference-implementation/test/stream-evidence.test.ts @@ -39,6 +39,7 @@ import { } from "../server/ref-control.ts"; import { createRequestConnectorInstanceStore } from "../server/request-store-factories.ts"; import { admitOwnerRunConnection } from "../server/stores/connector-instance-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; const NOT_STATE_STREAM_DECLARED_PATTERN = /which the manifest does not declare with a\s+static state_stream parent/; const OUTCOMES_SUM_MISMATCH_PATTERN = /outcomes.*do not sum to considered/i; @@ -163,41 +164,44 @@ function fakeAdmitRunConnection( * stream used only for the manifest-shape rejection test. */ function streamEvidenceManifest(connectorId: string) { - return { - capabilities: { human_interaction: [] }, - connector_id: connectorId, - display_name: connectorId, - manifest_uri: `https://sources.example/${connectorId}`, - protocol_version: "0.1.0", - streams: [ - { - name: "messages", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - coverage_strategy: "checkpoint_window", - name: "message_bodies", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "append_only", - state_stream: "messages", - }, - { - coverage_strategy: "parent_detail_accounting", - name: "message_attachments", - parent_streams: ["messages"], - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "0.1.0", - }; + return withFixtureSourceDeclaration( + { + capabilities: { human_interaction: [] }, + connector_id: connectorId, + display_name: connectorId, + manifest_uri: `https://sources.example/${connectorId}`, + protocol_version: "0.1.0", + streams: [ + { + name: "messages", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + coverage_strategy: "checkpoint_window", + name: "message_bodies", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "append_only", + state_stream: "messages", + }, + { + coverage_strategy: "parent_detail_accounting", + name: "message_attachments", + parent_streams: ["messages"], + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "0.1.0", + }, + { sourceId: `https://sources.example/${connectorId}` } + ); } /** From 297ac32cd5df79db903e044a348a45d32ff42cfd Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 6 Nov 2026 07:09:53 -0600 Subject: [PATCH 4/5] fix(connector-install): reinstall a published root that has no active record instead of adopting it The previous commit on this branch adopted a published digest root that had no active record: it hashed the files found there into a new record and activated it. That let the bytes on disk vouch for themselves. A root altered between the crash and the retry (the executable, the provenance file or the source-declaration sidecar) became the verified install, and an altered sidecar could be recorded as the pinned declaration. With no record there is no retained verification, so nothing in the root can be trusted. The install now removes the leftover root and installs again from the verified artifact, which is what it already did for a root that failed its checks. The crash window still heals without manual cleanup, and the crash point after the rename stays so the kill-and-restart test reaches it. Tests: the happy-path test is renamed to say a retry reinstalls. Two new tests alter the root after the crash (the executable alone; the executable, provenance, an added sidecar and an extra file) and check that the retry installs the artifact's bytes and nothing the crash left survives. Both failed against the adopting code. Signed-off-by: Tim Nunamaker Assisted-by: AI Signed-off-by: Tim Nunamaker --- .../server/connector-install/index.ts | 16 ++---- ...nnector-install-registration-crash.test.ts | 56 +++++++++++++++++-- 2 files changed, 55 insertions(+), 17 deletions(-) diff --git a/reference-implementation/server/connector-install/index.ts b/reference-implementation/server/connector-install/index.ts index 92bec0625..75e7cc99a 100644 --- a/reference-implementation/server/connector-install/index.ts +++ b/reference-implementation/server/connector-install/index.ts @@ -1013,18 +1013,10 @@ export function createConnectorInstallService(options: { ); } else { // A crash can leave the published directory between rename and the - // active-record commit. A root that still verifies against the catalog - // entry is adopted: activate and register it as an install would. - // Anything else is an invalid leftover and is removed. - let adoptable = true; - try { - readVerifiedRecord(root, entry); - } catch { - adoptable = false; - } - if (adoptable) { - return await reuseExistingInstall(root, entry, null, null, dataDir, options.registerManifest, store, signal); - } + // active-record commit. No record means no retained verification: hashing + // the bytes found here into a new record would make whatever is on disk the + // verified bytes. The leftover is removed and the install below takes the + // bytes from the verified artifact again. removePublishedRootSafely(root, dataDir, connectorId, entry.digest); if (existsSync(root)) { throw new Error("Connector digest root already exists without a matching active record."); diff --git a/reference-implementation/test/connector-install-registration-crash.test.ts b/reference-implementation/test/connector-install-registration-crash.test.ts index 080aa2c1e..065c00035 100644 --- a/reference-implementation/test/connector-install-registration-crash.test.ts +++ b/reference-implementation/test/connector-install-registration-crash.test.ts @@ -26,7 +26,8 @@ */ import assert from "node:assert/strict"; -import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; @@ -464,7 +465,7 @@ test("a retried install after a kill between pointer and registry also restores }); }); -test("killed after the root is published and before the pointer write: a retried install adopts the root", async () => { +test("killed after the root is published and before the pointer write: a retried install reinstalls the root", async () => { await withSandbox(async ({ dataDir, dbPath }) => { const env = { PDPP_DATA_DIR: dataDir }; await killChildAt(CHILD, "connector-install:after-publish", { @@ -475,9 +476,54 @@ test("killed after the root is published and before the pointer write: a retried assert.equal(existsSync(rootOf(dataDir, GMAIL_INSTALL_DIGEST)), true, "the crash left a published root"); assert.equal(await activeRecord(), null, "no pointer was written"); - const adopted = await installService(dataDir).install("gmail", GMAIL_INSTALL_DIGEST); + const installed = await installService(dataDir).install("gmail", GMAIL_INSTALL_DIGEST); - assert.equal(adopted.digest, GMAIL_INSTALL_DIGEST); - assert.equal((await agreement()).agree, true, "the adopted root is active and registered"); + assert.equal(installed.digest, GMAIL_INSTALL_DIGEST); + assert.equal((await agreement()).agree, true, "the reinstalled root is active and registered"); + }); +}); + +// A root with no active record has no retained verification, so nothing it contains can vouch for +// itself. Hashing its bytes into a fresh record would make the altered bytes the verified ones, so +// the install removes the leftover and takes the bytes from the verified artifact again. +test("a root left by a crash and then altered is not adopted: the retry installs the verified bytes", async () => { + await withSandbox(async ({ dataDir, dbPath }) => { + const env = { PDPP_DATA_DIR: dataDir }; + await killChildAt(CHILD, "connector-install:after-publish", { + args: [dbPath, dataDir, "install"], + env, + }); + await initDb(dbPath); + const root = rootOf(dataDir, GMAIL_INSTALL_DIGEST); + writeFileSync(join(root, "dist", "collection-profile.mjs"), "process.exit(1);\n"); + writeFileSync(join(root, "provenance.json"), '{"altered":true}\n'); + writeFileSync(join(root, "source-declaration.json"), '{"altered":true}'); + writeFileSync(join(root, "planted-extra.mjs"), "export {};\n"); + + const installed = await installService(dataDir).install("gmail", GMAIL_INSTALL_DIGEST); + + assert.equal(readFileSync(join(installed.root, "dist", "collection-profile.mjs"), "utf8"), "export {};\n"); + assert.equal(readFileSync(join(installed.root, "provenance.json"), "utf8"), "{}\n"); + assert.equal(existsSync(join(installed.root, "source-declaration.json")), false, "an altered sidecar is gone"); + assert.equal(existsSync(join(installed.root, "planted-extra.mjs")), false, "nothing the crash left survives"); + assert.equal(installed.sourceDeclarationSha256, undefined, "no sidecar digest comes from observed bytes"); + assert.equal((await agreement()).agree, true); + }); +}); + +test("a root left by a crash with only its executable altered is not adopted", async () => { + await withSandbox(async ({ dataDir, dbPath }) => { + const env = { PDPP_DATA_DIR: dataDir }; + await killChildAt(CHILD, "connector-install:after-publish", { + args: [dbPath, dataDir, "install"], + env, + }); + await initDb(dbPath); + writeFileSync(join(rootOf(dataDir, GMAIL_INSTALL_DIGEST), "dist", "collection-profile.mjs"), "process.exit(1);\n"); + + const installed = await installService(dataDir).install("gmail", GMAIL_INSTALL_DIGEST); + + assert.equal(readFileSync(join(installed.root, "dist", "collection-profile.mjs"), "utf8"), "export {};\n"); + assert.equal(installed.entrypointSha256, `sha256:${createHash("sha256").update("export {};\n").digest("hex")}`); }); }); From a4fb70dfdde3d43d1da243b39947b1190b34304c Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 6 Nov 2026 07:10:04 -0600 Subject: [PATCH 5/5] fix(connector-install): reject a repeated object key in an installed Collection Profile An installed profile was read with JSON.parse, which keeps the last value of a repeated key. A bindings map that declared `network` twice reached registration as one binding with the second value. The HTTP and local source paths already reject a repeat (parseManifestJsonText), so one declaration was accepted or refused depending on how it arrived. Both reads of the profile in the installer, at install time and when an active record is verified, now use parseManifestJsonText on the text as written. The error says only that the profile repeats an object key: the text is not trusted and the error can reach the owner, so it carries no key or path. Tests: a repeat at the bindings level, a repeat nested in one binding, and an escaped spelling of the same key are each refused at install with nothing active, nothing registered and the root removed. A record whose profile bytes repeat a key under their own recorded hash is not runnable. All four failed before. Signed-off-by: Tim Nunamaker Assisted-by: AI Signed-off-by: Tim Nunamaker --- .../server/connector-install/index.ts | 21 +++++- .../test/connector-install.test.ts | 69 +++++++++++++++++++ 2 files changed, 88 insertions(+), 2 deletions(-) diff --git a/reference-implementation/server/connector-install/index.ts b/reference-implementation/server/connector-install/index.ts index 75e7cc99a..ec7afce8e 100644 --- a/reference-implementation/server/connector-install/index.ts +++ b/reference-implementation/server/connector-install/index.ts @@ -25,6 +25,7 @@ import { dirname, join, resolve, sep } from "node:path"; import { isDeepStrictEqual } from "node:util"; import { crashPoint } from "../crash-point.ts"; import { canonicalConnectorKey } from "../connector-key.ts"; +import { BindingDeclarationError, parseManifestJsonText } from "../../runtime/binding-declarations.ts"; import { snapshotSourceDeclaration } from "../source-declaration.ts"; import { assertConnectorLifecycleAvailable, @@ -581,6 +582,22 @@ export async function resolveActiveConnectorPath( }); } +/** + * Parses a Collection Profile from the text on disk. A repeated object key is rejected, as on the HTTP and + * local-source paths: `JSON.parse` alone keeps the last value and the repeat is gone before registration. The + * error names no key or path, because the profile text is not trusted and an install error reaches the owner. + */ +function parseProfileText(text: string): unknown { + try { + return parseManifestJsonText(text); + } catch (error) { + if (error instanceof BindingDeclarationError) { + throw new Error("Connector profile repeats an object key."); + } + throw new Error("Connector profile is not valid JSON."); + } +} + function verifyStoredRecord(root: string, active: ConnectorInstallRecord, dataDir?: string): ConnectorInstallRecord { assertStoredRecordMetadata(active); const normalizedRoot = resolve(root); @@ -595,7 +612,7 @@ function verifyStoredRecord(root: string, active: ConnectorInstallRecord, dataDi if (!manifestFile) { throw new Error("Active connector manifest path is invalid."); } - const manifest = JSON.parse(readFileSync(manifestFile, "utf8")) as Record; + const manifest = parseProfileText(readFileSync(manifestFile, "utf8")) as Record; const manifestKey = canonicalManifestKey(manifest); if (manifestKey !== active.connectorId) { throw new Error("Active connector manifest identity does not match its record."); @@ -716,7 +733,7 @@ function readVerifiedRecord( if (lstatSync(manifestFile).size > MAX_CONFIG_BYTES) { throw new Error("Connector profile exceeds the 1 MiB RI limit."); } - const manifest: unknown = JSON.parse(readFileSync(manifestFile, "utf8")); + const manifest: unknown = parseProfileText(readFileSync(manifestFile, "utf8")); if (!manifest || typeof manifest !== "object") { throw new Error("Connector profile is invalid."); } diff --git a/reference-implementation/test/connector-install.test.ts b/reference-implementation/test/connector-install.test.ts index 5c76c5a91..cf1a387f8 100644 --- a/reference-implementation/test/connector-install.test.ts +++ b/reference-implementation/test/connector-install.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; import { existsSync, mkdirSync, @@ -622,6 +623,74 @@ test("a second store handle cannot create a source while uninstall owns the data } }); +const DUPLICATE_BINDING_PROFILES = { + escaped: + '{"connector_id":"github","version":"1.0.0","runtime_requirements":{"bindings":{"network":{"required":true},"n\\u0065twork":{"required":false}}}}', + nested: + '{"connector_id":"github","version":"1.0.0","runtime_requirements":{"bindings":{"network":{"required":true,"required":false}}}}', + plain: + '{"connector_id":"github","version":"1.0.0","runtime_requirements":{"bindings":{"network":{"required":true},"network":{"required":false}}}}', +} as const; +const RE_PROFILE_REPEATS_KEY = /Connector profile repeats an object key\./; + +for (const [name, profileText] of Object.entries(DUPLICATE_BINDING_PROFILES)) { + test(`an installed profile with a repeated object key (${name}) is refused, as the HTTP and local paths refuse it`, async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-connector-install-")); + try { + const store = createFileConnectorInstallStore(dataDir); + const registered: unknown[] = []; + const service = createConnectorInstallService({ + catalogLoader: async () => [entry], + dataDir, + installArtifact: (root) => { + writeFixture(root); + writeFileSync(join(root, "profile", "collection-profile.json"), profileText); + }, + registerManifest: (manifest) => { + registered.push(manifest); + return Promise.resolve(); + }, + store, + }); + await assert.rejects(service.install("github", digest), RE_PROFILE_REPEATS_KEY); + assert.equal(await store.getActive("github"), null, "nothing became active"); + assert.deepEqual(registered, [], "nothing was registered, so no last-wins value reached the registry"); + assert.equal(existsSync(join(dataDir, "connectors", "github", digest)), false, "the published root is removed"); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } + }); +} + +test("an active record whose profile bytes repeat an object key is not runnable", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-connector-install-")); + try { + const store = createFileConnectorInstallStore(dataDir); + const service = createConnectorInstallService({ + catalogLoader: async () => [entry], + dataDir, + installArtifact: (root) => { + writeFixture(root); + }, + registerManifest: () => Promise.resolve(), + store, + }); + const active = await service.install("github", digest); + // Re-record the altered profile under its own hash, so only the repeated key is wrong. + const profile = join(active.root, "profile", "collection-profile.json"); + writeFileSync(profile, DUPLICATE_BINDING_PROFILES.plain); + const manifestSha256 = `sha256:${createHash("sha256").update(readFileSync(profile)).digest("hex")}`; + await store.activate({ ...active, manifestSha256 }); + + const inspected = await inspectActiveConnector(store, "github"); + assert.equal(inspected.status, "invalid"); + assert.match(inspected.status === "invalid" ? inspected.reason : "", RE_PROFILE_REPEATS_KEY); + assert.equal(await resolveActiveConnectorPath(store, "github"), null); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + test("a corrupted active entrypoint fails closed", async () => { const dataDir = mkdtempSync(join(tmpdir(), "pdpp-connector-install-")); try {