diff --git a/PG-PROFILE-51-REPORT.md b/PG-PROFILE-51-REPORT.md index 1c29ee3c9..82acc66a1 100644 --- a/PG-PROFILE-51-REPORT.md +++ b/PG-PROFILE-51-REPORT.md @@ -168,7 +168,7 @@ tail: The completed profile receipt recorded 1,034 files; that historical receipt is unchanged. The template equivalence registry is a different measure: -159 eligible plus 37 cold-required files, for **196** total. The +162 eligible plus 37 cold-required files, for **199** total. The handle-generation, issuance-cutover, opaque-cursor and search-index repair tests remain registered. These counts are recomputed from the merged registry arrays. diff --git a/reference-implementation/operations/as-polyfill-connector-register/index.ts b/reference-implementation/operations/as-polyfill-connector-register/index.ts index 8ae884a99..dd4b547bc 100644 --- a/reference-implementation/operations/as-polyfill-connector-register/index.ts +++ b/reference-implementation/operations/as-polyfill-connector-register/index.ts @@ -33,8 +33,14 @@ export interface AsPolyfillConnectorRegisterDependencies { * whether or not that version is still current; null when none was recorded. */ getAcceptedDeclarationDigest: (connectorId: string, version: string) => Promise | string | null; - /** Produces the exact representation `registerConnector` persists, and its storage key. */ - normalizeManifestForStorage: (manifest: Record) => { + /** + * Produces the exact representation `registerConnector` persists, and its storage key. `context` carries + * the declaration already retained for the connector, when there is one. + */ + normalizeManifestForStorage: ( + manifest: Record, + context: { retainedSourceDeclaration?: unknown } + ) => { connectorId: string; storedManifest: Record; }; @@ -100,7 +106,13 @@ export async function executeAsPolyfillConnectorRegister( // equivocation and the previously accepted content is retained. Same // content is an idempotent re-registration; a new version is a new key. deps.validateManifest(manifest as Record); - const { connectorId, storedManifest } = deps.normalizeManifestForStorage(manifest as Record); + // The candidate is projected under the identity the stored row already holds, exactly as registerConnector + // does. Without it, a manifest that names no source would be refused on the fresh path. + const retained = await acceptedManifestFor(deps, connectorKey); + const { connectorId, storedManifest } = deps.normalizeManifestForStorage( + manifest as Record, + retained?.source_declaration === undefined ? {} : { retainedSourceDeclaration: retained.source_declaration } + ); const version = storedManifest.version; const incomingDigest = deps.declarationDigest(storedManifest); let acceptedDigest: string | null = null; diff --git a/reference-implementation/runtime/binding-declarations.ts b/reference-implementation/runtime/binding-declarations.ts new file mode 100644 index 000000000..5c860337d --- /dev/null +++ b/reference-implementation/runtime/binding-declarations.ts @@ -0,0 +1,425 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +// Collection Profile `runtime_requirements.bindings`: one shared validator for +// the registration gate, the stored-manifest read and the direct `runConnector` +// path, plus the pre-spawn placement plan that consumes its result. +// +// Normative sources (data-connectors a87edcdc099284e8582d24a60c9630474bd2af8b): +// docs/spec/collection-profile.md section 3.3 (registry, instance keys, +// interface, duplicate keys), 3.3.2-3.3.7 (constraint grammars, features, +// rationale), 3.3.8 (features), 3.3.9 (lifecycle outcome), and 3.4 +// (capabilities.human_interaction). The wire grammar is the canonical +// `connector-manifest.schema.json` and `connector-binding-grammar.mjs` beside +// this file, byte copies of the pinned data-connectors files. Nothing here +// re-derives their host, endpoint, IDNA or IPv6 rules. +// +// Scope. This module decides two things: whether a declaration is well formed, +// and whether this runtime can supply each required contract at all. It does +// not decide authority. Passing validation is never a grant, never enforcement, +// and never a claim that a channel is `enforced` (section 3.3.10). The +// effective-authority record, per-channel coverage and the weaker-coverage +// policy source are not decided here. + +import { Ajv2020, type ValidateFunction } from "ajv/dist/2020.js"; + +import { + canonicalizeConnectHost, + canonicalizeNetworkHost, + canonicalizeWebHost, + findDuplicateJsonKeys, + GrammarError, + normalizeFilesystemInputs, +} from "./connector-binding-grammar.mjs"; +import manifestSchema from "./connector-manifest.schema.json" with { type: "json" }; + +// --------------------------------------------------------------------------- +// Errors +// --------------------------------------------------------------------------- + +/** A declaration or manifest text that the canonical rules reject. */ +export class BindingDeclarationError extends Error { + readonly code: "invalid_binding_declaration" | "duplicate_json_key"; + + constructor(message: string, code: BindingDeclarationError["code"] = "invalid_binding_declaration") { + super(message); + this.name = "BindingDeclarationError"; + this.code = code; + } +} + +/** A required binding this runtime cannot supply. Refused before spawn. */ +export class BindingPlacementError extends Error { + readonly code = "unsupported_required_binding" as const; + readonly bindingKeys: readonly string[]; + + constructor(failures: readonly { key: string; reason: string }[]) { + const detail = failures.map((failure) => `${failure.key} (${failure.reason})`).join("; "); + super(`Runtime cannot satisfy required binding: ${failures.map((failure) => failure.key).join(", ")}. ${detail}`); + this.name = "BindingPlacementError"; + this.bindingKeys = failures.map((failure) => failure.key); + } +} + +// --------------------------------------------------------------------------- +// Canonical registry and grammar, read from the vendored schema +// --------------------------------------------------------------------------- + +/** Registry binding names (section 3.3). Read from the canonical schema. */ +export const REGISTRY_BINDING_NAMES: ReadonlySet = new Set(manifestSchema.$defs.registryBinding.enum); + +/** Namespaced extension binding name `/` (section 3.3). */ +const NAMESPACED_BINDING_NAME = new RegExp(namespacedBindingPattern()); + +function namespacedBindingPattern(): string { + const branch = manifestSchema.$defs.bindingName.anyOf.find((entry) => "pattern" in entry) as + | { pattern: string } + | undefined; + if (branch === undefined) { + throw new Error("connector-manifest.schema.json: bindingName has no namespaced pattern"); + } + return branch.pattern; +} + +// --------------------------------------------------------------------------- +// Raw JSON text +// --------------------------------------------------------------------------- + +/** + * Parses manifest JSON text, rejecting any object key that repeats anywhere in + * the text, including nested keys and escaped spellings of the same key. The + * duplicate check runs on the text as written, before `JSON.parse` collapses + * repeats to the last value. Re-serializing an already-parsed object cannot + * recover this, so callers must pass the original text when they have it. + */ +export function parseManifestJsonText(text: string): unknown { + let duplicates: Array<{ path: string; key: string }>; + try { + duplicates = findDuplicateJsonKeys(text); + } catch (error) { + if (error instanceof GrammarError) { + throw new SyntaxError(`manifest JSON is malformed: ${error.message}`, { cause: error }); + } + throw error; + } + const [first] = duplicates; + if (first !== undefined) { + throw new BindingDeclarationError( + `manifest JSON repeats object key "${first.key}" at ${first.path}; a repeated key is rejected, not last-wins`, + "duplicate_json_key" + ); + } + return JSON.parse(text); +} + +// --------------------------------------------------------------------------- +// Shape validation (canonical JSON Schema) and canonical grammar checks +// --------------------------------------------------------------------------- + +export interface BindingInstance { + /** Deep copy of the declaration. Binding-specific fields are preserved. */ + readonly declaration: Readonly>; + /** Stated interface version, exactly as declared. Absent means unspecified. */ + readonly interface: string | undefined; + /** Instance key exactly as declared. */ + readonly key: string; + /** Resolved kind: the key for shorthand, `kind` for a free-form key. */ + readonly kind: string; + readonly required: boolean; +} + +let schemaValidator: ValidateFunction | null = null; + +function bindingsValidator(): ValidateFunction { + if (schemaValidator === null) { + schemaValidator = new Ajv2020({ allErrors: false, strict: true }).compile(manifestSchema); + } + return schemaValidator; +} + +const RUNTIME_REQUIREMENTS_BINDINGS_PREFIX = /^\/runtime_requirements\/bindings/; + +function describeSchemaError(error: { instancePath: string; message?: string }): string { + const relative = error.instancePath.replace(RUNTIME_REQUIREMENTS_BINDINGS_PREFIX, "").replaceAll("/", "."); + return `runtime_requirements.bindings${relative} ${error.message ?? "is invalid"}`; +} + +const REGEX_SPECIAL_CHARACTERS = /[.*+?^${}()|[\]\\]/g; + +function escapeRegExp(value: string): string { + return value.replace(REGEX_SPECIAL_CHARACTERS, "\\$&"); +} + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function fail(message: string): never { + throw new BindingDeclarationError(message); +} + +/** Canonical host check: the stated form must equal its own canonicalization. */ +function assertCanonicalHost(label: string, raw: unknown, canonicalize: (value: string) => string): void { + if (typeof raw !== "string") { + return; + } + let canonical: string; + try { + canonical = canonicalize(raw); + } catch (error) { + const reason = error instanceof Error ? error.message : String(error); + fail(`${label} "${raw}" is not a valid canonical form: ${reason}`); + } + if (canonical !== raw) { + fail(`${label} "${raw}" must be stated in canonical form "${canonical}"`); + } +} + +function assertNonEmptyRationaleWhenExceptional( + key: string, + declaration: Record, + exceptional: boolean +): void { + const { rationale } = declaration; + if (rationale !== undefined && (typeof rationale !== "string" || rationale.trim().length === 0)) { + fail(`runtime_requirements.bindings.${key}.rationale must be a non-empty string when declared`); + } + if (exceptional && (typeof rationale !== "string" || rationale.trim().length === 0)) { + fail( + `runtime_requirements.bindings.${key} declares an exceptional grant and must carry a non-empty rationale (section 3.3.13)` + ); + } +} + +/** The interface version must be `@` with no leading zero. */ +function assertInterfaceVersion(key: string, kind: string, declaration: Record): void { + const iface = declaration.interface; + if (iface === undefined) { + return; + } + const expected = new RegExp(`^${escapeRegExp(kind)}@[1-9][0-9]*$`); + if (typeof iface !== "string" || !expected.test(iface)) { + fail(`runtime_requirements.bindings.${key}.interface must be "${kind}@" with no leading zero`); + } +} + +/** Both declared filesystem forms must agree (section 3.3.5), and env_var names must be unique. */ +function assertFilesystemInputs(key: string, declaration: Record): void { + let inputs: unknown[] | undefined; + try { + inputs = normalizeFilesystemInputs(declaration as { inputs?: unknown[]; constraints?: { inputs?: unknown[] } }); + } catch (error) { + fail(`runtime_requirements.bindings.${key}: ${error instanceof Error ? error.message : String(error)}`); + } + const envVars = new Set(); + for (const input of inputs ?? []) { + const envVar = isRecord(input) ? input.env_var : undefined; + if (typeof envVar === "string") { + if (envVars.has(envVar)) { + fail(`runtime_requirements.bindings.${key} repeats filesystem input env_var ${envVar}`); + } + envVars.add(envVar); + } + } +} + +/** + * Canonical-form checks for browser and network host constraints. Returns true + * when the declaration is an exceptional grant (section 3.3.13). + */ +function assertHostConstraints(kind: string, constraints: Record): boolean { + if (kind === "browser") { + for (const [index, host] of ((constraints.navigate as unknown[] | undefined) ?? []).entries()) { + assertCanonicalHost(`browser.constraints.navigate[${index}]`, host, canonicalizeWebHost); + } + for (const [index, host] of ((constraints.connect as unknown[] | undefined) ?? []).entries()) { + assertCanonicalHost(`browser.constraints.connect[${index}]`, host, canonicalizeConnectHost); + } + return false; + } + if (kind !== "network") { + return false; + } + let exceptional = false; + const hosts = (constraints.hosts as unknown[] | undefined) ?? []; + for (const [index, entry] of hosts.entries()) { + if (isRecord(entry)) { + if (entry.allow_private === true) { + exceptional = true; + } + continue; + } + assertCanonicalHost(`network.constraints.hosts[${index}]`, entry, canonicalizeNetworkHost); + } + return exceptional; +} + +/** Constraint, interface and rationale checks the JSON Schema cannot express. */ +function assertInstanceContract(key: string, kind: string, declaration: Record): void { + assertInterfaceVersion(key, kind, declaration); + const constraints = isRecord(declaration.constraints) ? declaration.constraints : {}; + if (kind === "filesystem") { + assertFilesystemInputs(key, declaration); + } + let exceptional = assertHostConstraints(kind, constraints); + if (kind === "desktop_session" && Array.isArray(constraints.items) && constraints.items.length > 0) { + // Access to a named sensitive resource is an exceptional grant (section 3.3.13). + exceptional = true; + } + assertNonEmptyRationaleWhenExceptional(key, declaration, exceptional); +} + +/** + * Validates `runtime_requirements.bindings` and resolves each instance. + * + * Absent (`undefined`) bindings yield an empty list. `null` is rejected: the + * canonical schema types the map as an object. The result keeps declaration + * order, one entry per instance key, and never mutates its input. + */ +export function normalizeBindingDeclarations(bindings: unknown): readonly BindingInstance[] { + if (bindings === undefined) { + return []; + } + const validate = bindingsValidator(); + if (!validate({ runtime_requirements: { bindings } })) { + const first = validate.errors?.[0]; + fail(first ? describeSchemaError(first) : "runtime_requirements.bindings is invalid"); + } + if (!isRecord(bindings)) { + fail("runtime_requirements.bindings must be an object"); + } + + return Object.entries(bindings).map(([key, value]) => { + if (!isRecord(value)) { + fail(`runtime_requirements.bindings.${key} must be an object`); + } + const shorthand = REGISTRY_BINDING_NAMES.has(key) || NAMESPACED_BINDING_NAME.test(key); + // The schema has already rejected a shorthand key that carries `kind`, and + // a free-form key without one. Here the kind is resolved from the key. + const kind = shorthand ? key : (value.kind as string); + assertInstanceContract(key, kind, value); + return Object.freeze({ + declaration: Object.freeze(structuredClone(value)), + interface: typeof value.interface === "string" ? value.interface : undefined, + key, + kind, + required: value.required as boolean, + }); + }); +} + +// --------------------------------------------------------------------------- +// Pre-spawn placement +// --------------------------------------------------------------------------- + +/** + * What this runtime supplies. It publishes no feature and enforces no + * constraint, so any declared feature or constraint is an unmet contract. + * `kinds` matches the static START.bindings advertisement in runtime/index.ts. + */ +export interface BindingSupport { + readonly constraintKinds: ReadonlySet; + readonly features: ReadonlySet; + readonly interfaceVersions: ReadonlyMap>; + readonly kinds: ReadonlySet; +} + +export const RUNTIME_BINDING_SUPPORT: BindingSupport = { + constraintKinds: new Set(), + features: new Set(), + interfaceVersions: new Map([ + ["browser", new Set([1])], + ["filesystem", new Set([1])], + ["network", new Set([1])], + ]), + kinds: new Set(["browser", "filesystem", "network"]), +}; + +/** Why an instance cannot be supplied, or `null` when the runtime supplies its contract. */ +export function unsupportedReason(instance: BindingInstance, support: BindingSupport): string | null { + if (!support.kinds.has(instance.kind)) { + return `kind ${instance.kind} is not implemented by this runtime`; + } + if (instance.interface !== undefined) { + const version = Number(instance.interface.slice(instance.interface.indexOf("@") + 1)); + if (!support.interfaceVersions.get(instance.kind)?.has(version)) { + return `interface ${instance.interface} is not implemented by this runtime`; + } + } + const { features } = instance.declaration; + if (Array.isArray(features) && features.length > 0) { + const unmet = features.filter((feature) => !support.features.has(String(feature))); + if (unmet.length > 0) { + return `features ${unmet.join(", ")} are not provided by this runtime`; + } + } + const { constraints } = instance.declaration; + const declaresConstraints = isRecord(constraints) && Object.keys(constraints).length > 0; + const declaresInputs = Array.isArray(instance.declaration.inputs) && instance.declaration.inputs.length > 0; + if ((declaresConstraints || declaresInputs) && !support.constraintKinds.has(instance.kind)) { + return "constraints are declared, and this runtime does not enforce them"; + } + if (instance.key !== instance.kind) { + return "named binding instances need per-instance handles, which this runtime does not provide"; + } + return null; +} + +export interface BindingPlacement { + /** Optional instances this runtime omits, with the reason. */ + readonly omitted: readonly { instance: BindingInstance; reason: string }[]; + /** Instances whose contract this runtime supplies. */ + readonly placed: readonly BindingInstance[]; +} + +/** + * Plans placement before spawn. A required instance this runtime cannot supply + * throws `BindingPlacementError`, and no connector code runs. An optional one + * may be omitted (section 3.3), and the omission must show in START. + */ +export function planBindingPlacement( + instances: readonly BindingInstance[], + support: BindingSupport = RUNTIME_BINDING_SUPPORT +): BindingPlacement { + const placed: BindingInstance[] = []; + const omitted: { instance: BindingInstance; reason: string }[] = []; + const failures: { key: string; reason: string }[] = []; + for (const instance of instances) { + const reason = unsupportedReason(instance, support); + if (reason === null) { + placed.push(instance); + } else if (instance.required) { + failures.push({ key: instance.key, reason }); + } else { + omitted.push({ instance, reason }); + } + } + if (failures.length > 0) { + throw new BindingPlacementError(failures); + } + return { omitted, placed }; +} + +/** + * START.bindings for this run keeps the keys of resolved instances. An omitted + * instance cannot borrow another instance's availability through their shared + * kind. Undeclared kinds keep the existing static availability advertisement; + * that advertisement is not a grant or a claim of channel enforcement. + */ +export function startBindingsFor( + instances: readonly BindingInstance[], + plan: BindingPlacement, + support: BindingSupport = RUNTIME_BINDING_SUPPORT +): Record> { + const declaredKinds = new Set(instances.map((instance) => instance.kind)); + const advertised: Record> = {}; + for (const kind of [...support.kinds].sort()) { + if (!declaredKinds.has(kind)) { + advertised[kind] = {}; + } + } + for (const instance of plan.placed) { + advertised[instance.key] = {}; + } + return advertised; +} diff --git a/reference-implementation/runtime/classify-runtime-failure.ts b/reference-implementation/runtime/classify-runtime-failure.ts index 9a61778dc..4dd615b03 100644 --- a/reference-implementation/runtime/classify-runtime-failure.ts +++ b/reference-implementation/runtime/classify-runtime-failure.ts @@ -31,7 +31,8 @@ export function classifyRuntimeFailure(err: unknown): FailureReason { } if ( message === "Connector emitted INTERACTION while already waiting" || - message === "Connector emitted INTERACTION but START.bindings omitted interactive" || + message === "Connector emitted INTERACTION but no interaction handler is configured" || + message.startsWith("Connector emitted INTERACTION.kind not declared in capabilities.human_interaction:") || message.includes(" while waiting for INTERACTION_RESPONSE") || message.startsWith("Connector emitted invalid INTERACTION.") || message.startsWith("Connector emitted invalid STATE.") || diff --git a/reference-implementation/runtime/collection-profile-support.ts b/reference-implementation/runtime/collection-profile-support.ts index a00996ed4..a567194ab 100644 --- a/reference-implementation/runtime/collection-profile-support.ts +++ b/reference-implementation/runtime/collection-profile-support.ts @@ -1,8 +1,9 @@ // Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 -// Admission of a manifest's Collection Profile declaration on the run path -// (runtime/index.ts). +// Admission of a manifest's Collection Profile declaration, shared by the +// registration gate (server/connector-manifest-validation.ts) and the run +// path (runtime/index.ts) so both apply one rule set. // // Normative source: data-connectors docs spec sections 3.4 (protocol_version // and capabilities) and 3.8 (diagnostic_descriptors). A manifest that declares @@ -12,9 +13,25 @@ import { COLLECTION_PROFILE_VERSIONS, collectionProfileDeclarationError, OBSERVATION_CAPABILITY, + STREAM_EVIDENCE_CAPABILITY, } from "@pdpp/connector-protocol/connector-runtime-protocol"; import { type DiagnosticDescriptors, ObservationContractError, validateDiagnosticDescriptors } from "./observations.ts"; +/** Implemented wire features, with outstanding full-profile obligations disclosed. */ +export const REFERENCE_COLLECTION_PROFILE_SUPPORT = Object.freeze({ + compatible_protocol_versions: Object.freeze([...COLLECTION_PROFILE_VERSIONS]), + conformance: Object.freeze({ outstanding: Object.freeze(["binding_authority"]), status: "partial" }), + protocol_capabilities: Object.freeze([OBSERVATION_CAPABILITY, STREAM_EVIDENCE_CAPABILITY]), + protocol_version: "0.2.0", +}); + +/** + * `failure_reason` of a run refused before spawn because the manifest's + * Collection Profile declaration or required capabilities cannot be honored + * (Section 3.4: the runtime must not start that connector). + */ +export const PROTOCOL_CAPABILITY_UNSUPPORTED_FAILURE_REASON = "protocol_capability_unsupported"; + export interface CollectionProfileDeclaration { readonly descriptors: DiagnosticDescriptors; readonly observationDeclared: boolean; diff --git a/reference-implementation/runtime/connector-binding-grammar.d.mts b/reference-implementation/runtime/connector-binding-grammar.d.mts new file mode 100644 index 000000000..19da639d4 --- /dev/null +++ b/reference-implementation/runtime/connector-binding-grammar.d.mts @@ -0,0 +1,22 @@ +// Type declarations for the vendored `connector-binding-grammar.mjs`, a byte +// copy of data-connectors schemas/connector-binding-grammar.mjs at commit +// a87edcdc099284e8582d24a60c9630474bd2af8b. Declarations mirror the exports as +// they are written in that module; they add no behaviour. + +export class GrammarError extends Error {} + +export function canonicalizeWebHost(raw: string, options?: { schemes?: Set }): string; +export function isCanonicalWebHost(raw: string, options?: { schemes?: Set }): boolean; +export function hostMatchesWildcardApex(apex: string, candidateHost: string): boolean; +export function canonicalizeConnectHost(raw: string): string; +export function isCanonicalConnectHost(raw: string): boolean; +export function canonicalizeEndpoint(raw: string): string; +export function isCanonicalEndpoint(raw: string): boolean; +export function canonicalizeNetworkHost(raw: string): string; +export function isCanonicalNetworkHost(raw: string): boolean; +export function deepEqualJson(a: unknown, b: unknown): boolean; +export function normalizeFilesystemInputs(instance: { + inputs?: unknown[]; + constraints?: { inputs?: unknown[] }; +}): unknown[] | undefined; +export function findDuplicateJsonKeys(jsonText: string): Array<{ path: string; key: string }>; diff --git a/reference-implementation/runtime/connector-binding-grammar.mjs b/reference-implementation/runtime/connector-binding-grammar.mjs new file mode 100644 index 000000000..c6c7dbf09 --- /dev/null +++ b/reference-implementation/runtime/connector-binding-grammar.mjs @@ -0,0 +1,534 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Typed parsers and canonicalizers for the binding constraint grammars + * defined in docs/spec/collection-profile.md Section 3.3. JSON Schema can + * check shape (strings, required members, enums) but cannot perform IDNA + * canonicalization, IPv6 normalization, or "is this string already in its + * one canonical form" equivalence checks. This module does that work, and + * schemas/connector-manifest.schema.json's string patterns are deliberately + * written to accept only the output these functions already produce. + * + * Node's `url.domainToASCII` was evaluated for the IDNA/UTS46 step. This + * module uses the WHATWG `URL` constructor instead, because the same + * host-parsing algorithm that backs `domainToASCII` also normalizes IPv4 + * and IPv6 literals and drops a scheme's default port, so one call covers + * every canonicalization rule in the web-origin grammar instead of hand + * -rolling IPv6 compression or IPv4 interpretation separately. Both paths + * share the same underlying WHATWG URL Standard host parser, so the result + * is the one `domainToASCII` would give for the host component alone. + * + * Grammar (docs/spec/collection-profile.md Section 3.3.2): + * `[scheme://]host[:port]`, exact or one leading `*.` wildcard label, no + * path, query, or fragment. Section 3.3.3 (browser) and 3.3.4 (network) layer + * their own scheme sets and endpoint rules on top of this shared grammar. + */ + +import { URL } from "node:url"; +import net from "node:net"; + +export class GrammarError extends Error { + constructor(message) { + super(message); + this.name = "GrammarError"; + } +} + +// Schemes `browser.navigate` accepts: a navigation always loads a document. +const NAVIGATE_SCHEMES = new Set(["http", "https"]); + +// Schemes `browser.connect` accepts: a page can open a WebSocket from a +// document loaded over http(s), so `connect` is scheme-aware in a way +// `navigate` is not (Section 3.3.3, live capture evidence from chatgpt: +// wss://ws.chatgpt.com, wss://codex-cloud-backend.chatgpt.com). +const CONNECT_SCHEMES = new Set(["http", "https", "ws", "wss"]); + +// network.hosts without an explicit scheme, and any bare http(s) entry, +// reuses the same grammar as `navigate` (Section 3.3.4). +const WEB_SCHEMES = NAVIGATE_SCHEMES; + +// Well-known default ports, used to drop a redundant explicit port from the +// canonical form. Only schemes with one conventional default port are +// listed; an endpoint whose scheme is not here must always state its port +// (Section 3.3.4). ws/wss share http/https's ports because a WebSocket +// upgrade begins as an http(s) request. +// Object.create(null) has no prototype chain, so a scheme named +// "constructor", "__proto__", "toString", or "hasOwnProperty" cannot read an +// inherited value through `DEFAULT_PORTS[scheme]` and be mistaken for a +// scheme with a real default port. A plain object literal is vulnerable to +// exactly this (schemas/connector-binding-grammar.test.mjs tests it). +const DEFAULT_PORTS = Object.assign(Object.create(null), { + http: 80, + https: 443, + ws: 80, + wss: 443, + imap: 143, + imaps: 993, + pop3: 110, + pop3s: 995, + smtp: 25, + smtps: 465, + submission: 587, + ldap: 389, + ldaps: 636, + ftp: 21, + ftps: 990, +}); + +function assertNoWhitespaceOrControl(raw) { + if (typeof raw !== "string" || raw.length === 0) { + throw new GrammarError("must be a non-empty string"); + } + for (let i = 0; i < raw.length; i += 1) { + const code = raw.charCodeAt(i); + if (code <= 0x20 || code === 0x7f) { + throw new GrammarError("must not contain whitespace or control characters"); + } + } +} + +function splitScheme(raw) { + const match = raw.match(/^([A-Za-z][A-Za-z0-9+.-]*):\/\//); + if (!match) return { scheme: null, rest: raw }; + return { scheme: match[1].toLowerCase(), rest: raw.slice(match[0].length) }; +} + +function splitWildcard(rest) { + if (rest.startsWith("*.")) { + const apex = rest.slice(2); + if (apex.includes("*")) { + throw new GrammarError("a wildcard must be exactly one leading '*.' label"); + } + return { wildcard: true, rest: apex }; + } + if (rest.includes("*")) { + throw new GrammarError("a wildcard must be exactly one leading '*.' label"); + } + return { wildcard: false, rest }; +} + +function rejectPathQueryFragment(rest) { + if (/[/?#]/.test(rest)) { + throw new GrammarError("must not contain a path, query, or fragment"); + } +} + +function rejectUserinfo(rest) { + if (rest.includes("@")) { + throw new GrammarError("must not contain userinfo; this grammar has no '@' component"); + } +} + +/** + * Splits `rest` (no scheme, no wildcard, no path) into a host token and an + * optional port token, respecting bracketed IPv6 literals. + */ +function splitHostPort(rest) { + if (rest.startsWith("[")) { + const end = rest.indexOf("]"); + if (end === -1) throw new GrammarError("unterminated IPv6 literal"); + const host = rest.slice(0, end + 1); + const tail = rest.slice(end + 1); + if (tail.length === 0) return { host, port: null, isIPv6Literal: true }; + if (!tail.startsWith(":")) throw new GrammarError("unexpected characters after an IPv6 literal"); + return { host, port: tail.slice(1), isIPv6Literal: true }; + } + const colonIdx = rest.lastIndexOf(":"); + if (colonIdx === -1) return { host: rest, port: null, isIPv6Literal: false }; + return { host: rest.slice(0, colonIdx), port: rest.slice(colonIdx + 1), isIPv6Literal: false }; +} + +function validatePort(port) { + if (!/^[0-9]+$/.test(port)) throw new GrammarError("port must be digits only"); + const value = Number(port); + if (!Number.isInteger(value) || value < 1 || value > 65535 || String(value) !== port) { + throw new GrammarError("port must be a canonical integer from 1 to 65535"); + } + return value; +} + +function stripTrailingDot(host) { + if (host.startsWith("[") || !host.endsWith(".")) return host; + const stripped = host.slice(0, -1); + if (stripped.length === 0 || stripped.endsWith(".")) { + throw new GrammarError("host has more than one trailing dot, or is empty after stripping it"); + } + return stripped; +} + +/** + * Validates DNS label syntax (LDH rule) for a non-IP hostname. The WHATWG + * URL host parser accepts some strings this grammar must reject, such as an + * empty label from a doubled dot (`a..b.example`). + */ +function assertValidDnsLabels(host) { + if (host.length > 253) throw new GrammarError("host exceeds 253 characters"); + for (const label of host.split(".")) { + if (!/^[a-z0-9]([a-z0-9-]*[a-z0-9])?$/.test(label)) { + throw new GrammarError(`invalid host label '${label}'`); + } + if (label.length > 63) throw new GrammarError(`host label '${label}' exceeds 63 octets`); + } +} + +/** + * Canonicalizes the host[:port] portion shared by the web and endpoint + * grammars. `scheme` is the already-lowercased scheme, or null when the web + * grammar's scheme is omitted. Returns `{ host, isIP, port }`, where `port` + * is the canonical port string or `""` when the port is implied. + */ +function canonicalizeHostAndPort(rest, { scheme, allowWildcard }) { + const { wildcard, rest: afterWildcard } = allowWildcard + ? splitWildcard(rest) + : { wildcard: false, rest: (() => { if (rest.includes("*")) throw new GrammarError("a wildcard is not allowed here"); return rest; })() }; + rejectPathQueryFragment(afterWildcard); + rejectUserinfo(afterWildcard); + const { host: hostToken, port: portToken, isIPv6Literal } = splitHostPort(afterWildcard); + if (hostToken.length === 0) throw new GrammarError("host must not be empty"); + if (wildcard && isIPv6Literal) throw new GrammarError("a wildcard cannot apply to an IP literal"); + + const portValue = portToken === null ? null : validatePort(portToken); + const hostForUrl = stripTrailingDot(hostToken); + + // Always probe through a "special" WHATWG scheme (any of http, https, ws, + // wss, ftp, file) so the URL constructor runs its full host algorithm + // (IDNA, IPv4, IPv6). For a non-special scheme such as "imaps", the same + // constructor treats the host as opaque and skips all of that -- compared + // in schemas/connector-binding-grammar.test.mjs. The real scheme is used + // below only to decide default-port dropping, never for this probe. + let probe; + try { + probe = new URL(`https://${hostForUrl}/`); + } catch { + throw new GrammarError(`invalid host '${hostToken}'`); + } + const canonicalHost = probe.hostname; + const isIP = net.isIP(canonicalHost.replace(/^\[|\]$/g, "")) !== 0; + + if (!isIPv6Literal) { + const bareInput = hostForUrl.toLowerCase(); + if (net.isIPv4(canonicalHost) && bareInput !== canonicalHost) { + throw new GrammarError(`'${hostToken}' is an IPv4-looking label; use the exact dotted-quad address`); + } + } + if (!isIP) { + assertValidDnsLabels(canonicalHost); + } + if (wildcard) { + if (isIP) throw new GrammarError("a wildcard cannot apply to an IP literal"); + if (canonicalHost.split(".").length < 2) { + throw new GrammarError("a wildcard apex must have at least one more label"); + } + } + + let canonicalPort = ""; + if (scheme !== null) { + const schemeDefault = DEFAULT_PORTS[scheme]; + if (portValue !== null && portValue !== schemeDefault) canonicalPort = String(portValue); + } else if (portValue !== null) { + canonicalPort = String(portValue); + } + + return { host: canonicalHost, isIP, wildcard, port: canonicalPort }; +} + +/** + * Canonicalizes a web-origin or web-host constraint: `browser.navigate`, an + * HTTP(S) entry in `network`'s `hosts` list (Section 3.3.4), or (with + * `schemes: CONNECT_SCHEMES`) `browser.connect` (Section 3.3.3), which also + * accepts `ws`/`wss` because a page can open a WebSocket from a document + * loaded over http(s). + * + * Matching semantics (stated here because the grammar alone does not fix + * them): an omitted scheme matches every scheme `schemes` allows. An omitted + * port matches the matching scheme's own default port (80 for http/ws, 443 + * for https/wss) -- it does NOT mean "any port", unlike a CSP host-source. + * An explicit port matches exactly that port for the matching scheme(s). + * + * @param {string} raw + * @param {{ schemes?: Set }} [options] `schemes` defaults to `{http, https}`. + * @returns {string} the canonical form + * @throws {GrammarError} + */ +export function canonicalizeWebHost(raw, { schemes = NAVIGATE_SCHEMES } = {}) { + assertNoWhitespaceOrControl(raw); + const { scheme, rest } = splitScheme(raw); + if (scheme !== null && !schemes.has(scheme)) { + const allowed = [...schemes].join("', '"); + throw new GrammarError(`scheme '${scheme}' is not one of '${allowed}'; use canonicalizeEndpoint for a non-HTTP, non-WebSocket scheme`); + } + const { host, wildcard, port } = canonicalizeHostAndPort(rest, { scheme, allowWildcard: true }); + const prefix = scheme === null ? "" : `${scheme}://`; + const wildcardPrefix = wildcard ? "*." : ""; + const portSuffix = port === "" ? "" : `:${port}`; + return `${prefix}${wildcardPrefix}${host}${portSuffix}`; +} + +/** + * Returns whether `raw` is already in `canonicalizeWebHost`'s canonical + * form (idempotent fixed point), without throwing on invalid input. + */ +export function isCanonicalWebHost(raw, options) { + try { + return canonicalizeWebHost(raw, options) === raw; + } catch { + return false; + } +} + +/** + * Whether `candidateHost` (already canonical: lowercase, IDNA ASCII, no + * trailing dot) is matched by the wildcard apex `apex` (the canonical host + * text after a leading `*.`, for example `"chase.com"` from + * `"*.chase.com"`). Section 3.3.2's rule follows the CSP host-source + * convention: a wildcard matches one or more leftmost labels prepended to + * the apex, at any depth, and never the bare apex itself, which an author + * lists as its own separate entry when it also needs to be reachable. A + * single leading `*.` label is this grammar's SYNTAX (Section 3.3.2); it + * does not bound the MATCHING depth, which real captures need: the reddit + * capture loaded `b.thumbs.redditmedia.com`, two labels under + * `redditmedia.com`, which `*.redditmedia.com` must cover. + * + * Examples for apex `"chase.com"`: matches `"secure.chase.com"` and + * `"a.b.chase.com"`; does not match `"chase.com"` itself; does not match + * `"evilchase.com"` (a label boundary is required, not just a string + * suffix). + * + * @param {string} apex + * @param {string} candidateHost + * @returns {boolean} + */ +export function hostMatchesWildcardApex(apex, candidateHost) { + const suffix = `.${apex}`; + if (!candidateHost.endsWith(suffix)) return false; + const prefix = candidateHost.slice(0, candidateHost.length - suffix.length); + return prefix.length > 0; +} + +/** + * Canonicalizes one entry of `browser.connect` (Section 3.3.3): the same + * grammar as `canonicalizeWebHost`, with `ws` and `wss` also accepted. + * + * @param {string} raw + * @returns {string} + * @throws {GrammarError} + */ +export function canonicalizeConnectHost(raw) { + return canonicalizeWebHost(raw, { schemes: CONNECT_SCHEMES }); +} + +export function isCanonicalConnectHost(raw) { + return isCanonicalWebHost(raw, { schemes: CONNECT_SCHEMES }); +} + +/** + * Canonicalizes a scheme-aware non-HTTP network endpoint (Section 3.3.4), + * for example an IMAP mailbox host. Unlike the web grammar: the scheme is + * REQUIRED (there is no implied protocol to fall back to), no wildcard is + * allowed (an endpoint names one account's exact host), and an omitted port + * is accepted only when the scheme has a listed default port. + * + * @param {string} raw + * @returns {string} the canonical form + * @throws {GrammarError} + */ +export function canonicalizeEndpoint(raw) { + assertNoWhitespaceOrControl(raw); + const { scheme, rest } = splitScheme(raw); + if (scheme === null) { + throw new GrammarError("a non-HTTP endpoint must state its scheme explicitly"); + } + if (CONNECT_SCHEMES.has(scheme)) { + throw new GrammarError(`scheme '${scheme}' is an HTTP or WebSocket scheme; use canonicalizeWebHost or canonicalizeConnectHost`); + } + const { host, port } = canonicalizeHostAndPort(rest, { scheme, allowWildcard: false }); + if (port === "" && DEFAULT_PORTS[scheme] === undefined) { + throw new GrammarError(`scheme '${scheme}' has no known default port; state the port explicitly`); + } + const portSuffix = port === "" ? "" : `:${port}`; + return `${scheme}://${host}${portSuffix}`; +} + +export function isCanonicalEndpoint(raw) { + try { + return canonicalizeEndpoint(raw) === raw; + } catch { + return false; + } +} + +/** + * Canonicalizes one entry of a `network` binding's `hosts` list (Section + * 3.3.4), dispatching to the web or endpoint grammar by scheme. A bare host + * with no scheme, or an `http`/`https` scheme, uses the web grammar + * (wildcard allowed). Any other explicit scheme uses the endpoint grammar. + * + * @param {string} raw + * @returns {string} + * @throws {GrammarError} + */ +export function canonicalizeNetworkHost(raw) { + assertNoWhitespaceOrControl(raw); + const { scheme } = splitScheme(raw); + if (scheme === null || WEB_SCHEMES.has(scheme)) return canonicalizeWebHost(raw); + return canonicalizeEndpoint(raw); +} + +export function isCanonicalNetworkHost(raw) { + try { + return canonicalizeNetworkHost(raw) === raw; + } catch { + return false; + } +} + +/** + * Resolves the `filesystem.inputs` alias (Section 3.3.5): a filesystem-kind + * binding instance can declare its inputs at the legacy top-level `inputs` + * member, at `constraints.inputs`, at both, or at neither. When both are + * present they MUST describe the same inputs; this normalizes them to one + * array (order-independent, keyed by `env_var`) and throws when they + * disagree. JSON Schema cannot express this cross-field equality, so it is + * checked here instead. + * + * @param {{ inputs?: unknown[], constraints?: { inputs?: unknown[] } }} instance + * @returns {unknown[] | undefined} the normalized inputs, or undefined when neither form is present + * @throws {GrammarError} when both forms are present and disagree + */ +/** + * Structural equality for plain JSON values (objects, arrays, strings, + * numbers, booleans, null). Unlike `JSON.stringify(a) === JSON.stringify(b)`, + * this does not depend on an object's member order -- two filesystem input + * entries with the same fields written in a different key order compare + * equal. Array element order still matters (an array is an ordered list, + * e.g. `accepted_extensions`, and reordering it is a real difference). + */ +export function deepEqualJson(a, b) { + if (a === b) return true; + if (typeof a !== typeof b || a === null || b === null) return false; + if (Array.isArray(a) !== Array.isArray(b)) return false; + if (Array.isArray(a)) { + return a.length === b.length && a.every((value, i) => deepEqualJson(value, b[i])); + } + if (typeof a === "object") { + const keysA = Object.keys(a); + const keysB = Object.keys(b); + return keysA.length === keysB.length && keysA.every((key) => Object.hasOwn(b, key) && deepEqualJson(a[key], b[key])); + } + return false; +} + +export function normalizeFilesystemInputs(instance) { + const topLevel = instance?.inputs; + const nested = instance?.constraints?.inputs; + if (topLevel === undefined) return nested; + if (nested === undefined) return topLevel; + + const byEnvVar = (list) => { + const map = new Map(); + for (const entry of list) map.set(entry?.env_var, entry); + return map; + }; + const a = byEnvVar(topLevel); + const b = byEnvVar(nested); + const sameKeys = a.size === b.size && [...a.keys()].every((key) => b.has(key)); + const sameEntries = sameKeys && [...a.entries()].every(([key, value]) => deepEqualJson(value, b.get(key))); + if (!sameEntries) { + throw new GrammarError("filesystem.inputs and constraints.inputs disagree; declare one, or make both equal"); + } + return topLevel; +} + +/** + * Finds duplicate JSON object keys in raw manifest text. `JSON.parse` + * silently keeps the last value for a repeated key, which hides an author + * or generator mistake and is a real interop hazard: two readers could + * reasonably disagree about which value won. This performs its own + * lightweight tokenization so the check runs on the text as written, not on + * the already-collapsed object. + * + * @param {string} jsonText + * @returns {Array<{ path: string, key: string }>} every duplicate found + */ +export function findDuplicateJsonKeys(jsonText) { + const duplicates = []; + const stack = []; // { kind: "object"|"array", keys?: Set, path: string, index?: number } + let i = 0; + const n = jsonText.length; + + function skipWhitespace() { + while (i < n && /\s/.test(jsonText[i])) i += 1; + } + + function readString() { + // Assumes jsonText[i] === '"'. + const start = i; + i += 1; + while (i < n) { + const c = jsonText[i]; + if (c === "\\") { + i += 2; + continue; + } + if (c === '"') { + i += 1; + return JSON.parse(jsonText.slice(start, i)); + } + i += 1; + } + throw new GrammarError("unterminated string in JSON text"); + } + + // The path a newly opened `{` or `[` should take: the path of the key (in + // an object) or index (in an array) whose value it is. `null` at the + // document root. + let pendingPath = "$"; + + function childPath(parentTop, segment) { + if (!parentTop) return "$"; + return parentTop.path === "$" ? String(segment) : `${parentTop.path}.${segment}`; + } + + skipWhitespace(); + while (i < n) { + const c = jsonText[i]; + const top = stack[stack.length - 1]; + if (c === "{") { + stack.push({ kind: "object", keys: new Set(), path: pendingPath }); + i += 1; + } else if (c === "[") { + // Array element paths are not tracked per-index: a duplicate *key* + // can occur only inside an object, so the generic "[]" segment below + // is precise enough for this check's error messages. + stack.push({ kind: "array", path: pendingPath }); + pendingPath = childPath({ path: pendingPath }, "[]"); + i += 1; + } else if (c === "}" || c === "]") { + stack.pop(); + i += 1; + } else if (c === '"') { + const str = readString(); + if (top?.kind === "object") { + skipWhitespace(); + if (jsonText[i] === ":") { + // This string is a key, not a value. + if (top.keys.has(str)) { + duplicates.push({ path: childPath(top, str), key: str }); + } else { + top.keys.add(str); + } + pendingPath = childPath(top, str); + } + } + } else if (c === ":" || c === ",") { + i += 1; + } else { + // number, true, false, null -- skip to the next structural character. + i += 1; + } + skipWhitespace(); + } + + return duplicates; +} diff --git a/reference-implementation/runtime/connector-manifest.schema.json b/reference-implementation/runtime/connector-manifest.schema.json new file mode 100644 index 000000000..3c3636e7d --- /dev/null +++ b/reference-implementation/runtime/connector-manifest.schema.json @@ -0,0 +1,381 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://github.com/PDP-Connect/data-connectors/schemas/connector-manifest.schema.json", + "title": "Collection Profile manifest runtime requirements", + "description": "Machine-readable half of docs/spec/collection-profile.md Section 3.3: binding instances, kinds, interface versions, typed constraints, binding features, and filesystem inputs. Other manifest fields are outside this schema's scope, except the coverage_stores declaration rule of Section 7.1.1, which reads only `streams` and `protocol_capabilities`. String patterns for constraint grammars (Section 3.3.4, 3.3.5) are a necessary but not sufficient structural check: they accept the canonical output of schemas/connector-binding-grammar.mjs, but full IDNA canonicalization, IPv4-looking-label rejection, and IPv6 normalization are that module's job, not this schema's. A string this schema accepts can still be non-canonical in a way only that module detects.", + "type": "object", + "properties": { + "runtime_requirements": { + "type": "object", + "properties": { + "bindings": { "$ref": "#/$defs/bindings" } + } + } + }, + "allOf": [ + { + "$comment": "Section 7.1.1: a stream that declares coverage_stores requires RUN_COVERAGE_V1 in protocol_capabilities.", + "if": { + "required": ["streams"], + "properties": { + "streams": { "type": "array", "contains": { "type": "object", "required": ["coverage_stores"], "properties": { "coverage_stores": {} } } } + } + }, + "then": { + "required": ["protocol_capabilities"], + "properties": { + "protocol_capabilities": { "type": "array", "contains": { "const": "RUN_COVERAGE_V1" } } + } + } + } + ], + "$defs": { + "bindings": { + "type": "object", + "propertyNames": { "$ref": "#/$defs/bindingMapKey" }, + "additionalProperties": false, + "patternProperties": { + "^browser$": { "$ref": "#/$defs/shorthandBrowserInstance" }, + "^desktop_session$": { "$ref": "#/$defs/shorthandDesktopSessionInstance" }, + "^filesystem$": { "$ref": "#/$defs/shorthandFilesystemInstance" }, + "^network$": { "$ref": "#/$defs/shorthandNetworkInstance" }, + "^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+/[a-z0-9]([a-z0-9._-]*[a-z0-9])?$": { + "$ref": "#/$defs/namespacedExtensionInstance" + }, + "^(?!browser$|desktop_session$|filesystem$|network$)[a-z][a-z0-9_]*$": { "$ref": "#/$defs/namedInstance" } + } + }, + "bindingMapKey": { + "type": "string", + "description": "A binding map key is either a registry or namespaced binding name used as shorthand for one instance of that kind, or a free-form author-local instance key that declares its kind explicitly (Section 3.3).", + "anyOf": [{ "$ref": "#/$defs/bindingName" }, { "$ref": "#/$defs/instanceKeyPattern" }] + }, + "instanceKeyPattern": { + "type": "string", + "pattern": "^(?!browser$|desktop_session$|filesystem$|network$)[a-z][a-z0-9_]*$" + }, + "bindingName": { + "type": "string", + "description": "A registry binding name, or an extension binding namespaced as /. Unqualified names are reserved for the registry.", + "anyOf": [ + { "$ref": "#/$defs/registryBinding" }, + { "pattern": "^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+/[a-z0-9]([a-z0-9._-]*[a-z0-9])?$" } + ] + }, + "registryBinding": { + "type": "string", + "description": "Binding registry:\n- browser: A runtime-managed browser surface.\n- desktop_session: The owner's active, logged-in desktop session and its operating-system facilities.\n- filesystem: Access to local files.\n- network: Outbound network access.", + "enum": ["browser", "desktop_session", "filesystem", "network"] + }, + + "instanceRequired": { + "type": "object", + "required": ["required"], + "properties": { "required": { "type": "boolean" } } + }, + "noKind": { "not": { "type": "object", "required": ["kind"], "properties": { "kind": true } } }, + + "shorthandBrowserInstance": { + "allOf": [{ "$ref": "#/$defs/instanceRequired" }, { "$ref": "#/$defs/noKind" }, { "$ref": "#/$defs/browserExtra" }] + }, + "shorthandNetworkInstance": { + "allOf": [{ "$ref": "#/$defs/instanceRequired" }, { "$ref": "#/$defs/noKind" }, { "$ref": "#/$defs/networkExtra" }] + }, + "shorthandFilesystemInstance": { + "allOf": [{ "$ref": "#/$defs/instanceRequired" }, { "$ref": "#/$defs/noKind" }, { "$ref": "#/$defs/filesystemExtra" }] + }, + "shorthandDesktopSessionInstance": { + "allOf": [ + { "$ref": "#/$defs/instanceRequired" }, + { "$ref": "#/$defs/noKind" }, + { "$ref": "#/$defs/desktopSessionExtra" } + ] + }, + "namespacedExtensionInstance": { + "allOf": [ + { "$ref": "#/$defs/instanceRequired" }, + { "$ref": "#/$defs/noKind" }, + { "$ref": "#/$defs/genericExtensionExtra" } + ] + }, + "namedInstance": { + "description": "A free-form author-local binding key. It MUST declare kind (Section 3.3), which selects the constraint and feature shape below by value, the same shape a shorthand key of that kind would use.", + "allOf": [ + { "$ref": "#/$defs/instanceRequired" }, + { "type": "object", "required": ["kind"], "properties": { "kind": true } }, + { "type": "object", "properties": { "kind": { "$ref": "#/$defs/bindingName" } } }, + { + "if": { "type": "object", "properties": { "kind": { "const": "browser" } } }, + "then": { "$ref": "#/$defs/browserExtra" } + }, + { + "if": { "type": "object", "properties": { "kind": { "const": "network" } } }, + "then": { "$ref": "#/$defs/networkExtra" } + }, + { + "if": { "type": "object", "properties": { "kind": { "const": "filesystem" } } }, + "then": { "$ref": "#/$defs/filesystemExtra" } + }, + { + "if": { "type": "object", "properties": { "kind": { "const": "desktop_session" } } }, + "then": { "$ref": "#/$defs/desktopSessionExtra" } + }, + { + "if": { + "type": "object", + "properties": { + "kind": { "type": "string", "pattern": "^[a-z0-9]([a-z0-9-]*[a-z0-9])?(\\.[a-z0-9]([a-z0-9-]*[a-z0-9])?)+/[a-z0-9]([a-z0-9._-]*[a-z0-9])?$" } + } + }, + "then": { "$ref": "#/$defs/genericExtensionExtra" } + } + ] + }, + + "browserExtra": { + "type": "object", + "properties": { + "interface": { "type": "string", "pattern": "^browser@[1-9][0-9]*$" }, + "features": { + "type": "array", + "items": { "anyOf": [{ "$ref": "#/$defs/browserFeature" }, { "$ref": "#/$defs/legacyBrowserFeature" }] }, + "uniqueItems": true + }, + "constraints": { "$ref": "#/$defs/browserConstraints" } + } + }, + "networkExtra": { + "type": "object", + "properties": { + "interface": { "type": "string", "pattern": "^network@[1-9][0-9]*$" }, + "features": { + "type": "array", + "items": { "$ref": "#/$defs/networkFeature" }, + "uniqueItems": true + }, + "constraints": { "$ref": "#/$defs/networkConstraints" } + } + }, + "filesystemExtra": { + "type": "object", + "not": { "type": "object", "required": ["features"], "properties": { "features": true } }, + "properties": { + "interface": { "type": "string", "pattern": "^filesystem@[1-9][0-9]*$" }, + "inputs": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/filesystemInput" } + }, + "constraints": { "$ref": "#/$defs/filesystemConstraints" } + } + }, + "desktopSessionExtra": { + "type": "object", + "not": { "type": "object", "required": ["features"], "properties": { "features": true } }, + "properties": { + "interface": { "type": "string", "pattern": "^desktop_session@[1-9][0-9]*$" }, + "constraints": { "$ref": "#/$defs/desktopSessionConstraints" } + } + }, + "genericExtensionExtra": { + "description": "An extension kind defines its own feature and constraint vocabulary. This schema only checks that a declared feature, if any, is drawn from the shared feature enum, matching the leniency extension bindings already had before binding instances existed; constraints are an open object.", + "type": "object", + "properties": { + "interface": { "type": "string", "minLength": 1 }, + "features": { + "type": "array", + "items": { "$ref": "#/$defs/bindingFeature" }, + "uniqueItems": true + }, + "constraints": { "type": "object" } + } + }, + + "browserFeature": { + "type": "string", + "description": "Features the browser kind provides (Section 3.3.1):\n- page_navigation: Navigate the active page to a URL.\n- page_script_evaluation: Run script in the active page context.\n- page_content_read: Read rendered page content.\n- page_condition_wait: Wait until a condition in the active page becomes true.\n- host_download_capture: Capture content downloaded by the active page.\n- host_archive_extraction: Extract downloaded archive contents in the host runtime.\n- host_archive_entry_chunk_read: Read an extracted archive entry in chunks.\n- page_input: Interact with the active page: click and type.\n- cookie_read: Read the active page's cookie jar.\n- page_response_observation: Observe page network responses, including response body content.\n- host_cookie_jar_request: Make an HTTP request from the host runtime using the active page's cookie jar.", + "enum": [ + "page_navigation", + "page_script_evaluation", + "page_content_read", + "page_condition_wait", + "host_download_capture", + "host_archive_extraction", + "host_archive_entry_chunk_read", + "page_input", + "cookie_read", + "page_response_observation", + "host_cookie_jar_request" + ] + }, + "legacyBrowserFeature": { + "type": "string", + "description": "Section 3.3.8: host_http_request is also valid on browser as a named legacy dual placement, not the target shape. It is not added to browserFeature, so a validator can tell a legacy placement from the one-kind-per-feature rule new features follow. github_browser and strava_browser already declare it here, and scripts/pageshim/capabilities.mjs already supports it on both kinds. A future versioned alias, coordinated with the mobile host, is expected to resolve this; this schema does not perform that move.", + "enum": ["host_http_request"] + }, + "networkFeature": { + "type": "string", + "description": "Features the network kind provides (Section 3.3.1):\n- same_origin_page_fetch: Fetch same-origin resources from the active page context.\n- host_http_request: Make an HTTP request from the host runtime outside the page context.", + "enum": ["same_origin_page_fetch", "host_http_request"] + }, + "bindingFeature": { + "type": "string", + "description": "Host capability definitions (Section 3.3.8; each belongs to exactly one kind, see browserFeature and networkFeature):\n- page_navigation: Navigate the active page to a URL.\n- page_script_evaluation: Run script in the active page context.\n- page_content_read: Read rendered page content.\n- page_condition_wait: Wait until a condition in the active page becomes true.\n- same_origin_page_fetch: Fetch same-origin resources from the active page context.\n- host_http_request: Make an HTTP request from the host runtime outside the page context.\n- host_download_capture: Capture content downloaded by the active page.\n- host_archive_extraction: Extract downloaded archive contents in the host runtime.\n- host_archive_entry_chunk_read: Read an extracted archive entry in chunks.\n- page_input: Interact with the active page: click and type.\n- cookie_read: Read the active page's cookie jar.\n- page_response_observation: Observe page network responses, including response body content.\n- host_cookie_jar_request: Make an HTTP request from the host runtime using the active page's cookie jar.", + "enum": [ + "page_navigation", + "page_script_evaluation", + "page_content_read", + "page_condition_wait", + "same_origin_page_fetch", + "host_http_request", + "host_download_capture", + "host_archive_extraction", + "host_archive_entry_chunk_read", + "page_input", + "cookie_read", + "page_response_observation", + "host_cookie_jar_request" + ] + }, + + "browserConstraints": { + "type": "object", + "additionalProperties": false, + "properties": { + "navigate": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/webHostString" } + }, + "connect": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "$ref": "#/$defs/webConnectHostString" } + } + } + }, + "networkConstraints": { + "type": "object", + "additionalProperties": false, + "properties": { + "hosts": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { + "anyOf": [ + { "$ref": "#/$defs/webHostString" }, + { "$ref": "#/$defs/endpointString" }, + { "$ref": "#/$defs/setupFieldHost" } + ] + } + } + } + }, + "setupFieldHost": { + "type": "object", + "description": "A host or endpoint supplied through a manifest setup field rather than a literal (Section 3.3.4). This is a typed reference, not string interpolation: setup_field names the field, and the runtime resolves and freezes its value before the owner approves the grant.", + "additionalProperties": false, + "required": ["setup_field"], + "properties": { + "setup_field": { "type": "string", "minLength": 1 }, + "allow_private": { "type": "boolean" } + } + }, + "webHostString": { + "type": "string", + "description": "Canonical [scheme://]host[:port] for browser.navigate and an HTTP(S) network host (Section 3.3.3, 3.3.4): lowercase, IDNA ASCII, default http/https port dropped, normalized IPv6, no IPv4-looking non-canonical label, no trailing dot, exact host or one leading '*.' label, no path. This pattern is a structural approximation; schemas/connector-binding-grammar.mjs#canonicalizeWebHost is the reference implementation.", + "pattern": "^(https?://)?(\\*\\.([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?|(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(\\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}|\\[[0-9a-f:]+\\]|([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)*[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)(:[1-9][0-9]{0,4})?$" + }, + "webConnectHostString": { + "type": "string", + "description": "Canonical [scheme://]host[:port] for browser.connect (Section 3.3.3), which is scheme-aware: a page loaded over http(s) can open a ws/wss WebSocket (live capture evidence: wss://ws.chatgpt.com), so connect accepts http, https, ws, or wss where navigate accepts only http/https. Otherwise the same canonicalization as webHostString. schemas/connector-binding-grammar.mjs#canonicalizeConnectHost is the reference implementation.", + "pattern": "^((https?|wss?)://)?(\\*\\.([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)+[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?|(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(\\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}|\\[[0-9a-f:]+\\]|([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)*[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)(:[1-9][0-9]{0,4})?$" + }, + "endpointString": { + "type": "string", + "description": "Canonical scheme://host[:port] for a non-HTTP, non-WebSocket network endpoint, for example IMAP (Section 3.3.4): scheme required, no wildcard, otherwise the same host canonicalization as webHostString. ws/wss are reserved for browser.connect (Section 3.3.3) and are not valid here. schemas/connector-binding-grammar.mjs#canonicalizeEndpoint is the reference implementation.", + "pattern": "^(?!(https?|wss?)://)[a-z][a-z0-9+.-]*://((25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])(\\.(25[0-5]|2[0-4][0-9]|1[0-9]{2}|[1-9]?[0-9])){3}|\\[[0-9a-f:]+\\]|([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\\.)*[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?)(:[1-9][0-9]{0,4})?$" + }, + + "filesystemConstraints": { + "type": "object", + "additionalProperties": false, + "properties": { + "inputs": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/filesystemInput" } + }, + "outputs": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/filesystemOutput" } + } + } + }, + "filesystemInput": { + "type": "object", + "description": "A local path the connector reads. env_var values must be unique within inputs; JSON Schema cannot express that, so the profile states it and the schema test checks it for checked-in manifests.", + "additionalProperties": false, + "required": ["env_var", "kind", "access"], + "properties": { + "env_var": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]*$" }, + "kind": { "type": "string", "enum": ["file", "dir"] }, + "access": { "type": "string", "enum": ["read"] }, + "accepted_extensions": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "pattern": "^\\.[a-z0-9]+(\\.[a-z0-9]+)*$" } + } + } + }, + "filesystemOutput": { + "type": "object", + "description": "A declared output root the connector may write under (Section 3.3.3). The reserved slot \"scratch\" names the runtime's ephemeral per-run directory and MUST NOT be durable.", + "additionalProperties": false, + "required": ["slot", "access"], + "properties": { + "slot": { "type": "string", "pattern": "^[a-z][a-z0-9_]*$" }, + "access": { "type": "string", "enum": ["write"] }, + "overwrite": { "type": "boolean" }, + "delete": { "type": "boolean" }, + "durable": { "type": "boolean" } + }, + "if": { "type": "object", "properties": { "slot": { "const": "scratch" } } }, + "then": { "type": "object", "properties": { "durable": { "const": false } } } + }, + + "desktopSessionConstraints": { + "type": "object", + "additionalProperties": false, + "required": ["items"], + "properties": { + "items": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/desktopSessionItem" } + } + } + }, + "desktopSessionItem": { + "type": "object", + "description": "One exact secret item and the operations brokered on it (Section 3.3.6). selector is an opaque, connector- and OS-specific identifier; the runtime broker resolves it, never the manifest.", + "additionalProperties": false, + "required": ["service", "selector", "operations"], + "properties": { + "service": { "type": "string", "minLength": 1 }, + "selector": { "type": "string", "minLength": 1 }, + "operations": { + "type": "array", + "minItems": 1, + "uniqueItems": true, + "items": { "type": "string", "enum": ["read", "unwrap"] } + } + } + } + } +} diff --git a/reference-implementation/runtime/index.ts b/reference-implementation/runtime/index.ts index 235106c22..6905047f8 100644 --- a/reference-implementation/runtime/index.ts +++ b/reference-implementation/runtime/index.ts @@ -17,12 +17,9 @@ import { DatabaseSync } from "node:sqlite"; import { type ConnectorProtocolCapability, DETAIL_COVERAGE_KEY_SETS_CAPABILITY, - HOST_BLOB_CAPABILITY, isConnectorProtocolCapabilityArray, - OBSERVATION_CAPABILITY, PERMANENT_DETAIL_GAP_REASON, RETRYABLE_DETAIL_GAP_REASONS, - STREAM_EVIDENCE_CAPABILITY, unsupportedRequiredProtocolCapabilities, validateRuntimeContinuationFact, } from "@pdpp/connector-protocol/connector-runtime-protocol"; @@ -93,9 +90,12 @@ import { cursorAuditFact } from "./cursor-audit-fact.ts"; import { describeCursorBandViolation, evaluateCursorBand } from "./cursor-band-contiguity.ts"; import { declaredReasonTokensFor } from "./declared-reason-tokens.ts"; import { createDetailGapPageReader, validateDetailGapsPageRequest } from "./detail-gap-paging.ts"; +import { normalizeBindingDeclarations, planBindingPlacement, startBindingsFor } from "./binding-declarations.ts"; import { collectionProfileDeclarationProblem, + PROTOCOL_CAPABILITY_UNSUPPORTED_FAILURE_REASON, readCollectionProfileDeclaration, + REFERENCE_COLLECTION_PROFILE_SUPPORT, } from "./collection-profile-support.ts"; import type { DoneError } from "./done-validators.ts"; import { @@ -209,14 +209,6 @@ interface AttachmentOutcome { [key: string]: unknown; } -/** Runtime-advertised child bindings (Collection Profile START.bindings). */ -interface AvailableBindings { - browser: Record; - filesystem: Record; - interactive?: Record; - network: Record; -} - /** RS ingest response, after `readIngestResponse` proves the counters and receipt vector. */ type IngestResult = StrictIngestResult; @@ -937,18 +929,6 @@ function buildScopeFields( return appendUniqueFields(streamScope.fields, [...requiredFields, ...primaryKeyFields, ...timeRangeFields]); } -function buildAvailableBindings(onInteraction: unknown): AvailableBindings { - // In this reference runtime, connectors run as local Node child processes - // with full filesystem access by virtue of being child processes. We - // advertise `filesystem` so file-based connectors can declare it as a - // required binding per the Collection Profile spec. - const bindings: AvailableBindings = { browser: {}, filesystem: {}, network: {} }; - if (typeof onInteraction === "function") { - bindings.interactive = {}; - } - return bindings; -} - // Strict, index-validating, oracle-tested parser (test/read-ingest-response-oracle.test.ts). // The hosted RS route (server/routes/rs-mutation.ts) always sets // hostedRejectionReceipts: true, so this runtime's ingest responses always @@ -2602,15 +2582,13 @@ export function buildBrowserSurfaceLaunchEnv({ /** * Protocol capabilities this runtime implements, listed in - * START.protocol_capabilities. It accepts DETAIL_COVERAGE empty_keys and - * unchanged_keys, validates STREAM_EVIDENCE and accepts OBSERVATION (Section - * 5.10). It has no BLOB transport, so it does not list it. + * START.protocol_capabilities. Key-set extension credit rules are not part of + * portable coverage, so DETAIL_COVERAGE_KEY_SETS is omitted: connectors use + * their negotiated base-profile fallback. BLOB is required and unsupported, + * so a connector requiring it is refused before spawn. */ -const RUNTIME_PROTOCOL_CAPABILITIES: readonly ConnectorProtocolCapability[] = [ - DETAIL_COVERAGE_KEY_SETS_CAPABILITY, - OBSERVATION_CAPABILITY, - STREAM_EVIDENCE_CAPABILITY, -]; +const RUNTIME_PROTOCOL_CAPABILITIES: readonly ConnectorProtocolCapability[] = + REFERENCE_COLLECTION_PROFILE_SUPPORT.protocol_capabilities; /** Type of a Collection Profile 0.2.0 OBSERVATION message (Section 5.10). */ const OBSERVATION_MESSAGE_TYPE = "OBSERVATION"; @@ -2624,50 +2602,29 @@ function isObservationMessage(value: unknown): boolean { return typeof value === "object" && value !== null && (value as { type?: unknown }).type === OBSERVATION_MESSAGE_TYPE; } -/** - * Declared capabilities this runtime starts a connector with although it does - * not implement them. BLOB is a runtime-specific extension (Collection Profile - * Section 7). chatgpt and anthropic declare it but emit BLOB only for an item - * over the inline cap, so most of their runs never send one. Such a run - * completes; a run that does send BLOB fails at that line as an unknown - * message type. BLOB stays out of START until this runtime implements it. - */ -const SPAWN_TOLERATED_PROTOCOL_CAPABILITIES: readonly ConnectorProtocolCapability[] = [HOST_BLOB_CAPABILITY]; - /** * Pre-spawn capability gate (Collection Profile Sections 3.4 and 4). An * absent `protocol_capabilities` declares none. A malformed list or an unknown * name fails closed, and so does a required capability this runtime does not - * implement, except BLOB (see SPAWN_TOLERATED_PROTOCOL_CAPABILITIES). An - * advisory capability never blocks the run. + * implement. An advisory capability never blocks the run. */ +function protocolCapabilityRefusal(message: string): Error { + return Object.assign(new Error(message), { failure_reason: PROTOCOL_CAPABILITY_UNSUPPORTED_FAILURE_REASON }); +} + function validateRequiredProtocolCapabilities(manifest: ConnectorManifest): void { const declared = manifest.protocol_capabilities === undefined ? [] : manifest.protocol_capabilities; if (!isConnectorProtocolCapabilityArray(declared)) { - throw new Error("Connector manifest protocol_capabilities must be an array of known capability names"); + throw protocolCapabilityRefusal("Connector manifest protocol_capabilities must be an array of known capability names"); } - const unsupported = unsupportedRequiredProtocolCapabilities(declared, [ - ...RUNTIME_PROTOCOL_CAPABILITIES, - ...SPAWN_TOLERATED_PROTOCOL_CAPABILITIES, - ]); + const unsupported = unsupportedRequiredProtocolCapabilities(declared, RUNTIME_PROTOCOL_CAPABILITIES); if (unsupported.length > 0) { - throw new Error( + throw protocolCapabilityRefusal( `Connector requires protocol capabilities this runtime does not support: ${unsupported.join(", ")}` ); } } -function validateRequiredRuntimeBindings( - requiredBindings: Record, - availableBindings: AvailableBindings -): void { - for (const [binding, req] of Object.entries(requiredBindings)) { - if (req.required && !(binding in availableBindings)) { - throw new Error(`Runtime cannot satisfy required binding: ${binding}`); - } - } -} - function requestedRuntimeStreams(scope: { streams?: unknown } | null): Set | null { if (!Array.isArray(scope?.streams)) { return null; @@ -3074,14 +3031,18 @@ export async function runConnector(opts: RuntimeRunConnectorOptions): Promise = new Set( + Array.isArray(declaredHumanInteractionValues) ? declaredHumanInteractionValues : [] + ); const declarationProblem = collectionProfileDeclarationProblem(manifest as unknown as Record); if (declarationProblem !== null) { - throw new Error(`Connector manifest ${declarationProblem}`); + throw protocolCapabilityRefusal(`Connector manifest ${declarationProblem}`); } validateRequiredProtocolCapabilities(manifest); const profileDeclaration = readCollectionProfileDeclaration(manifest as unknown as Record); @@ -6382,8 +6343,11 @@ export async function runConnector(opts: RuntimeRunConnectorOptions): Promise key !== null) .sort((a, b) => a.localeCompare(b)); +const legacySourceBindings = validateLegacySourceBindings( + pinnedPolyfillManifests.map(({ manifest, file }) => { + const key = manifest.connector_key; + const sourceId = manifest.connector_id; + if (typeof key !== "string") { + throw new Error(`Invalid pinned connector_key in ${file}`); + } + if (typeof sourceId !== "string") { + throw new Error(`Invalid pinned legacy connector_id in ${file}`); + } + return [key, sourceId] as const; + }) +); + function jsonArrayLiteral(values: readonly string[]): string { return `[${values.map((value) => JSON.stringify(value)).join(", ")}]`; } @@ -288,6 +306,10 @@ function jsonObjectLiteral(record: Record): string { return `{\n${lines.join("\n")}\n}`; } +function jsonTupleArrayLiteral(values: readonly (readonly [string, string])[]): string { + return `[${values.map(([key, sourceId]) => `[${JSON.stringify(key)}, ${JSON.stringify(sourceId)}]`).join(", ")}]`; +} + const output = `// Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 @@ -328,6 +350,9 @@ export const PROVIDER_AUTH_LIFECYCLE_PROVEN_KEYS: readonly string[] = Object.fre /** Manifests declaring capabilities.proven.static_secret_live.proven === true. */ export const STATIC_SECRET_LIVE_PROVEN_KEYS: readonly string[] = Object.freeze(${jsonArrayLiteral(staticSecretLiveProvenKeys)}); + +/** Reviewed historical source bindings derived from the pinned connector package. */ +export const LEGACY_FIRST_PARTY_SOURCE_ENTRIES: readonly (readonly [string, string])[] = Object.freeze((${jsonTupleArrayLiteral(legacySourceBindings)} as [string, string][]).map(([key, sourceId]) => Object.freeze([key, sourceId] as const))); `; if (process.argv[2]) { diff --git a/reference-implementation/scripts/legacy-source-binding-validation.ts b/reference-implementation/scripts/legacy-source-binding-validation.ts new file mode 100644 index 000000000..1d76ef36e --- /dev/null +++ b/reference-implementation/scripts/legacy-source-binding-validation.ts @@ -0,0 +1,40 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import { createHash } from "node:crypto"; + +export type LegacySourceBinding = readonly [connectorKey: string, sourceId: string]; + +export const LEGACY_SOURCE_BINDINGS_PIN = "7e8f726725b7a14705571339c84a52d8f05888ffc0996b77cce3d4ae13dae9e6"; +const LEGACY_BINDING_KEY = /^[a-z0-9]+(?:-[a-z0-9]+)*$/; + +/** Validate and order the exact historical mapping approved for generation. */ +export function validateLegacySourceBindings(bindings: readonly LegacySourceBinding[]): LegacySourceBinding[] { + for (const [key, sourceId] of bindings) { + if (!LEGACY_BINDING_KEY.test(key)) { + throw new Error(`Invalid pinned connector_key: ${key}`); + } + if (!URL.canParse(sourceId)) { + throw new Error(`Invalid pinned legacy connector_id for ${key}`); + } + } + + if ( + new Set(bindings.map(([key]) => key)).size !== bindings.length || + new Set(bindings.map(([, sourceId]) => sourceId)).size !== bindings.length + ) { + throw new Error("Pinned historical source bindings contain duplicate keys or source IDs"); + } + + const ordered = [...bindings].sort(([a], [b]) => { + if (a === b) { + return 0; + } + return a < b ? -1 : 1; + }); + const pin = createHash("sha256").update(JSON.stringify(ordered), "utf8").digest("hex"); + if (pin !== LEGACY_SOURCE_BINDINGS_PIN) { + throw new Error(`Pinned historical source mapping changed: sha256:${pin}`); + } + return ordered; +} diff --git a/reference-implementation/scripts/postgres-template-eligibility.ts b/reference-implementation/scripts/postgres-template-eligibility.ts index 12c9352b4..964c1ff95 100644 --- a/reference-implementation/scripts/postgres-template-eligibility.ts +++ b/reference-implementation/scripts/postgres-template-eligibility.ts @@ -53,6 +53,9 @@ export const POSTGRES_TEMPLATE_ELIGIBLE_FILES: readonly string[] = [ "test/connector-attention-store.test.ts", // Exercises install, lock and post-install repair behavior over seeded data, not schema bootstrap. "test/connector-install-postgres.test.ts", + // Install-to-registry journey over a fresh database: Postgres is a fixture for registration and reuse, not schema bootstrap. + "test/connector-install-registry-integration.test.ts", + "test/connector-install-sql-persistence.test.ts", "test/connector-instance-credential-revoke-postgres.test.ts", "test/connector-instance-delete-upsert-two-process-race.test.ts", "test/connector-instance-delete-vs-queued-write-fence.test.ts", @@ -63,6 +66,8 @@ export const POSTGRES_TEMPLATE_ELIGIBLE_FILES: readonly string[] = [ "test/connector-manifest-revision-fence.test.ts", // Uses Postgres as a fixture for the diagnostic run store's writes, retention and locking, not schema bootstrap. "test/connector-observation-store.test.ts", + // Registration identity boundary: Postgres is a fixture for the connectors row and its persist guard. + "test/connector-source-identity-boundary.test.ts", "test/connector-summary-dirty-hooks.test.ts", "test/connector-summary-evidence-canonical-count-cancel-isolation.test.ts", "test/connector-summary-evidence-direct-write-source-revision-detection.test.ts", diff --git a/reference-implementation/server/auth.ts b/reference-implementation/server/auth.ts index 0e4d07068..aec94e612 100644 --- a/reference-implementation/server/auth.ts +++ b/reference-implementation/server/auth.ts @@ -38,8 +38,10 @@ import { clientAssertionIssuer, verifyPrivateKeyJwtClientAssertion, } from "./oauth-client-auth.ts"; +import { parseManifestJsonText } from "../runtime/binding-declarations.ts"; import { listActiveBindingsForGrant, projectBindingForWire } from "./connection-identity.ts"; import { canonicalConnectorKey, canonicalConnectorKeyFromManifest, legacyLocalAliasMap } from "./connector-key.ts"; +import { legacyFirstPartySourceIdFor } from "./connector-legacy-source-map.ts"; import { invalidConnectorManifest, resolveManifestSensitivity, @@ -77,7 +79,7 @@ import { ownerPasswordVerifierRevision, } from "./stores/owner-password-verifier-store.ts"; import { buildGrantedAuthorizationDetail } from "./source-approved-authorization.ts"; -import { snapshotSourceDeclaration } from "./source-declaration.ts"; +import { InvalidSourceDeclarationError, snapshotSourceDeclaration } from "./source-declaration.ts"; import { snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest } from "./source-declaration-legacy-collection.ts"; import { type AcceptedSourceDeclarationRevisionStore, @@ -3524,6 +3526,7 @@ async function persistManifestAndAdvanceGenerations( connectorId, ]) ).rows[0]?.manifest; + assertRetainedSourceIdentity(connectorId, previous, manifestJson); // A manifest stored before digests were retained is the only record of // its version's accepted content: keep it before it is replaced. const previousDigest = previous ? declarationDigestOf(previous) : null; @@ -3595,6 +3598,7 @@ async function persistManifestAndAdvanceGenerations( const existing = db .prepare("SELECT manifest FROM connectors WHERE connector_id = ?") .get<{ manifest?: string }>(connectorId); + assertRetainedSourceIdentity(connectorId, existing?.manifest, manifestJson); if (options.queueDerivedRepair) { const plan = derivedRepairPlan(existing?.manifest ?? null, manifestJson); const instances = db @@ -5550,8 +5554,12 @@ export async function registerConnector( } = {} ): Promise { validateConnectorManifest(manifest); - const { connectorId, storedManifest } = normalizeConnectorManifestForStorage(manifest); + // The retained identity is read here and re-checked under the publication lock in + // persistManifestAndAdvanceGenerations, which is what makes the choice atomic. + const storageContext = await retainedStorageContextForConnector(storageConnectorIdOf(manifest)); + const { connectorId, storedManifest } = normalizeConnectorManifestForStorage(manifest, storageContext); if (!options.skipManifestPersistence) { + await maybeRegisterConnectorPhaseForTest("before-manifest-persisted", { connectorId, manifest: storedManifest }); await persistManifestAndAdvanceGenerations(connectorId, JSON.stringify(storedManifest), { queueDerivedRepair: options.queueDerivedRepair === true, replaceAcceptedDigest: options.replaceAcceptedDigest === true, @@ -5642,14 +5650,138 @@ export async function registerConnector( * authority. In particular, legacy manifest projection adds a durable source * declaration that is not present in the shipped JSON. */ -export function normalizeConnectorManifestForStorage(manifest: Record): { +/** + * Facts a caller reads under the same authority that will persist the result. + * Normalization stays a pure function of its arguments: it never looks anything up. + */ +export interface ConnectorManifestStorageContext { + /** + * The SourceDeclaration an earlier accepted registration retained under this + * connector key. When the incoming manifest carries no explicit declaration, + * the retained `source.id` is kept. Its projection is re-derived from the + * incoming content, so a legitimate content update yields a new + * `declaration_version` under the same source identity. + */ + readonly retainedSourceDeclaration?: unknown; +} + +function storageConnectorIdOf(manifest: Record): unknown { + return canonicalConnectorKeyFromManifest(manifest) ?? manifest.connector_id; +} + +/** The `source_declaration` of a stored or incoming manifest; undefined when absent or unparseable. */ +function sourceDeclarationOf(manifest: unknown): Record | undefined { + try { + const parsed: unknown = typeof manifest === "string" ? JSON.parse(manifest) : manifest; + return isRecord(parsed) && isRecord(parsed.source_declaration) ? parsed.source_declaration : undefined; + } catch { + return undefined; + } +} + +function sourceIdOfDeclaration(declaration: Record | undefined): string | undefined { + const source = declaration?.source; + return isRecord(source) && typeof source.id === "string" && source.id.length > 0 ? source.id : undefined; +} + +/** + * Storage context for a writer that replaces `previousManifest`. Call it with the row read under the same + * lock or transaction that persists the result. A row with no retained declaration gives no context, so + * its write takes the fresh path: a reviewed mapping or an explicit declaration, else a refusal. + */ +export function retainedStorageContext(previousManifest: unknown): ConnectorManifestStorageContext { + const declaration = sourceDeclarationOf(previousManifest); + return declaration === undefined ? {} : { retainedSourceDeclaration: declaration }; +} + +/** The declaration currently retained under a connector key, read before the registration transaction. */ +async function retainedStorageContextForConnector(connectorId: unknown): Promise { + if (!isNonEmptyString(connectorId)) { + return {}; + } + const row = await getConnectorCatalogStore().getManifestById(connectorId); + return retainedStorageContext(row?.manifest); +} + +/** Core Section 5: a retained source identity is never replaced by a registration that names another one. */ +export function assertRetainedSourceIdentity(connectorId: string, previousManifest: unknown, incomingManifestJson: string) { + const retained = sourceIdOfDeclaration(sourceDeclarationOf(previousManifest)); + if (retained === undefined) { + return; + } + const incoming = sourceIdOfDeclaration(sourceDeclarationOf(incomingManifestJson)); + if (incoming === retained) { + return; + } + throw invalidConnectorManifest( + `Connector ${connectorId} is already bound to source ${retained}; this registration names ${incoming ?? "no source"}. A source identity change needs an explicit identity-change path.` + ); +} + +/** Validate an explicit declaration through the Core boundary before it is retained. */ +function acceptedSourceDeclaration(value: unknown): Record { + try { + return snapshotSourceDeclaration(value) as unknown as Record; + } catch (error) { + if (error instanceof InvalidSourceDeclarationError) { + throw invalidConnectorManifest(error.message); + } + throw error; + } +} + +/** + * Project the legacy manifest under the source identity a retained declaration + * already holds. The projection's content is the incoming manifest's; the + * identity is the retained one, never the manifest's provenance. + */ +function projectRetainedSourceDeclaration( + storedManifest: Record, + retainedValue: unknown +): Record { + const retained = acceptedSourceDeclaration(retainedValue); + const source = retained.source as { id: string; kind: string }; + const publisher = retained.publisher as { id: string }; + if (source.kind !== "connector") { + throw invalidConnectorManifest( + `Connector ${String(storedManifest.connector_key)} is bound to a ${source.kind} source; a connector registration cannot change its source kind` + ); + } + return snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest(storedManifest, { + connectorImplementationId: implementationIdOf(storedManifest, source.id), + publisherId: publisher.id, + sourceId: source.id, + }) as unknown as Record; +} + +/** + * The implementation id a legacy projection records: the manifest document's provenance URI when it + * is absolute, else the source id. The alias stamp in `normalizeConnectorManifestForStorage` can put a + * non-URI alias in `manifest_uri`; that value is not a provenance URI. + */ +function implementationIdOf(storedManifest: Record, sourceId: string): string { + const provenance = storedManifest.manifest_uri; + return isNonEmptyString(provenance) && URL.canParse(provenance) ? provenance : sourceId; +} + +export function normalizeConnectorManifestForStorage( + manifest: Record, + context: ConnectorManifestStorageContext = {} +): { connectorId: string; storedManifest: Record; } { - const connectorId = canonicalConnectorKeyFromManifest(manifest) ?? manifest.connector_id; + const connectorId = storageConnectorIdOf(manifest); if (!isNonEmptyString(connectorId)) { throw invalidConnectorManifest("connector_id is required"); } + if (URL.canParse(connectorId)) { + // Known aliases have already resolved to slugs. Any remaining URL is unsupported as an operational + // key, whether or not the manifest carries a source declaration. + throw invalidConnectorManifest( + `connector_id ${connectorId} is a URL with no connector_key; register a connector key, and keep the URL as manifest_uri` + ); + } const storedManifest: Record = { ...cloneJson(manifest), connector_id: connectorId, @@ -5661,21 +5793,27 @@ export function normalizeConnectorManifestForStorage(manifest: Record; - const expected = activationValues(record); + // The row's own canonical declaration is the identity it was published under. + const expected = activationValues(record, retainedStorageContext(canonicalManifest)); if ( expected.activationId !== row.activation_id || expected.manifestRevision !== row.manifest_revision || @@ -107,17 +122,26 @@ function parseRow(row: ActivationRow | undefined): ConnectorActivation | null { /** Atomically publish the registry manifest, generation advance, and executable tuple. */ export async function publishConnectorActivation(record: ConnectorInstallRecord): Promise { - const values = activationValues(record); const attemptId = randomUUID(); const id = record.connectorId; + // The retained declaration is read under the same lock that persists the new manifest, so the + // identity choice cannot race a concurrent registration. + let values: ActivationValues; if (isPostgresStorageBackend()) { - await withPostgresTransaction(async (client) => { + values = await withPostgresTransaction(async (client) => { await client.query("SELECT pg_advisory_xact_lock(hashtext('connector_activation'), hashtext($1))", [id]); + const registry = await client.query<{ manifest: unknown }>( + "SELECT manifest FROM connectors WHERE connector_id=$1 FOR UPDATE", + [id] + ); + const previous = registry.rows[0]?.manifest; + const derived = activationValues(record, retainedStorageContext(previous)); + assertRetainedSourceIdentity(id, previous, derived.canonicalManifestJson); const changed = await client.query( `INSERT INTO connectors(connector_id, manifest) VALUES($1, $2::jsonb) ON CONFLICT(connector_id) DO UPDATE SET manifest = EXCLUDED.manifest WHERE connectors.manifest IS DISTINCT FROM EXCLUDED.manifest RETURNING connector_id`, - [id, values.canonicalManifestJson] + [id, derived.canonicalManifestJson] ); if (changed.rowCount) { await client.query( @@ -135,23 +159,33 @@ export async function publishConnectorActivation(record: ConnectorInstallRecord) canonical_manifest_json=EXCLUDED.canonical_manifest_json,manifest_revision=EXCLUDED.manifest_revision, activation_id=EXCLUDED.activation_id,attempt_id=EXCLUDED.attempt_id, repair_reason='Derived manifest repair pending',repair_error_json=NULL,updated_at=EXCLUDED.updated_at`, - [id, values.recordJson, values.canonicalManifestJson, values.manifestRevision, values.activationId, attemptId] + [ + id, + derived.recordJson, + derived.canonicalManifestJson, + derived.manifestRevision, + derived.activationId, + attemptId, + ] ); await client.query( `INSERT INTO connector_installs(connector_id,record_json,updated_at) VALUES($1,$2,clock_timestamp()::text) ON CONFLICT(connector_id) DO UPDATE SET record_json=EXCLUDED.record_json,updated_at=EXCLUDED.updated_at`, - [id, values.recordJson] + [id, derived.recordJson] ); + return derived; }); } else { - getDb() + values = getDb() .transaction(() => { const db = getDb(); const old = db.prepare("SELECT manifest FROM connectors WHERE connector_id = ?").get<{ manifest: string }>(id); - if (!old || canonicalJson(JSON.parse(old.manifest)) !== values.canonicalManifestJson) { + const derived = activationValues(record, retainedStorageContext(old?.manifest)); + assertRetainedSourceIdentity(id, old?.manifest, derived.canonicalManifestJson); + if (!old || canonicalJson(JSON.parse(old.manifest)) !== derived.canonicalManifestJson) { db.prepare( "INSERT INTO connectors(connector_id,manifest) VALUES(?,?) ON CONFLICT(connector_id) DO UPDATE SET manifest=excluded.manifest" - ).run(id, values.canonicalManifestJson); + ).run(id, derived.canonicalManifestJson); db.prepare( "UPDATE connector_instances SET manifest_generation=manifest_generation+1 WHERE connector_id=?" ).run(id); @@ -163,15 +197,16 @@ export async function publishConnectorActivation(record: ConnectorInstallRecord) manifest_revision=excluded.manifest_revision,activation_id=excluded.activation_id,attempt_id=excluded.attempt_id, repair_reason='Derived manifest repair pending',repair_error_json=NULL,updated_at=excluded.updated_at`).run( id, - values.recordJson, - values.canonicalManifestJson, - values.manifestRevision, - values.activationId, + derived.recordJson, + derived.canonicalManifestJson, + derived.manifestRevision, + derived.activationId, attemptId ); db.prepare( "INSERT INTO connector_installs(connector_id,record_json,updated_at) VALUES(?,?,datetime('now')) ON CONFLICT(connector_id) DO UPDATE SET record_json=excluded.record_json,updated_at=excluded.updated_at" - ).run(id, values.recordJson); + ).run(id, derived.recordJson); + return derived; }) .immediate(); } @@ -311,11 +346,11 @@ async function migrateLegacyActivation(connectorId: string): Promise { return; } const record = JSON.parse(legacy.rows[0].record_json) as ConnectorInstallRecord; - const values = activationValues(record); const registry = await client.query<{ manifest: unknown }>( "SELECT manifest FROM connectors WHERE connector_id=$1", [connectorId] ); + const values = activationValues(record, retainedStorageContext(registry.rows[0]?.manifest)); const coherent = registry.rows[0] && canonicalJson(registry.rows[0].manifest) === values.canonicalManifestJson; await client.query( `INSERT INTO connector_activations(connector_id,state,record_json,canonical_manifest_json,manifest_revision,activation_id,attempt_id,repair_reason,updated_at) @@ -346,10 +381,10 @@ async function migrateLegacyActivation(connectorId: string): Promise { return; } const record = JSON.parse(legacy.record_json) as ConnectorInstallRecord; - const values = activationValues(record); const registry = db .prepare("SELECT manifest FROM connectors WHERE connector_id=?") .get<{ manifest: string }>(connectorId); + const values = activationValues(record, retainedStorageContext(registry?.manifest)); const coherent = registry && canonicalJson(JSON.parse(registry.manifest)) === values.canonicalManifestJson; db.prepare(`INSERT INTO connector_activations(connector_id,state,record_json,canonical_manifest_json,manifest_revision,activation_id,attempt_id,repair_reason,updated_at) VALUES(?,?,?,?,?,?,?,?,datetime('now'))`).run( diff --git a/reference-implementation/server/connector-install/index.ts b/reference-implementation/server/connector-install/index.ts index dcf2c284a..ec7afce8e 100644 --- a/reference-implementation/server/connector-install/index.ts +++ b/reference-implementation/server/connector-install/index.ts @@ -22,8 +22,11 @@ import { writeFileSync, } from "node:fs"; import { dirname, join, resolve, sep } from "node:path"; +import { isDeepStrictEqual } from "node:util"; import { crashPoint } from "../crash-point.ts"; import { canonicalConnectorKey } from "../connector-key.ts"; +import { BindingDeclarationError, parseManifestJsonText } from "../../runtime/binding-declarations.ts"; +import { snapshotSourceDeclaration } from "../source-declaration.ts"; import { assertConnectorLifecycleAvailable, clearConnectorLifecycleUninstalled, @@ -48,6 +51,7 @@ const CORE_MODULE = "@opendatalabs/data-connectors-tools/installer-core"; const MAX_CONFIG_BYTES = 1024 * 1024; const DIGEST = /^sha256:[a-f0-9]{64}$/; const CONNECTOR_ID = /^[a-z0-9][a-z0-9-]*$/; +const SOURCE_DECLARATION_PATH = "source-declaration.json"; const MANIFEST_REQUEST = /\/manifests\/(sha256:[0-9a-f]{64})$/; const BLOB_REQUEST = /\/blobs\/(sha256:[0-9a-f]{64})$/; @@ -64,6 +68,11 @@ export interface ConnectorInstallRecord { readonly manifestSha256: string; readonly provenancePath: string; readonly provenanceSha256: string; + /** + * Digest of the verified `source-declaration.json` sidecar. Absent on legacy records: those keep their + * retained registry identity, and their sidecar bytes are never read as trusted. + */ + readonly sourceDeclarationSha256?: string; readonly registry: "ghcr.io"; readonly repository: string; readonly root: string; @@ -420,7 +429,8 @@ function assertStoredRecordMetadata(active: ConnectorInstallRecord): void { active.repository !== `pdp-connect/connector/${active.connectorId}` || active.manifestPath !== "profile/collection-profile.json" || active.entrypointPath !== "dist/collection-profile.mjs" || - active.provenancePath !== "provenance.json" + active.provenancePath !== "provenance.json" || + (active.sourceDeclarationSha256 !== undefined && !DIGEST.test(active.sourceDeclarationSha256)) ) { throw new Error("Active connector record is invalid."); } @@ -572,6 +582,22 @@ export async function resolveActiveConnectorPath( }); } +/** + * Parses a Collection Profile from the text on disk. A repeated object key is rejected, as on the HTTP and + * local-source paths: `JSON.parse` alone keeps the last value and the repeat is gone before registration. The + * error names no key or path, because the profile text is not trusted and an install error reaches the owner. + */ +function parseProfileText(text: string): unknown { + try { + return parseManifestJsonText(text); + } catch (error) { + if (error instanceof BindingDeclarationError) { + throw new Error("Connector profile repeats an object key."); + } + throw new Error("Connector profile is not valid JSON."); + } +} + function verifyStoredRecord(root: string, active: ConnectorInstallRecord, dataDir?: string): ConnectorInstallRecord { assertStoredRecordMetadata(active); const normalizedRoot = resolve(root); @@ -586,12 +612,63 @@ function verifyStoredRecord(root: string, active: ConnectorInstallRecord, dataDi if (!manifestFile) { throw new Error("Active connector manifest path is invalid."); } - const manifest = JSON.parse(readFileSync(manifestFile, "utf8")) as Record; + const manifest = parseProfileText(readFileSync(manifestFile, "utf8")) as Record; const manifestKey = canonicalManifestKey(manifest); if (manifestKey !== active.connectorId) { throw new Error("Active connector manifest identity does not match its record."); } - return { ...active, manifest, root: normalizedRoot }; + return { + ...active, + manifest: withVerifiedSourceDeclaration(normalizedRoot, active, manifest), + root: normalizedRoot, + }; +} + +/** + * Binds the verified Core source declaration to the profile manifest that registration reads. Only a + * record that carries the sidecar digest may supply it, so a legacy record (no digest) keeps its + * retained registry identity and its sidecar bytes are never trusted. + */ +function withVerifiedSourceDeclaration( + root: string, + active: ConnectorInstallRecord, + manifest: Record +): Record { + if (active.sourceDeclarationSha256 === undefined) { + return manifest; + } + const file = confinedFile(root, SOURCE_DECLARATION_PATH); + if (!file || lstatSync(file).size > MAX_CONFIG_BYTES || sha256(file) !== active.sourceDeclarationSha256) { + throw new Error("Active source declaration bytes failed integrity verification."); + } + const declaration = snapshotSourceDeclaration(JSON.parse(readFileSync(file, "utf8"))); + if (manifest.source_declaration !== undefined && !isDeepStrictEqual(manifest.source_declaration, declaration)) { + throw new Error( + "Connector profile embeds a source declaration that conflicts with the verified source declaration." + ); + } + return { ...manifest, source_declaration: declaration }; +} + +/** + * The sidecar digest when a source declaration file is present; undefined only when it is genuinely absent + * (ENOENT). Any other inspection error propagates: an unreadable sidecar is not a legacy root. + */ +function sourceDeclarationDigestOf(root: string): string | undefined { + const path = join(root, SOURCE_DECLARATION_PATH); + try { + lstatSync(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") { + return undefined; + } + throw error; + } + const file = confinedFile(root, SOURCE_DECLARATION_PATH); + if (!file || lstatSync(file).size > MAX_CONFIG_BYTES) { + throw new Error("Verified source declaration is invalid."); + } + return sha256(file); } function assertCatalogEntry(entry: ConnectorCatalogEntry, connectorId: string, digest?: string): void { @@ -611,7 +688,35 @@ function assertCatalogEntry(entry: ConnectorCatalogEntry, connectorId: string, d } } -function readVerifiedRecord(root: string, entry: ConnectorCatalogEntry): ConnectorInstallRecord { +/** + * A pinned install must carry exactly the sidecar its installer core verified. Absence and any other digest are + * refusals; no pinned digest means a legacy install, which accepts no sidecar check here. + */ +function assertPinnedSourceDeclaration(observed: string | undefined, pinned: string | undefined): void { + if (pinned === undefined) { + return; + } + if (!DIGEST.test(pinned)) { + throw new Error("Pinned source declaration digest is invalid."); + } + if (observed === undefined) { + throw new Error("Verified source declaration is missing for a pinned install."); + } + if (observed !== pinned) { + throw new Error("Verified source declaration does not match installer-core verification."); + } +} + +/** + * Reads a root into an install record. `pinnedSourceDeclarationSha256` is the digest the pinned installer core + * verified: the sidecar must then exist and match it, and the record takes that digest as its authority. Without + * it (legacy installers), the sidecar is optional and its digest is observed from the staged bytes. + */ +function readVerifiedRecord( + root: string, + entry: ConnectorCatalogEntry, + pinnedSourceDeclarationSha256?: string +): ConnectorInstallRecord { const manifestPath = "profile/collection-profile.json"; const entrypointPath = "dist/collection-profile.mjs"; const provenancePath = "provenance.json"; @@ -628,12 +733,15 @@ function readVerifiedRecord(root: string, entry: ConnectorCatalogEntry): Connect if (lstatSync(manifestFile).size > MAX_CONFIG_BYTES) { throw new Error("Connector profile exceeds the 1 MiB RI limit."); } - const manifest: unknown = JSON.parse(readFileSync(manifestFile, "utf8")); + const manifest: unknown = parseProfileText(readFileSync(manifestFile, "utf8")); if (!manifest || typeof manifest !== "object") { throw new Error("Connector profile is invalid."); } const manifestRecord = manifest as Record; const manifestKey = canonicalManifestKey(manifestRecord); + const observedSourceDeclarationSha256 = sourceDeclarationDigestOf(root); + assertPinnedSourceDeclaration(observedSourceDeclarationSha256, pinnedSourceDeclarationSha256); + const sourceDeclarationSha256 = pinnedSourceDeclarationSha256 ?? observedSourceDeclarationSha256; if (manifestKey !== entry.connector_key) { throw new Error("Connector profile identity does not match the catalog target."); } @@ -654,6 +762,7 @@ function readVerifiedRecord(root: string, entry: ConnectorCatalogEntry): Connect manifestSha256: sha256(manifestFile), provenancePath, provenanceSha256: sha256(provenanceFile), + ...(sourceDeclarationSha256 === undefined ? {} : { sourceDeclarationSha256 }), registry: "ghcr.io", repository: `pdp-connect/connector/${entry.connector_key}`, root, @@ -704,12 +813,37 @@ async function reuseExistingInstall( return verified; } +/** + * A pinned installer reports the config digest it verified and the digest of the source-declaration bytes it + * verified. The staged install must then carry exactly those bytes; a missing or different sidecar is refused. + */ +interface PinnedInstallResult { + readonly configDigest: string; + readonly sourceDeclarationSha256: string; +} + +type InstallArtifactResult = + | string + | PinnedInstallResult + // biome-ignore lint/suspicious/noConfusingVoidType: Test fixtures may use synchronous installers. + | void; + +/** Legacy installers report a config digest string or nothing; the sidecar digest is then optional. */ type InstallArtifact = ( root: string, entry: ConnectorCatalogEntry, signal: AbortSignal - // biome-ignore lint/suspicious/noConfusingVoidType: Test fixtures may use synchronous installers. -) => Promise | string | void; +) => Promise | InstallArtifactResult; + +function installResultOf(result: InstallArtifactResult): { + readonly configDigest?: string; + readonly sourceDeclarationSha256?: string; +} { + if (typeof result === "string") { + return { configDigest: result }; + } + return result ? { configDigest: result.configDigest, sourceDeclarationSha256: result.sourceDeclarationSha256 } : {}; +} async function installStagedEntry(options: { readonly connectorId: string; @@ -726,15 +860,16 @@ async function installStagedEntry(options: { const staged = join(transaction, "next"); let published = false; try { - const discoveredConfigDigest = await options.installArtifact(staged, options.entry, options.signal); + const installed = installResultOf(await options.installArtifact(staged, options.entry, options.signal)); options.signal.throwIfAborted(); - const verifiedEntry = discoveredConfigDigest - ? { ...options.entry, config_digest: discoveredConfigDigest } + const verifiedEntry = installed.configDigest + ? { ...options.entry, config_digest: installed.configDigest } : options.entry; assertCatalogEntry(verifiedEntry, options.connectorId, options.entry.digest); - const record = readVerifiedRecord(staged, verifiedEntry); + const record = readVerifiedRecord(staged, verifiedEntry, installed.sourceDeclarationSha256); renameSync(staged, options.root); published = true; + crashPoint("connector-install:after-publish"); const active = { ...record, root: options.root }; const verifiedActive = verifyStoredRecord(options.root, active, options.dataDir); await options.store.activate(verifiedActive); @@ -895,19 +1030,10 @@ export function createConnectorInstallService(options: { ); } else { // A crash can leave the published directory between rename and the - // active-record commit. Do not delete a valid retained root only - // because its active record is missing. - try { - readVerifiedRecord(root, entry); - throw new Error("Connector digest root already exists without a matching active record."); - } catch (error) { - if ( - error instanceof Error && - error.message === "Connector digest root already exists without a matching active record." - ) { - throw error; - } - } + // active-record commit. No record means no retained verification: hashing + // the bytes found here into a new record would make whatever is on disk the + // verified bytes. The leftover is removed and the install below takes the + // bytes from the verified artifact again. removePublishedRootSafely(root, dataDir, connectorId, entry.digest); if (existsSync(root)) { throw new Error("Connector digest root already exists without a matching active record."); @@ -1395,7 +1521,7 @@ async function installPinnedArtifact( root: string, entry: ConnectorCatalogEntry, signal: AbortSignal -): Promise { +): Promise { const core = await loadPinnedCore(); const abortableFetch: FetchLike = (input, init) => fetch(input, { ...init, signal: init?.signal ? AbortSignal.any([init.signal, signal]) : signal }); @@ -1485,7 +1611,10 @@ async function installPinnedArtifact( ); } normalizeCoreInstallLayout(root, entry.connector_id, verifiedSourceDeclaration); - return configDigest; + return { + configDigest, + sourceDeclarationSha256: `sha256:${createHash("sha256").update(verifiedSourceDeclaration).digest("hex")}`, + }; } export function normalizeCoreInstallLayout( diff --git a/reference-implementation/server/connector-install/local-source.test.ts b/reference-implementation/server/connector-install/local-source.test.ts index c2d1790d7..e7f260bec 100644 --- a/reference-implementation/server/connector-install/local-source.test.ts +++ b/reference-implementation/server/connector-install/local-source.test.ts @@ -1,22 +1,31 @@ // Copyright The PDP-Connect Contributors // SPDX-License-Identifier: Apache-2.0 -import { strict as assert } from "node:assert" +import assert from "node:assert/strict" +import { createHash } from "node:crypto" import { + existsSync, mkdirSync, mkdtempSync, + readFileSync, rmSync, writeFileSync, - existsSync, } from "node:fs" import { join } from "node:path" import { afterEach, test } from "node:test" +import { withFixtureSourceDeclaration } from "../../test/helpers/fixture-source-declaration.ts" import { createFileLocalConnectorSourceStore, inspectActiveLocalConnectorSource, } from "./local-source.ts" const temporaryRoots: string[] = [] +const RE_LOCAL_EXPAND_INVALID = /query\.expand.*same-stream relationships\[\] entry/ +const RE_LOCAL_IDENTITY_MISMATCH = /connector_key must match the canonical key for connector_id/ +const RE_MALFORMED_LOCAL_BINDING = /runtime_requirements\.bindings\.usb must have required property 'kind'/ +const RE_MISSING_LOCAL_KEY = /connector_key/ +const RE_DUPLICATE_LOCAL_JSON_KEY = /duplicate JSON object keys/ +const RE_INVALID_ARTIFACT_URI = /connector_id must be an absolute URI or its declared connector_key/ afterEach(() => { for (const root of temporaryRoots.splice(0)) { @@ -27,6 +36,7 @@ afterEach(() => { function fixture( options: { readonly key?: string + readonly connectorId?: string readonly version?: string readonly binding?: string } = {} @@ -35,19 +45,18 @@ function fixture( temporaryRoots.push(root) mkdirSync(join(root, "profile")) mkdirSync(join(root, "dist")) - writeFileSync( - join(root, "profile", "collection-profile.json"), - JSON.stringify({ - connector_id: `https://registry.pdpp.dev/connectors/${options.key ?? "developer-fixture"}`, - connector_key: options.key ?? "developer-fixture", - display_name: "Developer fixture", - runtime_requirements: options.binding - ? { bindings: { [options.binding]: { required: true } } } - : { bindings: { network: { required: true } } }, - streams: [{ name: "items" }], - version: options.version ?? "0.1.0", - }) - ) + const manifest = JSON.parse( + readFileSync(new URL("../../test/fixtures/collection-profiles/github.json", import.meta.url), "utf8") + ) as Record + const connectorKey = options.key ?? "developer-fixture" + manifest.connector_id = options.connectorId ?? `https://vendor.example/connectors/${connectorKey}` + manifest.manifest_uri = manifest.connector_id + manifest.connector_key = connectorKey + manifest.version = options.version ?? "0.1.0" + if (options.binding) { + manifest.runtime_requirements = { bindings: { [options.binding]: { required: true } } } + } + writeFileSync(join(root, "profile", "collection-profile.json"), JSON.stringify(manifest)) writeFileSync( join(root, "dist", "collection-profile.mjs"), "export default {};\n" @@ -55,6 +64,16 @@ function fixture( return root } +function invalidateFirstExpand(root: string): void { + const path = join(root, "profile", "collection-profile.json") + const manifest = JSON.parse(readFileSync(path, "utf8")) as Record + const streams = manifest.streams as Record[] + const [firstStream] = streams + assert.ok(firstStream) + firstStream.query = { expand: [{ name: "missing_relationship" }] } + writeFileSync(path, JSON.stringify(manifest)) +} + test("local sources coexist by key, select explicitly, and remain outside verified installs", async () => { const dataDir = mkdtempSync(join(process.cwd(), "tmp-local-source-state-")) temporaryRoots.push(dataDir) @@ -94,6 +113,96 @@ test("local sources coexist by key, select explicitly, and remain outside verifi assert.equal((await store.list()).length, 1) }) +test("unknown URL artifacts require an explicit key and preserve original manifest bytes", async () => { + const dataDir = mkdtempSync(join(process.cwd(), "tmp-local-source-state-")) + temporaryRoots.push(dataDir) + const sourcePath = fixture({ key: "custom-provider" }) + const manifestPath = join(sourcePath, "profile", "collection-profile.json") + const originalText = readFileSync(manifestPath, "utf8") + const originalManifest = JSON.parse(originalText) as Record + originalManifest.source_declaration = withFixtureSourceDeclaration(originalManifest).source_declaration + writeFileSync(manifestPath, JSON.stringify(originalManifest)) + const originalBytes = readFileSync(manifestPath) + const store = createFileLocalConnectorSourceStore(dataDir) + + const record = await store.add(sourcePath) + + assert.equal(record.connectorId, "https://vendor.example/connectors/custom-provider") + assert.equal(record.connectorKey, "custom-provider") + assert.equal(record.manifest.manifest_uri, record.connectorId) + assert.deepEqual(record.manifest, originalManifest) + assert.deepEqual(record.manifest.source_declaration, originalManifest.source_declaration) + assert.equal(record.manifestSha256, `sha256:${createHash("sha256").update(originalBytes).digest("hex")}`) + assert.deepEqual(readFileSync(manifestPath), originalBytes) +}) + +test("local add rejects invalid profiles before publishing source state", async () => { + const dataDir = mkdtempSync(join(process.cwd(), "tmp-local-source-state-")) + temporaryRoots.push(dataDir) + const invalidPath = fixture({ key: "invalid-expand" }) + const store = createFileLocalConnectorSourceStore(dataDir) + const baseline = await store.list() + + invalidateFirstExpand(invalidPath) + await assert.rejects(() => store.add(invalidPath), RE_LOCAL_EXPAND_INVALID) + assert.deepEqual(await store.list(), baseline) +}) + +test("local reload rejects invalid profiles without changing published state", async () => { + const dataDir = mkdtempSync(join(process.cwd(), "tmp-local-source-state-")) + temporaryRoots.push(dataDir) + const validPath = fixture() + const store = createFileLocalConnectorSourceStore(dataDir) + const existing = await store.add(validPath) + const baseline = await store.list() + + invalidateFirstExpand(validPath) + await assert.rejects(() => store.reload(existing.sourceId), RE_LOCAL_EXPAND_INVALID) + assert.deepEqual(await store.list(), baseline) +}) + +test("local admission refuses a recognized URL and explicit key conflict", async () => { + const dataDir = mkdtempSync(join(process.cwd(), "tmp-local-source-state-")) + temporaryRoots.push(dataDir) + const store = createFileLocalConnectorSourceStore(dataDir) + await assert.rejects( + () => store.add(fixture({ connectorId: "https://registry.pdpp.dev/connectors/github", key: "different-key" })), + RE_LOCAL_IDENTITY_MISMATCH + ) + assert.deepEqual(await store.list(), []) + + const missingKeyPath = fixture() + const manifestPath = join(missingKeyPath, "profile", "collection-profile.json") + const manifest = JSON.parse(readFileSync(manifestPath, "utf8")) as Record + const manifestWithoutKey = Object.fromEntries(Object.entries(manifest).filter(([key]) => key !== "connector_key")) + writeFileSync(manifestPath, JSON.stringify(manifestWithoutKey)) + await assert.rejects(() => store.add(missingKeyPath), RE_MISSING_LOCAL_KEY) + assert.deepEqual(await store.list(), []) + + const duplicateKeyPath = fixture() + const duplicateManifestPath = join(duplicateKeyPath, "profile", "collection-profile.json") + const validText = readFileSync(duplicateManifestPath, "utf8") + const duplicateText = validText.replace( + '"connector_key":"developer-fixture"', + '"connector_key":"developer-fixture","connector_key":"other-key"' + ) + assert.notEqual(duplicateText, validText) + writeFileSync(duplicateManifestPath, duplicateText) + await assert.rejects(() => store.add(duplicateKeyPath), RE_DUPLICATE_LOCAL_JSON_KEY) + assert.deepEqual(await store.list(), []) +}) + +test("local admission rejects a malformed URL before publishing source state", async () => { + const dataDir = mkdtempSync(join(process.cwd(), "tmp-local-source-state-")) + temporaryRoots.push(dataDir) + const store = createFileLocalConnectorSourceStore(dataDir) + await assert.rejects( + () => store.add(fixture({ connectorId: "https://[invalid", key: "malformed-uri" })), + RE_INVALID_ARTIFACT_URI + ) + assert.deepEqual(await store.list(), []) +}) + test("reload refreshes hashes and drift blocks a selected local run until reload", async () => { const dataDir = mkdtempSync(join(process.cwd(), "tmp-local-source-state-")) temporaryRoots.push(dataDir) @@ -119,12 +228,12 @@ test("reload refreshes hashes and drift blocks a selected local run until reload ) }) -test("local admission rejects unsupported runtime bindings", async () => { +test("local admission rejects malformed runtime binding declarations", async () => { const dataDir = mkdtempSync(join(process.cwd(), "tmp-local-source-state-")) temporaryRoots.push(dataDir) const store = createFileLocalConnectorSourceStore(dataDir) await assert.rejects( - () => store.add(fixture({ key: "bad-binding", binding: "usb" })), - /unsupported runtime binding/ + () => store.add(fixture({ binding: "usb", key: "bad-binding" })), + RE_MALFORMED_LOCAL_BINDING ) }) diff --git a/reference-implementation/server/connector-install/local-source.ts b/reference-implementation/server/connector-install/local-source.ts index aaf490a92..56a8644a7 100644 --- a/reference-implementation/server/connector-install/local-source.ts +++ b/reference-implementation/server/connector-install/local-source.ts @@ -20,14 +20,18 @@ import { writeFileSync, } from "node:fs" import { isAbsolute, join, relative, resolve, sep } from "node:path" +import { findDuplicateJsonKeys } from "../../runtime/connector-binding-grammar.mjs" import { canonicalConnectorKey } from "../connector-key.ts" +import { + projectLocalConnectorArtifactForManifestValidation, + validateConnectorManifest, +} from "../connector-manifest-validation.ts" const MANIFEST_PATH = "profile/collection-profile.json" const ENTRYPOINT_PATH = "dist/collection-profile.mjs" const PROVENANCE_PATH = "provenance.json" const MAX_PROFILE_BYTES = 1024 * 1024 const CONNECTOR_KEY = /^[a-z0-9][a-z0-9._-]*$/ -const LOCAL_BINDINGS = new Set(["browser", "filesystem", "network"]) export interface LocalConnectorSourceRecord { readonly connectorId: string @@ -130,6 +134,23 @@ function canonicalRoot(sourcePath: string): string { return root } +type LocalManifestJson = + | { readonly kind: "parsed"; readonly value: unknown } + | { readonly kind: "invalid_json" } + | { readonly kind: "duplicate_key" } + +/** Parser failures stay inside this boundary because their messages can contain file contents. */ +function decodeLocalManifestJson(text: string): LocalManifestJson { + try { + if (findDuplicateJsonKeys(text).length > 0) { + return { kind: "duplicate_key" } + } + return { kind: "parsed", value: JSON.parse(text) } + } catch { + return { kind: "invalid_json" } + } +} + function readManifest(root: string): { readonly connectorId: string readonly connectorKey: string @@ -145,15 +166,14 @@ function readManifest(root: string): { if (lstatSync(manifestPath).size > MAX_PROFILE_BYTES) { throw new Error("Collection Profile manifest exceeds the 1 MiB limit.") } - let parsed: unknown - try { - parsed = JSON.parse(readFileSync(manifestPath, "utf8")) - } catch (error) { - throw new Error( - `Collection Profile manifest is invalid: ${error instanceof Error ? error.message : String(error)}`, - { cause: error } - ) + const decoded = decodeLocalManifestJson(readFileSync(manifestPath, "utf8")) + if (decoded.kind === "invalid_json") { + throw new Error("Collection Profile manifest is invalid JSON.") } + if (decoded.kind === "duplicate_key") { + throw new Error("Collection Profile manifest contains duplicate JSON object keys.") + } + const parsed = decoded.value assertRecord(parsed, "Collection Profile manifest") const connectorId = parsed.connector_id const connectorKey = parsed.connector_key @@ -170,39 +190,13 @@ function readManifest(root: string): { "Collection Profile manifest must declare a valid connector_id, connector_key, and version." ) } + validateConnectorManifest(projectLocalConnectorArtifactForManifestValidation(parsed)) const streams = parsed.streams if (!Array.isArray(streams) || streams.length === 0) { throw new Error( "Collection Profile manifest must declare at least one stream." ) } - const requirements = parsed.runtime_requirements - if (requirements !== undefined && requirements !== null) { - assertRecord(requirements, "Collection Profile runtime_requirements") - const bindings = requirements.bindings - if (bindings !== undefined && bindings !== null) { - assertRecord(bindings, "Collection Profile runtime_requirements.bindings") - for (const [binding, requirement] of Object.entries(bindings)) { - if (!LOCAL_BINDINGS.has(binding)) { - throw new Error( - `Collection Profile declares unsupported runtime binding: ${binding}` - ) - } - assertRecord( - requirement, - `Collection Profile runtime binding ${binding}` - ) - if ( - requirement.required !== undefined && - typeof requirement.required !== "boolean" - ) { - throw new Error( - `Collection Profile runtime binding ${binding}.required must be a boolean.` - ) - } - } - } - } return { connectorId, connectorKey, diff --git a/reference-implementation/server/connector-legacy-source-map.ts b/reference-implementation/server/connector-legacy-source-map.ts new file mode 100644 index 000000000..8027e6786 --- /dev/null +++ b/reference-implementation/server/connector-legacy-source-map.ts @@ -0,0 +1,23 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +import { LEGACY_FIRST_PARTY_SOURCE_ENTRIES } from "./generated/connector-registry.generated.ts"; + +export const LEGACY_SOURCE_MAP_VERSION = 1; + +const legacySourceIds: Readonly> = Object.freeze( + Object.fromEntries(LEGACY_FIRST_PARTY_SOURCE_ENTRIES) +); + +/** Return the reviewed historical source identity for a canonical key. */ +export function legacyFirstPartySourceIdFor(connectorKey: string): string | undefined { + return Object.hasOwn(legacySourceIds, connectorKey) ? legacySourceIds[connectorKey] : undefined; +} + +export const LEGACY_FIRST_PARTY_KEYS: readonly string[] = Object.freeze( + LEGACY_FIRST_PARTY_SOURCE_ENTRIES.map(([key]) => key) +); + +export const LEGACY_FIRST_PARTY_ENTRIES: ReadonlyArray = Object.freeze( + LEGACY_FIRST_PARTY_SOURCE_ENTRIES.map(([key, sourceId]) => Object.freeze([key, sourceId] as const)) +); diff --git a/reference-implementation/server/connector-manifest-validation.ts b/reference-implementation/server/connector-manifest-validation.ts index b2213ea93..7b59160df 100644 --- a/reference-implementation/server/connector-manifest-validation.ts +++ b/reference-implementation/server/connector-manifest-validation.ts @@ -10,7 +10,8 @@ * logic. */ -import { collectionProfileDeclarationError } from "@pdpp/connector-protocol"; +import { BindingDeclarationError, normalizeBindingDeclarations } from "../runtime/binding-declarations.ts"; +import { collectionProfileDeclarationProblem } from "../runtime/collection-profile-support.ts"; import { tuningEnvironmentKeyOwner } from "../runtime/connector-child-environment.ts"; import { normalizeStaticSecretCredentialCapture, @@ -183,6 +184,36 @@ export function invalidConnectorManifest(message: string, code = "invalid_reques return err; } +/** + * Prepare a developer-local artifact for the host's full manifest validator. + * + * Unknown URL identities are not inferred into connector keys. The artifact + * must declare its own key; this copy uses that key as the operational id only + * for validation. The returned projection is not a trusted or retained source + * identity, and callers must keep the original manifest unchanged. + */ +export function projectLocalConnectorArtifactForManifestValidation( + manifest: Record +): Record { + const connectorId = manifest.connector_id; + const connectorKey = manifest.connector_key; + if (!isNonEmptyString(connectorId)) { + throw invalidConnectorManifest("Collection Profile manifest must declare connector_id."); + } + if (typeof connectorKey !== "string" || !isConnectorKey(connectorKey)) { + throw invalidConnectorManifest("Collection Profile manifest must declare a valid connector_key."); + } + const normalizedKey = connectorKey.trim(); + const recognizedKey = canonicalConnectorKey(connectorId); + if (recognizedKey !== null && recognizedKey !== normalizedKey) { + throw invalidConnectorManifest("connector_key must match the canonical key for connector_id"); + } + if (recognizedKey === null && connectorId.trim() !== normalizedKey && !URL.canParse(connectorId)) { + throw invalidConnectorManifest("connector_id must be an absolute URI or its declared connector_key"); + } + return { ...manifest, connector_id: normalizedKey }; +} + /** * Collection Profile declaration gate (profile Sections 3.1 and 3.4): * `protocol_version` names a supported profile version, every @@ -191,12 +222,9 @@ export function invalidConnectorManifest(message: string, code = "invalid_reques * manifest without `protocol_version`; it is checked as a 0.1.0 manifest. */ function validateCollectionProfileDeclaration(manifest: Record, code: string): void { - const error = collectionProfileDeclarationError( - manifest.protocol_version === undefined ? "0.1.0" : manifest.protocol_version, - manifest.protocol_capabilities - ); - if (error !== null) { - throw invalidConnectorManifest(error, code); + const problem = collectionProfileDeclarationProblem(manifest); + if (problem !== null) { + throw invalidConnectorManifest(problem, code); } } @@ -283,13 +311,6 @@ export const REFRESH_POLICY_ALLOWED_KEYS = new Set([ // production code, but the read-site clamp holds even if it does. export const REFRESH_POLICY_MAX_COOLDOWN_CYCLES_RANGE = { max: 24, min: 1 } as const; export const REFRESH_POLICY_MAX_RECOVERY_ATTEMPTS_RANGE = { max: 20, min: 1 } as const; -export const RUNTIME_REQUIREMENT_BINDINGS = new Set([ - "browser", - "desktop_session", - "filesystem", - "interactive", - "network", -]); export const STREAM_AVAILABILITY_STATES = new Set(["supported", "unsupported_in_mode", "experimental", "deprecated"]); export const STREAM_AVAILABILITY_ALLOWED_KEYS = new Set(["future_modes", "mode", "reason", "state"]); export const STREAM_COVERAGE_POLICIES = new Set([ @@ -326,38 +347,19 @@ const SUPPORTED_RANGE_OPERATORS = new Set(["gte", "gt", "lte", "lt"]); const EXTERNAL_TOOL_ALLOWED_KEYS = new Set(["detect", "install_hint", "license", "min_version", "name", "purpose"]); const EXTERNAL_TOOL_DETECT_ALLOWED_KEYS = new Set(["args", "executable", "executable_env_override", "exit_code"]); -// Validates `runtime_requirements.bindings` (order-preserving). Returns `false` -// when `bindings` is absent — the original validator returns from the whole -// function in that case, so the caller must NOT go on to external_tools (a -// runtime_requirements object without bindings is accepted verbatim). Returns -// `true` once bindings are present and fully validated. -function validateRuntimeBindings(req: Record, code: string): boolean { - // biome-ignore lint/style/useDestructuring: Explicit property or positional access documents this compatibility boundary. - const bindings = req.bindings; - if (bindings === undefined || bindings === null) { - return false; - } - if (typeof bindings !== "object" || Array.isArray(bindings)) { - throw invalidConnectorManifest("runtime_requirements.bindings must be an object when declared", code); - } - const bindingsObj = bindings as Record; - const unknownBindings = Object.keys(bindingsObj).filter((binding) => !RUNTIME_REQUIREMENT_BINDINGS.has(binding)); - if (unknownBindings.length) { - throw invalidConnectorManifest( - `runtime_requirements.bindings has unsupported keys: ${unknownBindings.join(", ")}`, - code - ); - } - for (const [binding, requirement] of Object.entries(bindingsObj)) { - if (!requirement || typeof requirement !== "object" || Array.isArray(requirement)) { - throw invalidConnectorManifest(`runtime_requirements.bindings.${binding} must be an object`, code); - } - const reqObj = requirement as Record; - if (reqObj.required !== undefined && typeof reqObj.required !== "boolean") { - throw invalidConnectorManifest(`runtime_requirements.bindings.${binding}.required must be a boolean`, code); +// Validates `runtime_requirements.bindings` with the shared canonical validator +// (runtime/binding-declarations.ts), the same one `runConnector` runs before spawn. +// An absent map is valid. A present one must satisfy the canonical schema and +// the grammar rules, or registration refuses it with the validator's message. +function validateRuntimeBindings(req: Record, code: string): void { + try { + normalizeBindingDeclarations(req.bindings); + } catch (error) { + if (error instanceof BindingDeclarationError) { + throw invalidConnectorManifest(error.message, code); } + throw error; } - return true; } function validateExternalToolDetectExitCode(detect: Record, index: number, code: string): void { @@ -652,9 +654,7 @@ export function validateRuntimeRequirements(manifest: Record, c } const req = requirements as Record; validateTuningEnvironment(manifest, req, code); - if (!validateRuntimeBindings(req, code)) { - return; - } + validateRuntimeBindings(req, code); validateExternalTools(req, code, { allowLegacyCommand: !Array.isArray(manifest.streams) }); validateLocalPaths(req, code); } @@ -797,6 +797,37 @@ function validateRefreshPolicyInternalConsistency(pol: Record, } } +// Section 3.4: `capabilities.human_interaction` is an optional array of distinct +// values from this closed set. An INTERACTION.kind outside it is refused at run time. +const HUMAN_INTERACTION_VALUES = new Set(["credentials", "otp", "manual_action"]); + +function validateHumanInteractionCapability(manifest: Record, code: string): void { + const { capabilities } = manifest; + if (typeof capabilities !== "object" || capabilities === null || Array.isArray(capabilities)) { + return; + } + const values = (capabilities as Record).human_interaction; + if (values === undefined) { + return; + } + if (!Array.isArray(values)) { + throw invalidConnectorManifest("capabilities.human_interaction must be an array when declared", code); + } + const seen = new Set(); + for (const value of values) { + if (!HUMAN_INTERACTION_VALUES.has(value as string)) { + throw invalidConnectorManifest( + `capabilities.human_interaction value ${JSON.stringify(value)} must be one of: credentials, otp, manual_action`, + code + ); + } + if (seen.has(value)) { + throw invalidConnectorManifest(`capabilities.human_interaction repeats ${String(value)}`, code); + } + seen.add(value); + } +} + export function validateRefreshPolicyCapability(manifest: Record, code: string): void { validatePublicListingTier(manifest, code); // biome-ignore lint/style/useDestructuring: Explicit property or positional access documents this compatibility boundary. @@ -2232,6 +2263,7 @@ export function validateConnectorManifest( validateCollectionProfileDeclaration(manifest, code); } validateRuntimeRequirements(manifest, code); + validateHumanInteractionCapability(manifest, code); validateRefreshPolicyCapability(manifest, code); validateDeclaredReasonTokensCapability(manifest, code); validateProvenCapability(manifest, code); diff --git a/reference-implementation/server/generated/connector-registry.generated.ts b/reference-implementation/server/generated/connector-registry.generated.ts index 9c0c8a220..cc915daf9 100644 --- a/reference-implementation/server/generated/connector-registry.generated.ts +++ b/reference-implementation/server/generated/connector-registry.generated.ts @@ -45,3 +45,6 @@ export const PROVIDER_AUTH_LIFECYCLE_PROVEN_KEYS: readonly string[] = Object.fre /** Manifests declaring capabilities.proven.static_secret_live.proven === true. */ export const STATIC_SECRET_LIVE_PROVEN_KEYS: readonly string[] = Object.freeze(["github", "gmail", "slack", "ynab"]); + +/** Reviewed historical source bindings derived from the pinned connector package. */ +export const LEGACY_FIRST_PARTY_SOURCE_ENTRIES: readonly (readonly [string, string])[] = Object.freeze(([["amazon", "https://registry.pdpp.dev/connectors/amazon"], ["anthropic", "https://registry.pdpp.dev/connectors/anthropic"], ["apple-contacts", "https://registry.pdpp.dev/connectors/apple-contacts"], ["apple-health", "https://registry.pdpp.dev/connectors/apple-health"], ["apple-photos", "https://registry.pdpp.dev/connectors/apple-photos"], ["chase", "https://registry.pdpp.dev/connectors/chase"], ["chatgpt", "https://registry.pdpp.dev/connectors/chatgpt"], ["claude-code", "https://registry.pdpp.dev/connectors/claude-code"], ["codex", "https://registry.pdpp.dev/connectors/codex"], ["doordash", "https://registry.pdpp.dev/connectors/doordash"], ["github", "https://registry.pdpp.dev/connectors/github"], ["gmail", "https://registry.pdpp.dev/connectors/gmail"], ["google-calendar", "https://registry.pdpp.dev/connectors/google-calendar"], ["google-contacts", "https://registry.pdpp.dev/connectors/google-contacts"], ["google-maps", "https://registry.pdpp.dev/connectors/google-maps"], ["google-maps-data-portability", "https://registry.pdpp.dev/connectors/google-maps-data-portability"], ["google-messages", "https://registry.pdpp.dev/connectors/google-messages"], ["google-takeout", "https://registry.pdpp.dev/connectors/google-takeout"], ["groupme", "https://registry.pdpp.dev/connectors/groupme"], ["heb", "https://registry.pdpp.dev/connectors/heb"], ["ical", "https://registry.pdpp.dev/connectors/ical"], ["imessage", "https://registry.pdpp.dev/connectors/imessage"], ["jellyfin", "https://registry.pdpp.dev/connectors/jellyfin"], ["linkedin", "https://registry.pdpp.dev/connectors/linkedin"], ["loom", "https://registry.pdpp.dev/connectors/loom"], ["meta", "https://registry.pdpp.dev/connectors/meta"], ["netflix-export", "https://registry.pdpp.dev/connectors/netflix-export"], ["notion", "https://registry.pdpp.dev/connectors/notion"], ["oura", "https://registry.pdpp.dev/connectors/oura"], ["pocket", "https://registry.pdpp.dev/connectors/pocket"], ["reddit", "https://registry.pdpp.dev/connectors/reddit"], ["shopify", "https://registry.pdpp.dev/connectors/shopify"], ["signal", "https://registry.pdpp.dev/connectors/signal"], ["slack", "https://registry.pdpp.dev/connectors/slack"], ["spotify", "https://registry.pdpp.dev/connectors/spotify"], ["steam", "https://registry.pdpp.dev/connectors/steam"], ["strava", "https://registry.pdpp.dev/connectors/strava"], ["twitter-archive", "https://registry.pdpp.dev/connectors/twitter-archive"], ["uber", "https://registry.pdpp.dev/connectors/uber"], ["usaa", "https://registry.pdpp.dev/connectors/usaa"], ["venmo", "https://registry.pdpp.dev/connectors/venmo"], ["whatsapp", "https://registry.pdpp.dev/connectors/whatsapp"], ["wholefoods", "https://registry.pdpp.dev/connectors/wholefoods"], ["whoop", "https://registry.pdpp.dev/connectors/whoop"], ["ynab", "https://registry.pdpp.dev/connectors/ynab"]] as [string, string][]).map(([key, sourceId]) => Object.freeze([key, sourceId] as const))); diff --git a/reference-implementation/server/index.ts b/reference-implementation/server/index.ts index 43c5ace67..87c9292e2 100644 --- a/reference-implementation/server/index.ts +++ b/reference-implementation/server/index.ts @@ -69,6 +69,7 @@ import { type BrowserSurfaceReadinessProbe, createDefaultBrowserSurfaceReadinessProbe, } from "../runtime/browser-surface-readiness.ts"; +import { REFERENCE_COLLECTION_PROFILE_SUPPORT } from "../runtime/collection-profile-support.ts"; import { type ConnectorEnvironmentPolicy, parseConnectorEnvironmentPolicy, @@ -5954,6 +5955,14 @@ export function buildAsApp(opts: ServerOpts = {}) { pdppError, requireOwnerSession: ownerAuth.requireOwnerSession, } as unknown as Parameters[1]); + // Owner-only statement of the Collection Profile this runtime implements, with what is outstanding. + app.get( + "/_ref/collection-profile", + ownerAuth.requireOwnerSession, + (_req: unknown, res: { json: (body: unknown) => unknown }) => { + res.json({ object: "collection_profile_support", ...REFERENCE_COLLECTION_PROFILE_SUPPORT }); + } + ); // Owner-only diagnostic read of a run's OBSERVATION record (Collection Profile 0.2.0 Section 5.10). mountRefRunObservations(app, { createRequestConnectorInstanceStore, diff --git a/reference-implementation/server/transport.ts b/reference-implementation/server/transport.ts index 8f8f4ec7d..eb64bb513 100644 --- a/reference-implementation/server/transport.ts +++ b/reference-implementation/server/transport.ts @@ -53,6 +53,7 @@ declare module "fastify" { } import { publicManifests, referenceManifests } from "@pdpp/reference-contract"; +import { parseManifestJsonText } from "../runtime/binding-declarations.ts"; import { applyRequestValidation, applyResponseValidation, @@ -436,13 +437,18 @@ function buildFastify({ loggerInstance }: { loggerInstance: FastifyBaseLogger }) // payloads so routes like `POST /grants/:id/revoke` (no body, header still // JSON) reach their handlers instead of failing at the transport. fastify.removeContentTypeParser("application/json"); - fastify.addContentTypeParser("application/json", { parseAs: "string" }, (_req, body, done) => { + // Routes whose body is a Collection Profile manifest also reject any repeated + // object key in the raw text, so last-wins cannot change what gets validated. + const duplicateKeyRejectingRoutes = new Set(["POST /connectors"]); + fastify.addContentTypeParser("application/json", { parseAs: "string" }, (req, body, done) => { if (!body) { done(null, {}); return; } try { - done(null, JSON.parse(typeof body === "string" ? body : body.toString())); + const text = typeof body === "string" ? body : body.toString(); + const routeKey = `${req.method} ${req.routeOptions?.url ?? ""}`; + done(null, duplicateKeyRejectingRoutes.has(routeKey) ? parseManifestJsonText(text) : JSON.parse(text)); } catch (error: unknown) { const err = error instanceof Error ? error : new Error(String(error)); Object.assign(err, { statusCode: 400 }); diff --git a/reference-implementation/test/approval-review-artifact-rendering.test.ts b/reference-implementation/test/approval-review-artifact-rendering.test.ts index d77de19c2..34b2fc982 100644 --- a/reference-implementation/test/approval-review-artifact-rendering.test.ts +++ b/reference-implementation/test/approval-review-artifact-rendering.test.ts @@ -134,7 +134,7 @@ function sourceManifest() { async function setup(): Promise<{ asUrl: string; server: TestServerHandle }> { const server = (await startServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 })) as TestServerHandle; const asUrl = `http://localhost:${server.asPort}`; - const manifest = loadSpotifyManifest(); + const manifest = { ...loadSpotifyManifest(), source_declaration: sourceManifest().source_declaration }; const registered = await fetch(`${asUrl}/connectors`, { body: JSON.stringify(manifest), headers: { Accept: "application/json", "Content-Type": "application/json" }, diff --git a/reference-implementation/test/approval-review-seam.test.ts b/reference-implementation/test/approval-review-seam.test.ts index c2400e93f..147ae2ad0 100644 --- a/reference-implementation/test/approval-review-seam.test.ts +++ b/reference-implementation/test/approval-review-seam.test.ts @@ -79,8 +79,10 @@ async function setup(backend: Backend = "sqlite", databaseUrl = POSTGRES_URL) { } else { initDb(":memory:"); } + // Register the fixture's own source from the start. Otherwise the first registration retains the reviewed + // legacy source for `spotify`, and every later registration that names SOURCE_ID is a source rebinding. const manifest = loadSpotifyManifest(); - await registerConnector(manifest); + await registerConnector({ ...manifest, source_declaration: nativeManifest().source_declaration }); await seedPreRegisteredClients([ { client_id: CLIENT_ID, @@ -603,7 +605,7 @@ test("HTTP batch final review resumes the exact result without duplicate issuanc const asUrl = `http://localhost:${server.asPort}`; try { const register = await fetch(`${asUrl}/connectors`, { - body: JSON.stringify(manifest), + body: JSON.stringify({ ...manifest, source_declaration: nativeManifest().source_declaration }), headers: { Accept: "application/json", "Content-Type": "application/json" }, method: "POST", }); @@ -817,7 +819,7 @@ test("HTTP single consent must be reviewed before approve", async () => { const asUrl = `http://localhost:${server.asPort}`; try { const register = await fetch(`${asUrl}/connectors`, { - body: JSON.stringify(manifest), + body: JSON.stringify({ ...manifest, source_declaration: nativeManifest().source_declaration }), headers: { Accept: "application/json", "Content-Type": "application/json" }, method: "POST", }); @@ -914,7 +916,7 @@ test("HTTP single final approval derives custom subject from persisted review", const asUrl = `http://localhost:${server.asPort}`; try { const register = await fetch(`${asUrl}/connectors`, { - body: JSON.stringify(manifest), + body: JSON.stringify({ ...manifest, source_declaration: nativeManifest().source_declaration }), headers: { "Content-Type": "application/json" }, method: "POST", }); diff --git a/reference-implementation/test/attention-writer.test.ts b/reference-implementation/test/attention-writer.test.ts index 9d971d394..3bb64eda6 100644 --- a/reference-implementation/test/attention-writer.test.ts +++ b/reference-implementation/test/attention-writer.test.ts @@ -496,8 +496,8 @@ test( chmodSync(stubPath, 0o755); const manifest = { + capabilities: { human_interaction: ["otp"] }, connector_id: "codex", - runtime_requirements: { bindings: { interactive: { required: false } } }, streams: [ { name: "conversations", diff --git a/reference-implementation/test/auto-enroll-eligible-schedules-integration.test.ts b/reference-implementation/test/auto-enroll-eligible-schedules-integration.test.ts index b99b158dd..5bf1841aa 100644 --- a/reference-implementation/test/auto-enroll-eligible-schedules-integration.test.ts +++ b/reference-implementation/test/auto-enroll-eligible-schedules-integration.test.ts @@ -31,6 +31,7 @@ import { canonicalConnectorKey } from "../server/connector-key.ts"; import { closeDb, initDb } from "../server/db.ts"; import { createSqliteConnectorInstanceStore } from "../server/stores/connector-instance-store.ts"; import { getDefaultSchedulerStore } from "../server/stores/scheduler-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; const OWNER_SUBJECT_ID = "owner_local"; const NOW = "2026-08-16T00:00:00.000Z"; @@ -42,34 +43,37 @@ const NOW = "2026-08-16T00:00:00.000Z"; // stored it under (matching how `canonicalConnectorKey` derives short keys // from `https://registry.pdpp.dev/connectors/` in production). async function registerConnectorWithKey(slug: string): Promise { - const manifest = { - capabilities: { - auth: { kind: "env", required: [] }, - public_listing: { tier: "supported" }, - refresh_policy: { - background_safe: true, - interaction_posture: "none", - rationale: "Synthetic manifest for the getSchedule active-connection filter contract.", - recommended_interval_seconds: 3600, - recommended_mode: "automatic", + const manifest = withFixtureSourceDeclaration( + { + capabilities: { + auth: { kind: "env", required: [] }, + public_listing: { tier: "supported" }, + refresh_policy: { + background_safe: true, + interaction_posture: "none", + rationale: "Synthetic manifest for the getSchedule active-connection filter contract.", + recommended_interval_seconds: 3600, + recommended_mode: "automatic", + }, }, + connector_id: slug, + display_name: slug, + manifest_uri: `https://registry.pdpp.dev/connectors/${slug}`, + protocol_version: "0.1.0", + runtime_requirements: {}, + streams: [ + { + name: "records", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "1.0.0", }, - connector_id: `https://registry.pdpp.dev/connectors/${slug}`, - display_name: slug, - manifest_uri: `https://registry.pdpp.dev/connectors/${slug}`, - protocol_version: "0.1.0", - runtime_requirements: {}, - streams: [ - { - name: "records", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "1.0.0", - }; + { sourceId: `https://registry.pdpp.dev/connectors/${slug}` } + ); const connectorId = await registerConnector(manifest); return canonicalConnectorKey(connectorId) ?? connectorId; } @@ -129,39 +133,42 @@ async function seedSchedule({ connectorId, connectorInstanceId }: SeedScheduleOp const TEST_ENV_KEY = "TEST_AUTO_ENROLL_TOKEN"; function testManifest(recommendedMode: "automatic" | "manual" = "automatic"): Record { - return { - capabilities: { - auth: { kind: "env", required: [TEST_ENV_KEY] }, - public_listing: { tier: "supported" }, - // Mode is DERIVED from these facts, so the synthetic manifest has to - // declare a coherent pair rather than just asserting a mode string: - // the manual variant is an interactive-login connector that has not - // declared session persistence (the Chase/USAA shape), and the - // automatic variant needs no per-run owner gesture. - refresh_policy: { - background_safe: recommendedMode === "automatic", - interaction_posture: recommendedMode === "automatic" ? "none" : "otp_likely", - rationale: "Synthetic manifest for the scheduler integration contract.", - recommended_interval_seconds: 3600, - recommended_mode: recommendedMode, + return withFixtureSourceDeclaration( + { + capabilities: { + auth: { kind: "env", required: [TEST_ENV_KEY] }, + public_listing: { tier: "supported" }, + // Mode is DERIVED from these facts, so the synthetic manifest has to + // declare a coherent pair rather than just asserting a mode string: + // the manual variant is an interactive-login connector that has not + // declared session persistence (the Chase/USAA shape), and the + // automatic variant needs no per-run owner gesture. + refresh_policy: { + background_safe: recommendedMode === "automatic", + interaction_posture: recommendedMode === "automatic" ? "none" : "otp_likely", + rationale: "Synthetic manifest for the scheduler integration contract.", + recommended_interval_seconds: 3600, + recommended_mode: recommendedMode, + }, }, + connector_id: `test-auto-enroll-${recommendedMode}`, + display_name: `Test auto-enroll (${recommendedMode})`, + manifest_uri: `https://registry.pdpp.dev/connectors/test-auto-enroll-${recommendedMode}`, + protocol_version: "0.1.0", + runtime_requirements: {}, + streams: [ + { + name: "records", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "1.0.0", }, - connector_id: `https://registry.pdpp.dev/connectors/test-auto-enroll-${recommendedMode}`, - display_name: `Test auto-enroll (${recommendedMode})`, - manifest_uri: `https://registry.pdpp.dev/connectors/test-auto-enroll-${recommendedMode}`, - protocol_version: "0.1.0", - runtime_requirements: {}, - streams: [ - { - name: "records", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "1.0.0", - }; + { sourceId: `https://registry.pdpp.dev/connectors/test-auto-enroll-${recommendedMode}` } + ); } function withTmpDb(fn: () => Promise): () => Promise { @@ -281,7 +288,7 @@ test( ); test( - "a manual-default manifest does not auto-enroll but accepts an explicit owner schedule", + "a background-unsafe manual manifest neither auto-enrolls nor accepts an enabled owner schedule", withTmpDb(async () => { const manifest = testManifest("manual"); const connectorId = manifest.connector_id as string; @@ -295,6 +302,30 @@ test( assert.equal(summary.enrolled, 0, "manual-default connectors never auto-enroll on boot"); assert.equal(await controller.getSchedule(connectorId), null); + await assert.rejects(controller.upsertSchedule(connectorId, { enabled: true, interval_seconds: 21_600 }), { + code: "invalid_request", + message: "Connector refresh policy is not background-safe; automatic scheduling is disabled.", + }); + assert.equal(await controller.getSchedule(connectorId), null, "rejected scheduling must not persist a row"); + }) +); + +test( + "a background-safe connector accepts an explicit owner schedule when default enrollment is disabled", + withTmpDb(async () => { + const manifest = testManifest(); + const connectorId = manifest.connector_id as string; + await registerConnector(manifest); + const controller = createController({}); + const summary = await autoEnrollEligibleSchedules({ + controller, + enabled: false, + env: { [TEST_ENV_KEY]: "integration-token" }, + listConnectors: buildListConnectors(manifest), + }); + assert.equal(summary.enrolled, 0, "disabled default enrollment must not create a schedule"); + assert.equal(await controller.getSchedule(connectorId), null); + const result = await controller.upsertSchedule(connectorId, { enabled: true, interval_seconds: 21_600, diff --git a/reference-implementation/test/browser-enrollment-shell-route.test.ts b/reference-implementation/test/browser-enrollment-shell-route.test.ts index e6e9ce8c4..9caccf28a 100644 --- a/reference-implementation/test/browser-enrollment-shell-route.test.ts +++ b/reference-implementation/test/browser-enrollment-shell-route.test.ts @@ -15,6 +15,7 @@ import { getDb } from "../server/db.ts"; import { startServer } from "../server/index.ts"; import { BROWSER_ENROLLMENT_SHELL_TTL_MS } from "../server/routes/ref-browser-enrollment-shell.ts"; import { createSqliteConnectorInstanceStore } from "../server/stores/connector-instance-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; // Integration coverage for the browser-enrollment shell routes: // POST /_ref/connectors/:connectorId/browser-enrollment-shell (on AS) @@ -309,12 +310,12 @@ test("browser-enrollment shell: creates draft for a browser-bound connector outs await withServer(async ({ asUrl }) => { // A connector installed from the runtime catalog: its registered manifest // declares a browser binding, and no generated key list names it. - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...(loadManifest("amazon") as Record), connector_id: "acme-shop", connector_key: "acme-shop", display_name: "Acme Shop", - }; + }); const registered = await fetch(`${asUrl}/connectors`, { body: JSON.stringify(manifest), headers: { "Content-Type": "application/json" }, diff --git a/reference-implementation/test/checkpoint-dependency-profile-conformance-postgres.test.ts b/reference-implementation/test/checkpoint-dependency-profile-conformance-postgres.test.ts index 8ef5a08fb..57247ad08 100644 --- a/reference-implementation/test/checkpoint-dependency-profile-conformance-postgres.test.ts +++ b/reference-implementation/test/checkpoint-dependency-profile-conformance-postgres.test.ts @@ -138,7 +138,7 @@ if (POSTGRES_URL) { }); }); - test("SQLite/Postgres parity scenario (Postgres side): only a DETAIL_COVERAGE_KEY_SETS connector needs a record behind a hydrated key", async () => { + test("SQLite/Postgres parity scenario (Postgres side): a DETAIL_COVERAGE_KEY_SETS-declaring connector still needs an accepted record behind a hydrated key when START omits the extension; a nondeclaring connector's hydration credits by assertion", async () => { await withScenarioServer(baseUrl, async (server) => { const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; diff --git a/reference-implementation/test/checkpoint-dependency-profile-conformance.test.ts b/reference-implementation/test/checkpoint-dependency-profile-conformance.test.ts index 4eb01108f..92eec57c5 100644 --- a/reference-implementation/test/checkpoint-dependency-profile-conformance.test.ts +++ b/reference-implementation/test/checkpoint-dependency-profile-conformance.test.ts @@ -56,6 +56,7 @@ import { runSubsetParentCoverageScenario, runUndeclaredParentScenario, } from "./helpers/checkpoint-dependency-multi-parent-scenario.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; const FAILING_STREAM_NAME_PATTERN = /sibling_b/; @@ -152,30 +153,33 @@ function fakeAdmitRunConnection( // stream-scoped-failure exception (see runtime-stream-collection-failed-commit.test.ts // for the un-cancelled counterpart of this exact shape). function twoSiblingManifest(connectorId: string) { - return { - capabilities: { human_interaction: [] }, - connector_id: connectorId, - display_name: connectorId, - manifest_uri: `https://sources.example/${connectorId}`, - protocol_version: "0.1.0", - streams: [ - { - name: "sibling_a", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - name: "sibling_b", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "0.1.0", - }; + return withFixtureSourceDeclaration( + { + capabilities: { human_interaction: [] }, + connector_id: connectorId, + display_name: connectorId, + manifest_uri: `https://sources.example/${connectorId}`, + protocol_version: "0.1.0", + streams: [ + { + name: "sibling_a", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + name: "sibling_b", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "0.1.0", + }, + { sourceId: `https://sources.example/${connectorId}` } + ); } function writeConnectorStub(tmpDir: string, script: string): string { @@ -424,7 +428,7 @@ test("a stream declaring state_stream/parent_streams without the matching covera try { // state_stream requires coverage_strategy: "checkpoint_window" on the // SAME stream (spec Validation rule 7). Here it is entirely absent. - const missingStrategyManifest = { + const missingStrategyManifest = withFixtureSourceDeclaration({ capabilities: { human_interaction: [] }, connector_id: "checkpoint-profile-coverage-strategy-missing-test", display_name: "checkpoint-profile-coverage-strategy-missing-test", @@ -439,6 +443,7 @@ test("a stream declaring state_stream/parent_streams without the matching covera semantics: "mutable_state", }, { + coverage_strategy: "checkpoint_window", name: "child", primary_key: ["id"], schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, @@ -448,7 +453,12 @@ test("a stream declaring state_stream/parent_streams without the matching covera }, ], version: "0.1.0", - }; + }); + const missingStrategyStream = missingStrategyManifest.streams.find((stream) => stream.name === "child"); + if (!missingStrategyStream) { + throw new Error("missing child fixture stream"); + } + Reflect.deleteProperty(missingStrategyStream, "coverage_strategy"); const missingStrategyResp = await fetch(`${asUrl}/connectors`, { body: JSON.stringify(missingStrategyManifest), headers: { "Content-Type": "application/json" }, @@ -464,7 +474,7 @@ test("a stream declaring state_stream/parent_streams without the matching covera // on the SAME stream. Here it is set to an unrelated valid value // ("full_inventory") instead of being absent, proving the mismatch // check — not merely an absence check. - const mismatchedStrategyManifest = { + const mismatchedStrategyManifest = withFixtureSourceDeclaration({ capabilities: { human_interaction: [] }, connector_id: "checkpoint-profile-coverage-strategy-mismatch-test", display_name: "checkpoint-profile-coverage-strategy-mismatch-test", @@ -486,7 +496,7 @@ test("a stream declaring state_stream/parent_streams without the matching covera semantics: "mutable_state", }, { - coverage_strategy: "full_inventory", + coverage_strategy: "parent_detail_accounting", name: "detail", parent_streams: ["parent_a", "parent_b"], primary_key: ["id"], @@ -496,7 +506,12 @@ test("a stream declaring state_stream/parent_streams without the matching covera }, ], version: "0.1.0", - }; + }); + const mismatchedStrategyStream = mismatchedStrategyManifest.streams.find((stream) => stream.name === "detail"); + if (!mismatchedStrategyStream) { + throw new Error("missing detail fixture stream"); + } + mismatchedStrategyStream.coverage_strategy = "full_inventory"; const mismatchedStrategyResp = await fetch(`${asUrl}/connectors`, { body: JSON.stringify(mismatchedStrategyManifest), headers: { "Content-Type": "application/json" }, @@ -509,7 +524,7 @@ test("a stream declaring state_stream/parent_streams without the matching covera ); // Positive control: the correct coverage_strategy on each shape registers successfully. - const validManifest = { + const validManifest = withFixtureSourceDeclaration({ capabilities: { human_interaction: [] }, connector_id: "checkpoint-profile-coverage-strategy-valid-test", display_name: "checkpoint-profile-coverage-strategy-valid-test", @@ -534,7 +549,7 @@ test("a stream declaring state_stream/parent_streams without the matching covera }, ], version: "0.1.0", - }; + }); const validResp = await fetch(`${asUrl}/connectors`, { body: JSON.stringify(validManifest), headers: { "Content-Type": "application/json" }, @@ -568,7 +583,7 @@ test("SQLite/Postgres parity scenario (SQLite side): one detail stream independe } }); -test("SQLite/Postgres parity scenario (SQLite side): only a DETAIL_COVERAGE_KEY_SETS connector needs a record behind a hydrated key", async () => { +test("SQLite/Postgres parity scenario (SQLite side): a DETAIL_COVERAGE_KEY_SETS-declaring connector still needs an accepted record behind a hydrated key when START omits the extension; a nondeclaring connector's hydration credits by assertion", async () => { const server = await startServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; diff --git a/reference-implementation/test/cli.test.ts b/reference-implementation/test/cli.test.ts index 3c78d2135..0e6c059ce 100644 --- a/reference-implementation/test/cli.test.ts +++ b/reference-implementation/test/cli.test.ts @@ -7181,7 +7181,8 @@ rl.on('line', (line) => { await t.test("run timeline keeps partial checkpoint commit artifacts inspectable", async () => { const manifest = { - connector_id: "https://registry.pdpp.dev/connectors/cli-run-partial-checkpoint-test", + connector_id: "cli-run-partial-checkpoint-test", + connector_key: "cli-run-partial-checkpoint-test", source_declaration: { declaration_version: "cli-run-partial-checkpoint-test-v1", display: { name: "CLI Run Partial Checkpoint Test" }, diff --git a/reference-implementation/test/collection-profile-020.test.ts b/reference-implementation/test/collection-profile-020.test.ts index d1504adb7..337ac50ed 100644 --- a/reference-implementation/test/collection-profile-020.test.ts +++ b/reference-implementation/test/collection-profile-020.test.ts @@ -34,6 +34,9 @@ const PROTOCOL_VERSION_NAMED = /protocol_version/; const OBSERVATION_NEEDS_020 = /OBSERVATION require protocol_version 0\.2\.0/; const DIAGNOSTIC_DESCRIPTORS_NEEDS_020 = /diagnostic_descriptors require protocol_version 0\.2\.0/; const UNDECLARED_STEP = /descriptor_undeclared_step/; +const BLOB_REFUSED = /BLOB/; +// A refusal before spawn carries a stable code the scheduler and console can key on. +const CAPABILITY_REFUSAL_REASON = "protocol_capability_unsupported"; const RETRYABLE_FIELD = /DONE\.error\.retryable/; // Backend under test. With the dedicated private PostgreSQL URL the runtime, the // admission store and the observation store all use Postgres. Without it the @@ -132,6 +135,7 @@ type RunOutcome = readonly ok: false; readonly message: string; readonly connectorError: { readonly message?: string } | null; + readonly failureReason: string | null; }; // A runtime rejection becomes a value, so a failing assertion names the @@ -187,7 +191,8 @@ async function runStub( return { ok: true, result: result as unknown as Record }; } catch (caught) { const connectorError = (caught as { connector_error?: { message?: string } | null }).connector_error ?? null; - return { connectorError, message: (caught as Error).message, ok: false }; + const failureReason = (caught as { failure_reason?: string }).failure_reason ?? null; + return { connectorError, failureReason, message: (caught as Error).message, ok: false }; } finally { for (const [key, value] of Object.entries(savedEnv)) { if (value === undefined) { @@ -241,6 +246,11 @@ test("declared 0.2.0 OBSERVATION facts are accepted and the run succeeds", async assert.equal(outcome.result.status, "succeeded"); const start = JSON.parse(readFileSync(stub.markerPath, "utf8")) as { protocol_capabilities: string[] }; assert.ok(start.protocol_capabilities.includes("OBSERVATION"), "START advertises the implemented message"); + assert.equal( + start.protocol_capabilities.includes("DETAIL_COVERAGE_KEY_SETS"), + false, + "the runtime must not promise key-set extension credit rules it does not implement" + ); const record = await persistedRecord("run_c020_accept"); assert.ok(record, "the run's bounded diagnostic record must be persisted"); assert.deepEqual( @@ -513,6 +523,7 @@ test("OBSERVATION or diagnostic_descriptors without protocol_version 0.2.0 are r assert.equal(outcome.ok, false, "a 0.1.0 manifest may not declare OBSERVATION"); if (!outcome.ok) { assert.match(outcome.message, OBSERVATION_NEEDS_020); + assert.equal(outcome.failureReason, CAPABILITY_REFUSAL_REASON); } assert.equal(existsMarker(stub.markerPath), false, "the child must not execute"); } finally { @@ -556,6 +567,25 @@ test("an expectation naming an undeclared step is rejected before spawn", async } }); +test("a required BLOB capability is rejected before spawn, not waived", async () => { + const stub = makeStub([{ records_emitted: 0, status: "succeeded", type: "DONE" }], 0); + try { + const outcome = await runStub( + stub, + "run_c020_blob", + manifest({ diagnostic_descriptors: undefined, protocol_capabilities: ["BLOB"], protocol_version: "0.1.0" }) + ); + assert.equal(outcome.ok, false, "BLOB is not implemented, so a connector that requires it must not start"); + if (!outcome.ok) { + assert.match(outcome.message, BLOB_REFUSED); + assert.equal(outcome.failureReason, CAPABILITY_REFUSAL_REASON); + } + assert.equal(existsMarker(stub.markerPath), false, "the child must not execute"); + } finally { + rmSync(stub.dir, { force: true, recursive: true }); + } +}); + test("a failed DONE that cites an accepted fact yields a connector-attributed diagnosis and keeps the original text", async () => { const original = "Sign-in was refused by the provider"; const stub = makeStub( diff --git a/reference-implementation/test/collection-profile-binding-declarations.test.ts b/reference-implementation/test/collection-profile-binding-declarations.test.ts new file mode 100644 index 000000000..cd3b0415c --- /dev/null +++ b/reference-implementation/test/collection-profile-binding-declarations.test.ts @@ -0,0 +1,399 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Collection Profile binding declarations, one canonical validator for every + * entry point (data-connectors a87edcdc099284e8582d24a60c9630474bd2af8b). + * + * Three entry points are covered: the registration and stored-manifest gate + * (`validateConnectorManifest`), the direct `runConnector` path before spawn, + * and raw manifest JSON text (duplicate keys). A spawn is proven absent by a + * capture file that only the spawned connector writes. + * + * What passing means: a declaration is well formed, or the runtime can supply + * its required contract. It never means the runtime enforces an authority. + * Section 3.3.10 keeps every channel's coverage separate from this check. + */ + +import assert from "node:assert/strict"; +import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test, { after, before } from "node:test"; + +import { + BindingDeclarationError, + BindingPlacementError, + normalizeBindingDeclarations, + parseManifestJsonText, + startBindingsFor, +} from "../runtime/binding-declarations.ts"; +import { type RuntimeRunConnectorOptions, runConnector } from "../runtime/index.ts"; +import { validateConnectorManifest } from "../server/connector-manifest-validation.ts"; +import { closeDb, initDb } from "../server/db.ts"; + +const MINIMAL_MANIFEST = { + connector_id: "test", + display_name: "Test Connector", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + semantics: "append_only", + }, + ], + version: "1.0.0", +}; + +function withBindings(bindings: unknown): Record { + return { ...MINIMAL_MANIFEST, runtime_requirements: { bindings } }; +} + +// Assertion patterns, hoisted to module scope. +const REQUIRED_FLAG = /required/; +const INTERACTIVE_INSTANCE = /interactive/; +const SHORTHAND_KIND_FORBIDDEN = /browser must NOT be valid/; +const INSTANCE_KIND = /kind/; +const UNKNOWN_SHORTHAND = /scanner/; +const BINDINGS_OBJECT = /bindings/; +const EXTERNAL_TOOLS = /external_tools/; +const INTERFACE_SHAPE = /interface/; +const CANONICAL_NAVIGATION_HOST = /navigate\.0 must match pattern/; +const RATIONALE_REQUIRED = /rationale/; + +// runConnector reads detail-gap state from the DB before spawn, so it needs one. +before(() => { + initDb(":memory:"); +}); + +after(() => { + closeDb(); +}); + +// --------------------------------------------------------------------------- +// 1. Registration and stored-manifest gate: validateConnectorManifest +// --------------------------------------------------------------------------- + +test("registration gate: a required binding must state a boolean required flag", () => { + // Canonical 3.3: `required` is REQUIRED. Old gate accepted an absent flag. + assert.throws(() => validateConnectorManifest(withBindings({ browser: {} })), REQUIRED_FLAG); +}); + +test("registration gate: `interactive` is removed as a binding and is not accepted", () => { + // Canonical 3.3 removed-name table. Old gate accepted it as a registry name. + assert.throws( + () => validateConnectorManifest(withBindings({ interactive: { required: true } })), + INTERACTIVE_INSTANCE + ); + assert.throws( + () => validateConnectorManifest(withBindings({ interactive: { required: false } })), + INTERACTIVE_INSTANCE + ); +}); + +test("registration gate: a shorthand key must not carry kind; a named instance must", () => { + assert.throws( + () => validateConnectorManifest(withBindings({ browser: { kind: "network", required: false } })), + SHORTHAND_KIND_FORBIDDEN + ); + assert.throws(() => validateConnectorManifest(withBindings({ chase_site: { required: true } })), INSTANCE_KIND); +}); + +test("registration gate: a named instance with an explicit kind is valid", () => { + assert.doesNotThrow(() => + validateConnectorManifest(withBindings({ chase_site: { kind: "browser", required: true } })) + ); +}); + +test("registration gate: an unqualified unknown optional shorthand is rejected", () => { + assert.throws(() => validateConnectorManifest(withBindings({ scanner: { required: false } })), UNKNOWN_SHORTHAND); +}); + +test("registration gate: a canonical namespaced extension is shape-valid", () => { + // Shape-valid only. Placement, not validation, refuses it when required. + assert.doesNotThrow(() => validateConnectorManifest(withBindings({ "example.com/scanner": { required: true } }))); +}); + +test("registration gate: null bindings is rejected, since the canonical map is an object", () => { + // Canonical schema: `bindings` is type object. Old gate treated null as absent. + assert.throws(() => validateConnectorManifest(withBindings(null)), BINDINGS_OBJECT); +}); + +test("registration gate: external_tools is validated even when bindings is absent", () => { + // Old gate returned early on absent bindings and skipped external_tools. + const manifest = { ...MINIMAL_MANIFEST, runtime_requirements: { external_tools: "not-an-array" } }; + assert.throws(() => validateConnectorManifest(manifest), EXTERNAL_TOOLS); +}); + +test("registration gate: interface must match the resolved kind, with no leading zero", () => { + assert.throws( + () => + validateConnectorManifest( + withBindings({ chase_site: { interface: "browser@01", kind: "browser", required: true } }) + ), + INTERFACE_SHAPE + ); + assert.throws( + () => + validateConnectorManifest( + withBindings({ chase_site: { interface: "network@1", kind: "browser", required: true } }) + ), + INTERFACE_SHAPE + ); + assert.doesNotThrow(() => + validateConnectorManifest(withBindings({ chase_site: { interface: "browser@2", kind: "browser", required: true } })) + ); +}); + +test("registration gate: a web host constraint must be in canonical form", () => { + assert.throws( + () => + validateConnectorManifest( + withBindings({ browser: { constraints: { navigate: ["https://Secure.Chase.com"] }, required: true } }) + ), + CANONICAL_NAVIGATION_HOST + ); +}); + +test("registration gate: a desktop_session item is an exceptional grant and needs a rationale", () => { + const item = { operations: ["unwrap"], selector: "signal-safestorage", service: "os_keyring" }; + assert.throws( + () => + validateConnectorManifest( + withBindings({ signal_keyring: { constraints: { items: [item] }, kind: "desktop_session", required: true } }) + ), + RATIONALE_REQUIRED + ); + assert.doesNotThrow(() => + validateConnectorManifest( + withBindings({ + signal_keyring: { + constraints: { items: [item] }, + kind: "desktop_session", + rationale: "Keyring unwraps the local database key.", + required: true, + }, + }) + ) + ); +}); + +test("registration gate: a valid shorthand manifest keeps its meaning (baseline)", () => { + assert.doesNotThrow(() => validateConnectorManifest(withBindings({ network: { required: true } }))); + assert.doesNotThrow(() => validateConnectorManifest(withBindings({ filesystem: { required: false } }))); +}); + +// --------------------------------------------------------------------------- +// 2. Direct runConnector before spawn: the same validator, no child process +// --------------------------------------------------------------------------- + +function fakeAdmitRunConnection(): NonNullable { + return ({ connectorId, connectorInstanceId, ownerSubjectId }) => { + const owner = ownerSubjectId || "owner_binding_test"; + const exactId = connectorInstanceId ?? `cin_${owner}_${connectorId.replace(/[^a-z0-9]+/gi, "_")}`; + return Promise.resolve({ connectorId, connectorInstanceId: exactId, ownerSubjectId: owner }); + }; +} + +/** A connector that writes the START it receives to `capturePath`, then succeeds. */ +function startCaptureConnector(capturePath: string): { cleanup: () => void; connectorPath: string } { + const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-binding-start-capture-")); + const connectorPath = join(tmpDir, "connector.mjs"); + writeFileSync( + connectorPath, + ` +import { createInterface } from 'readline'; +import { writeFileSync } from 'node:fs'; +writeFileSync(${JSON.stringify(`${capturePath}.spawned`)}, 'spawned'); +const rl = createInterface({ input: process.stdin, terminal: false }); +rl.on('line', (line) => { + const msg = JSON.parse(line); + if (msg.type === 'START') { + writeFileSync(${JSON.stringify(capturePath)}, JSON.stringify(msg, null, 2)); + process.stdout.write(JSON.stringify({ type: 'DONE', status: 'succeeded', records_emitted: 0 }) + '\\n'); + rl.close(); + process.exit(0); + } +}); +`, + "utf-8" + ); + return { cleanup: () => rmSync(tmpDir, { force: true, recursive: true }), connectorPath }; +} + +interface DirectRunResult { + error: Error | null; + spawned: boolean; + start: Record | null; +} + +async function runWithBindings(bindings: unknown): Promise { + const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-binding-direct-")); + const capturePath = join(tmpDir, "start.json"); + const { cleanup, connectorPath } = startCaptureConnector(capturePath); + let error: Error | null = null; + try { + await runConnector({ + admitRunConnection: fakeAdmitRunConnection(), + collectionMode: "full_refresh", + connectorId: "test", + connectorPath, + manifest: withBindings(bindings), + onInteraction: async () => ({ data: {}, request_id: "unused", status: "success", type: "INTERACTION_RESPONSE" }), + ownerToken: "test", + persistState: true, + rsUrl: "http://localhost:9999", + state: null, + } as unknown as RuntimeRunConnectorOptions); + } catch (caught) { + error = caught instanceof Error ? caught : new Error(String(caught)); + } + // The launch marker is independent of START; a failure before START cannot hide a spawn. + const spawned = existsSync(`${capturePath}.spawned`); + const start = existsSync(capturePath) + ? (JSON.parse(readFileSync(capturePath, "utf8")) as Record) + : null; + cleanup(); + rmSync(tmpDir, { force: true, recursive: true }); + return { error, spawned, start }; +} + +test("direct run: a malformed binding (missing required) is refused before spawn", async () => { + const result = await runWithBindings({ browser: {} }); + assert.ok(result.error instanceof BindingDeclarationError); + assert.match(result.error.message, REQUIRED_FLAG); + assert.equal(result.spawned, false); +}); + +test("direct run: `interactive` is refused before spawn", async () => { + const result = await runWithBindings({ interactive: { required: true } }); + assert.ok(result.error instanceof BindingDeclarationError); + assert.match(result.error.message, INTERACTIVE_INSTANCE); + assert.equal(result.spawned, false); +}); + +test("direct run: a required unsupported feature is refused before spawn", async () => { + // Old direct path ignored features and spawned the connector. + const result = await runWithBindings({ browser: { features: ["page_navigation"], required: true } }); + assert.ok(result.error instanceof BindingPlacementError); + assert.deepEqual(result.error.bindingKeys, ["browser"]); + assert.ok(result.error.message.includes("features page_navigation are not provided")); + assert.equal(result.spawned, false); +}); + +test("direct run: a required unsupported constraint is refused before spawn", async () => { + const result = await runWithBindings({ + filesystem: { + constraints: { outputs: [{ access: "write", slot: "statements" }] }, + required: true, + }, + }); + assert.ok(result.error instanceof BindingPlacementError); + assert.deepEqual(result.error.bindingKeys, ["filesystem"]); + assert.ok(result.error.message.includes("does not enforce them")); + assert.equal(result.spawned, false); +}); + +test("direct run: a required unsupported interface version is refused before spawn", async () => { + const result = await runWithBindings({ chase_site: { interface: "browser@2", kind: "browser", required: true } }); + assert.ok(result.error instanceof BindingPlacementError); + assert.deepEqual(result.error.bindingKeys, ["chase_site"]); + assert.ok(result.error.message.includes("interface browser@2 is not implemented")); + assert.equal(result.spawned, false); +}); + +test("direct run: a required namespaced extension is refused before spawn", async () => { + const result = await runWithBindings({ "example.com/scanner": { required: true } }); + assert.ok(result.error instanceof BindingPlacementError); + assert.deepEqual(result.error.bindingKeys, ["example.com/scanner"]); + assert.ok(result.error.message.includes("kind example.com/scanner is not implemented")); + assert.equal(result.spawned, false); +}); + +test("direct run: an optional unsupported extension is omitted and START does not advertise it", async () => { + const result = await runWithBindings({ "example.com/scanner": { required: false } }); + assert.equal(result.error, null); + assert.equal(result.spawned, true); + assert.equal(Object.hasOwn(result.start?.bindings as object, "example.com/scanner"), false); +}); + +test("direct run: a required named instance is refused until the runtime supplies per-instance handles", async () => { + const result = await runWithBindings({ chase_site: { kind: "browser", required: true } }); + assert.ok(result.error instanceof BindingPlacementError); + assert.deepEqual(result.error.bindingKeys, ["chase_site"]); + assert.ok(result.error.message.includes("per-instance handles")); + assert.equal(result.spawned, false); +}); + +test("START encoding keeps resolved instance keys and omits a same-kind instance separately", () => { + // Encoder-only test with a supplied plan. This is not evidence that native + // execution can resolve these handles; the direct-run refusal above proves + // that this runtime currently cannot. + const instances = normalizeBindingDeclarations({ + optional_page: { features: ["page_navigation"], kind: "browser", required: false }, + provider_page: { kind: "browser", required: true }, + }); + const placed = instances.filter((instance) => instance.key === "provider_page"); + const optional = instances.find((instance) => instance.key === "optional_page"); + assert.ok(optional); + const bindings = startBindingsFor(instances, { + omitted: [{ instance: optional, reason: "unsupported feature" }], + placed, + }); + assert.deepEqual(bindings.provider_page, {}); + assert.equal(Object.hasOwn(bindings, "optional_page"), false); + assert.equal(Object.hasOwn(bindings, "browser"), false); +}); + +test("direct run: an omitted optional browser with unsupported features is not advertised in START", async () => { + const result = await runWithBindings({ browser: { features: ["page_navigation"], required: false } }); + assert.equal(result.error, null); + assert.equal(result.spawned, true); + assert.equal(Object.hasOwn(result.start?.bindings as object, "browser"), false); +}); + +test("direct run: a valid shorthand baseline spawns and advertises the static kinds", async () => { + const result = await runWithBindings({ network: { required: true } }); + assert.equal(result.error, null); + assert.equal(result.spawned, true); + const bindings = result.start?.bindings as Record; + assert.ok(Object.hasOwn(bindings, "network")); + assert.ok(Object.hasOwn(bindings, "browser")); + assert.ok(Object.hasOwn(bindings, "filesystem")); + assert.equal(Object.hasOwn(bindings, "interactive"), false); +}); + +// --------------------------------------------------------------------------- +// 3. Raw manifest JSON text: duplicate keys are rejected before last-wins +// --------------------------------------------------------------------------- + +test("raw JSON: a repeated instance key is rejected, not last-wins", () => { + const text = '{"runtime_requirements":{"bindings":{"browser":{"required":true},"browser":{"required":false}}}}'; + assert.throws( + () => parseManifestJsonText(text), + (error: unknown) => (error as { code?: string }).code === "duplicate_json_key" + ); +}); + +test("raw JSON: an escaped spelling of the same key is a duplicate", () => { + const text = '{"bindings":{"browser":{"required":true},"\\u0062rowser":{"required":false}}}'; + assert.throws( + () => parseManifestJsonText(text), + (error: unknown) => (error as { code?: string }).code === "duplicate_json_key" + ); +}); + +test("raw JSON: a repeated nested member is rejected anywhere in the text", () => { + const text = '{"runtime_requirements":{"bindings":{"network":{"required":true,"required":false}}}}'; + assert.throws( + () => parseManifestJsonText(text), + (error: unknown) => (error as { code?: string }).code === "duplicate_json_key" + ); +}); + +test("raw JSON: a text without duplicate keys parses to its value (baseline)", () => { + const parsed = parseManifestJsonText('{"runtime_requirements":{"bindings":{"network":{"required":true}}}}'); + assert.deepEqual(parsed, { runtime_requirements: { bindings: { network: { required: true } } } }); +}); diff --git a/reference-implementation/test/collection-profile-owner-journey.test.ts b/reference-implementation/test/collection-profile-owner-journey.test.ts index cfc215e6f..05973121c 100644 --- a/reference-implementation/test/collection-profile-owner-journey.test.ts +++ b/reference-implementation/test/collection-profile-owner-journey.test.ts @@ -2,8 +2,8 @@ // SPDX-License-Identifier: Apache-2.0 // Real HTTP/session + durable store boundary, beyond the route's fake adapter. -// Collection Profile 5.12: keep diagnostic records on the owner surface, -// uncached and namespace-scoped. +// Collection Profile 3.4 and 5.12: advertise support before any run, and keep +// diagnostic records on the owner surface, uncached and namespace-scoped. import assert from "node:assert/strict"; import test from "node:test"; import { closeDb, getDb } from "../server/db.ts"; @@ -24,7 +24,7 @@ import { dedicatedPostgresTestUrl } from "./helpers/dedicated-postgres-test-url. const POSTGRES_URL = dedicatedPostgresTestUrl(process.env.PDPP_TEST_POSTGRES_URL); const OWNER_PASSWORD = "collection-profile-owner-fixture-password"; -test("owner can read diagnostics through the real session gate", async () => { +test("owner can discover profile support and read diagnostics through the real session gate", async () => { if (POSTGRES_URL) { await initPostgresStorage({ backend: "postgres", databaseUrl: POSTGRES_URL }); } @@ -54,6 +54,20 @@ test("owner can read diagnostics through the real session gate", async () => { assert.ok(cookie, "runtime-owned sign-in must issue a real owner session"); const headers = { Accept: "application/json", Cookie: cookie }; + const supportResponse = await fetch(`${origin}/_ref/collection-profile`, { headers }); + assert.equal(supportResponse.status, 200); + assert.deepEqual( + await supportResponse.json(), + { + compatible_protocol_versions: ["0.1.0", "0.2.0"], + conformance: { outstanding: ["binding_authority"], status: "partial" }, + object: "collection_profile_support", + protocol_capabilities: ["OBSERVATION", "STREAM_EVIDENCE"], + protocol_version: "0.2.0", + }, + "profile support is discoverable before a connector starts" + ); + const connectorId = "owner-observations-fixture"; if (POSTGRES_URL) { await postgresQuery("INSERT INTO connectors(connector_id, manifest) VALUES ($1, '{}'::jsonb)", [connectorId]); diff --git a/reference-implementation/test/collection-profile.test.ts b/reference-implementation/test/collection-profile.test.ts index 7437b7f49..5035cb897 100644 --- a/reference-implementation/test/collection-profile.test.ts +++ b/reference-implementation/test/collection-profile.test.ts @@ -26,7 +26,6 @@ import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; - import { RECOVERY_ACTIONS } from "../runtime/connector-gap-bounding.ts"; import { cursorAuditFact } from "../runtime/cursor-audit-fact.ts"; import { @@ -44,6 +43,7 @@ import { createSqliteConnectorInstanceStore, } from "../server/stores/connector-instance-store.ts"; import { TEST_PRE_REGISTERED_PUBLIC_CLIENTS } from "./fixtures/demo-clients.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; type TestServer = Awaited>; @@ -712,6 +712,13 @@ const MINIMAL_MANIFEST = { version: "1.0.0", }; +// Section 3.4: an INTERACTION.kind must be declared in capabilities.human_interaction. +// Tests that emit INTERACTION use this manifest. Undeclared-kind refusal uses MINIMAL_MANIFEST. +const HUMAN_INTERACTION_MANIFEST = { + ...MINIMAL_MANIFEST, + capabilities: { human_interaction: ["credentials", "otp", "manual_action"] }, +}; + function buildMultiStreamManifest(connectorId = "test-multi-stream") { return { ...MINIMAL_MANIFEST, @@ -809,7 +816,6 @@ test("Collection Profile conformance", async (t) => { assert.deepEqual(captured.bindings, { browser: {}, filesystem: {}, - interactive: {}, network: {}, }); assert.ok(!("config" in captured), "legacy START.config should be absent"); @@ -2803,11 +2809,17 @@ rl.on('line', (line) => { assert.equal(advancedEvents.length, 2); assert.deepEqual( new Set(stagedEvents.map((event) => eventCursorKey(event))), - new Set([cursorEventKey("items", "cursor_items_success"), cursorEventKey("other_items", "cursor_other_success")]) + new Set([ + cursorEventKey("items", "cursor_items_success"), + cursorEventKey("other_items", "cursor_other_success"), + ]) ); assert.deepEqual( new Set(advancedEvents.map((event) => eventCursorKey(event))), - new Set([cursorEventKey("items", "cursor_items_success"), cursorEventKey("other_items", "cursor_other_success")]) + new Set([ + cursorEventKey("items", "cursor_items_success"), + cursorEventKey("other_items", "cursor_other_success"), + ]) ); assert.deepEqual( advancedEvents.map((event) => event.data.state_streams_committed as number).sort((a, b) => a - b), @@ -2943,6 +2955,8 @@ rl.on('line', (line) => { required: ["id"], type: "object", }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", }, { name: "other_items", @@ -2955,6 +2969,8 @@ rl.on('line', (line) => { required: ["id"], type: "object", }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", }, ], version: "0.1.0", @@ -3583,55 +3599,53 @@ rl.on('line', (line) => { } }); - await t.test("runtime satisfies required interactive bindings and advertises them in START", async () => { - const manifestWithInteractiveBinding = { - ...MINIMAL_MANIFEST, - runtime_requirements: { - bindings: { - interactive: { required: true }, - }, - }, - }; + await t.test( + "START.bindings never advertises interactive, even when the manifest declares human_interaction", + async () => { + const manifestWithInteractiveBinding = { + ...MINIMAL_MANIFEST, + capabilities: { human_interaction: ["credentials"] }, + }; - const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-interactive-binding-capture-")); - const capturePath = join(tmpDir, "start.json"); - const { connectorPath, cleanup } = createStartCaptureConnector(capturePath); + const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-interactive-binding-capture-")); + const capturePath = join(tmpDir, "start.json"); + const { connectorPath, cleanup } = createStartCaptureConnector(capturePath); - try { - const result = await runTestConnector({ - // Bare runtime unit test: no server, no registered connector, an - // unreachable rsUrl. A pure-echo admission just needs to clear the - // gate so the START.bindings capture under test is reached. - admitRunConnection: fakeEchoAdmitRunConnection(), - collectionMode: "full_refresh", - connectorId: "test", - connectorPath, - manifest: manifestWithInteractiveBinding, - onInteraction: async () => ({ - data: {}, - request_id: "unused", - status: "success", - type: "INTERACTION_RESPONSE", - }), - ownerToken: "test", - persistState: true, - rsUrl: "http://localhost:9999", - state: null, - }); + try { + const result = await runTestConnector({ + // Bare runtime unit test: no server, no registered connector, an + // unreachable rsUrl. A pure-echo admission just needs to clear the + // gate so the START.bindings capture under test is reached. + admitRunConnection: fakeEchoAdmitRunConnection(), + collectionMode: "full_refresh", + connectorId: "test", + connectorPath, + manifest: manifestWithInteractiveBinding, + onInteraction: async () => ({ + data: {}, + request_id: "unused", + status: "success", + type: "INTERACTION_RESPONSE", + }), + ownerToken: "test", + persistState: true, + rsUrl: "http://localhost:9999", + state: null, + }); - assert.equal(result.status, "succeeded"); - const captured = JSON.parse(readFileSync(capturePath, "utf8")); - assert.deepEqual(captured.bindings, { - browser: {}, - filesystem: {}, - interactive: {}, - network: {}, - }); - } finally { - cleanup(); - rmSync(tmpDir, { force: true, recursive: true }); + assert.equal(result.status, "succeeded"); + const captured = JSON.parse(readFileSync(capturePath, "utf8")); + assert.deepEqual(captured.bindings, { + browser: {}, + filesystem: {}, + network: {}, + }); + } finally { + cleanup(); + rmSync(tmpDir, { force: true, recursive: true }); + } } - }); + ); await t.test( "runtime omits interactive from START.bindings when interaction handling is explicitly disabled", @@ -5631,7 +5645,7 @@ rl.on('line', (line) => { await t.test("runtime reports manual-action known gaps without persisting interaction responses", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-manual-gap-")); @@ -5673,7 +5687,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => ({ data: { otp: "123456", password: "supersecret" }, request_id: "manual_login", @@ -6678,7 +6692,7 @@ rl.on('line', (line) => { await t.test("INTERACTION round-trip allows connector to continue collecting", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; // Connector that emits INTERACTION, waits for response, then emits RECORD + DONE @@ -6713,7 +6727,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, // biome-ignore lint/suspicious/useAwait: localized test assertion preserves its explicit contract. onInteraction: async (message: unknown) => { const msg = message as InteractionMessage; @@ -6803,7 +6817,7 @@ rl.on('line', (line) => { await t.test("resolved OTP assistance completes a pending interaction after page advancement", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const tmpDir = mkdtempSync(join(process.cwd(), ".scratch-otp-manual-")); const connectorPath = join(tmpDir, "connector.mjs"); writeFileSync( @@ -6834,7 +6848,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => new Promise(() => { // Keep the interaction pending until the connector advances the browser page. @@ -6854,7 +6868,7 @@ rl.on('line', (line) => { await t.test("late OTP assistance status is ignored after a console response", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-otp-late-status-")); const connectorPath = join(tmpDir, "connector.mjs"); writeFileSync( @@ -6884,7 +6898,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async (message: unknown) => ({ data: { code: "123456" }, request_id: (message as InteractionMessage).request_id, @@ -6906,7 +6920,7 @@ rl.on('line', (line) => { await t.test("browser-surface-backed otp INTERACTION projects streamable assistance with secret input", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-interaction-browser-otp-")); @@ -6951,7 +6965,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, // biome-ignore lint/suspicious/useAwait: localized test assertion preserves its explicit contract. onInteraction: async (message: unknown) => { const msg = message as InteractionMessage; @@ -7003,7 +7017,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, // biome-ignore lint/suspicious/useAwait: localized test assertion preserves its explicit contract. onInteraction: async (message: unknown) => { const msg = message as InteractionMessage; @@ -7434,7 +7448,7 @@ rl.on('line', (line) => { await t.test("runtime rejects connector output emitted while waiting for INTERACTION_RESPONSE", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-blocked-output-")); @@ -7481,7 +7495,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => new Promise(() => undefined), ownerToken, persistState: true, @@ -7533,7 +7547,7 @@ rl.on('line', (line) => { async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-blocked-state-")); @@ -7578,7 +7592,7 @@ rl.on('line', (line) => { collectionMode: "incremental", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => new Promise(() => undefined), ownerToken, persistState: true, @@ -7633,7 +7647,7 @@ rl.on('line', (line) => { async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-blocked-progress-")); @@ -7678,7 +7692,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => new Promise(() => undefined), ownerToken, persistState: true, @@ -7729,7 +7743,7 @@ rl.on('line', (line) => { async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-blocked-skip-result-")); @@ -7775,7 +7789,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => new Promise(() => undefined), ownerToken, persistState: true, @@ -7824,7 +7838,7 @@ rl.on('line', (line) => { await t.test("runtime rejects a second INTERACTION emitted while waiting for INTERACTION_RESPONSE", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-overlap-")); @@ -7872,7 +7886,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => new Promise(() => undefined), ownerToken, persistState: true, @@ -7925,7 +7939,7 @@ rl.on('line', (line) => { async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-blocked-done-")); @@ -7970,7 +7984,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => new Promise(() => undefined), ownerToken, persistState: true, @@ -8023,7 +8037,7 @@ rl.on('line', (line) => { async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-blocked-invalid-jsonl-")); @@ -8064,7 +8078,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, onInteraction: async () => new Promise(() => undefined), ownerToken, persistState: true, @@ -8116,7 +8130,7 @@ rl.on('line', (line) => { await t.test("runtime returns INTERACTION timeout responses when the handler does not answer in time", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-timeout-")); @@ -8163,7 +8177,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, // biome-ignore lint/suspicious/useAwait: localized test assertion preserves its explicit contract. onInteraction: async (message: unknown) => { const msg = message as InteractionMessage; @@ -8202,7 +8216,7 @@ rl.on('line', (line) => { await t.test("runtime returns INTERACTION cancelled responses when the handler aborts", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-cancelled-")); @@ -8249,7 +8263,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, // biome-ignore lint/suspicious/useAwait: localized test assertion preserves its explicit contract. onInteraction: async (message: unknown) => { const msg = message as InteractionMessage; @@ -8290,7 +8304,7 @@ rl.on('line', (line) => { async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; - const { ownerToken, connectorId } = await setupConnector(server, asPort); + const { ownerToken, connectorId } = await setupConnector(server, asPort, HUMAN_INTERACTION_MANIFEST); const asUrl = `http://localhost:${asPort}`; const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-test-int-invalid-response-")); @@ -8328,7 +8342,7 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId, connectorPath, - manifest: MINIMAL_MANIFEST, + manifest: HUMAN_INTERACTION_MANIFEST, // biome-ignore lint/suspicious/useAwait: localized test assertion preserves its explicit contract. onInteraction: async (message: unknown) => { const msg = message as InteractionMessage; @@ -8690,7 +8704,7 @@ rl.on('line', (line) => { }); await t.test( - "runtime rejects INTERACTION when START.bindings omit interactive and records no interaction artifacts", + "runtime rejects INTERACTION whose kind the manifest does not declare and records no interaction artifacts", async () => { const server = await startTestServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const { asPort, rsPort } = server; @@ -8742,7 +8756,10 @@ rl.on('line', (line) => { rejected = asRuntimeError(err); assert.equal(rejected.failure_reason, "connector_protocol_violation"); // biome-ignore lint/performance/useTopLevelRegex: localized test assertion preserves its explicit contract. - assert.match(rejected.message, /START\.bindings omitted interactive/); + assert.match( + rejected.message, + /INTERACTION\.kind not declared in capabilities\.human_interaction: credentials/ + ); assert.equal(checkpointSummaryOf(rejected).records_flushed, 0); assert.equal(checkpointSummaryOf(rejected).buffered_records_dropped, 0); assert.equal(checkpointSummaryOf(rejected).state_streams_staged, 0); @@ -8762,7 +8779,7 @@ rl.on('line', (line) => { const failedEvent = (runTimeline.data || []).find((event) => event.event_type === "run.failed"); assert.ok( failedEvent, - "run.failed should be emitted when INTERACTION arrives without an advertised interactive binding" + "run.failed should be emitted when INTERACTION names a kind the manifest does not declare" ); assert.equal(failedEvent.data.reason, "connector_protocol_violation"); assert.equal(failedEvent.data.records_flushed, 0); @@ -8912,7 +8929,10 @@ rl.on('line', (line) => { assert.equal(advancedEvents.length, 0); assert.deepEqual( new Set(stagedEvents.map((event) => eventCursorKey(event))), - new Set([cursorEventKey("items", "cursor_items_done_failed"), cursorEventKey("other_items", "cursor_other_done_failed")]) + new Set([ + cursorEventKey("items", "cursor_items_done_failed"), + cursorEventKey("other_items", "cursor_other_done_failed"), + ]) ); const failedEvent = (runTimeline.data || []).find((event) => event.event_type === "run.failed"); @@ -11191,25 +11211,26 @@ function testSourceIdForConnector(connectorId: string): string { } function withTestSourceDeclaration(manifest: ConnectorManifest): ConnectorManifest { - const streams = asArray(manifest.streams).map((stream) => { - const streamRecord = asRecord(stream); - return { - ...streamRecord, - selection: { - fields: true, - resources: true, + const declared = asRecord( + withFixtureSourceDeclaration( + { + ...manifest, + streams: asArray(manifest.streams).map((stream) => ({ + ...asRecord(stream), + selection: { fields: true, resources: true }, + })), }, - }; - }); + { + sourceId: testSourceIdForConnector(manifest.connector_id), + } + ) + ); return { ...manifest, source_declaration: { + ...asRecord(declared.source_declaration), declaration_version: `collection-profile-test:${manifest.connector_id}:${manifest.version}`, - display: { name: manifest.display_name }, - protocol_version: manifest.protocol_version, publisher: { id: "https://sources.example/publishers/collection-profile-tests" }, - source: { id: testSourceIdForConnector(manifest.connector_id), kind: "connector" }, - streams, }, }; } diff --git a/reference-implementation/test/composed-origin.test.ts b/reference-implementation/test/composed-origin.test.ts index 85a925f1a..27a2c4694 100644 --- a/reference-implementation/test/composed-origin.test.ts +++ b/reference-implementation/test/composed-origin.test.ts @@ -506,6 +506,7 @@ const COMPOSED_INGEST_STREAMS = [ name: "items", primary_key: ["id"], schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, semantics: "append_only", }, ]; @@ -639,7 +640,9 @@ test("composed browser origin carries metadata, owner session, console, device f await ensureConsoleBuild(); const webPort = await allocatePort(); const webOrigin = `http://127.0.0.1:${webPort}`; - const spotifyManifest = JSON.parse(await readFile(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8")); + const spotifyManifest = JSON.parse( + await readFile(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") + ); const server = await startServer({ asPort: 0, diff --git a/reference-implementation/test/connector-install-registration-crash.test.ts b/reference-implementation/test/connector-install-registration-crash.test.ts index d2ac80252..065c00035 100644 --- a/reference-implementation/test/connector-install-registration-crash.test.ts +++ b/reference-implementation/test/connector-install-registration-crash.test.ts @@ -26,7 +26,8 @@ */ import assert from "node:assert/strict"; -import { existsSync, mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { createHash } from "node:crypto"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; @@ -463,3 +464,66 @@ test("a retried install after a kill between pointer and registry also restores assert.equal((await agreement()).agree, true); }); }); + +test("killed after the root is published and before the pointer write: a retried install reinstalls the root", async () => { + await withSandbox(async ({ dataDir, dbPath }) => { + const env = { PDPP_DATA_DIR: dataDir }; + await killChildAt(CHILD, "connector-install:after-publish", { + args: [dbPath, dataDir, "install"], + env, + }); + await initDb(dbPath); + assert.equal(existsSync(rootOf(dataDir, GMAIL_INSTALL_DIGEST)), true, "the crash left a published root"); + assert.equal(await activeRecord(), null, "no pointer was written"); + + const installed = await installService(dataDir).install("gmail", GMAIL_INSTALL_DIGEST); + + assert.equal(installed.digest, GMAIL_INSTALL_DIGEST); + assert.equal((await agreement()).agree, true, "the reinstalled root is active and registered"); + }); +}); + +// A root with no active record has no retained verification, so nothing it contains can vouch for +// itself. Hashing its bytes into a fresh record would make the altered bytes the verified ones, so +// the install removes the leftover and takes the bytes from the verified artifact again. +test("a root left by a crash and then altered is not adopted: the retry installs the verified bytes", async () => { + await withSandbox(async ({ dataDir, dbPath }) => { + const env = { PDPP_DATA_DIR: dataDir }; + await killChildAt(CHILD, "connector-install:after-publish", { + args: [dbPath, dataDir, "install"], + env, + }); + await initDb(dbPath); + const root = rootOf(dataDir, GMAIL_INSTALL_DIGEST); + writeFileSync(join(root, "dist", "collection-profile.mjs"), "process.exit(1);\n"); + writeFileSync(join(root, "provenance.json"), '{"altered":true}\n'); + writeFileSync(join(root, "source-declaration.json"), '{"altered":true}'); + writeFileSync(join(root, "planted-extra.mjs"), "export {};\n"); + + const installed = await installService(dataDir).install("gmail", GMAIL_INSTALL_DIGEST); + + assert.equal(readFileSync(join(installed.root, "dist", "collection-profile.mjs"), "utf8"), "export {};\n"); + assert.equal(readFileSync(join(installed.root, "provenance.json"), "utf8"), "{}\n"); + assert.equal(existsSync(join(installed.root, "source-declaration.json")), false, "an altered sidecar is gone"); + assert.equal(existsSync(join(installed.root, "planted-extra.mjs")), false, "nothing the crash left survives"); + assert.equal(installed.sourceDeclarationSha256, undefined, "no sidecar digest comes from observed bytes"); + assert.equal((await agreement()).agree, true); + }); +}); + +test("a root left by a crash with only its executable altered is not adopted", async () => { + await withSandbox(async ({ dataDir, dbPath }) => { + const env = { PDPP_DATA_DIR: dataDir }; + await killChildAt(CHILD, "connector-install:after-publish", { + args: [dbPath, dataDir, "install"], + env, + }); + await initDb(dbPath); + writeFileSync(join(rootOf(dataDir, GMAIL_INSTALL_DIGEST), "dist", "collection-profile.mjs"), "process.exit(1);\n"); + + const installed = await installService(dataDir).install("gmail", GMAIL_INSTALL_DIGEST); + + assert.equal(readFileSync(join(installed.root, "dist", "collection-profile.mjs"), "utf8"), "export {};\n"); + assert.equal(installed.entrypointSha256, `sha256:${createHash("sha256").update("export {};\n").digest("hex")}`); + }); +}); diff --git a/reference-implementation/test/connector-install-registry-integration.test.ts b/reference-implementation/test/connector-install-registry-integration.test.ts new file mode 100644 index 000000000..aeba5604c --- /dev/null +++ b/reference-implementation/test/connector-install-registry-integration.test.ts @@ -0,0 +1,216 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Install service to registry journey: the verified source declaration is published to the real registry + * (SQLite, and PostgreSQL 18 when PDPP_TEST_POSTGRES_URL is set) through the production registration + * wiring, and survives reuse and a restart with a new store and service instance. + * + * The OCI artifact seam is a trusted verified-layout fixture (`installArtifact`); signatures are not exercised. + * The install record's sidecar digest is the durable binding this journey checks. + */ + +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { getConnectorManifest } from "../server/auth.ts"; +import { + type ConnectorCatalogEntry, + createConnectorInstallService, + createFileConnectorInstallStore, + inspectActiveConnector, +} from "../server/connector-install/index.ts"; +import { installedConnectorManifestRegistration } from "../server/connector-install/manifest-registration.ts"; +import { closeDb, initDb } from "../server/db.ts"; +import { closePostgresStorage, initPostgresStorage } from "../server/postgres-storage.ts"; +import { snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest } from "../server/source-declaration-legacy-collection.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; +import { readCollectionProfileFixture } from "./helpers/installed-collection-profiles.ts"; +import { withTemporaryPostgresDatabase } from "./helpers/postgres-temp-database.ts"; + +const SLUG = "fixture-registry-journey"; +const CONFIG_DIGEST = `sha256:${"d".repeat(64)}`; +const OTHER_SOURCE = "https://sources.example/other/registry-journey"; +const ALREADY_BOUND = /already bound to source/; + +function profileFor(version: string): Record { + return { + ...readCollectionProfileFixture("github"), + connector_id: SLUG, + connector_key: SLUG, + manifest_uri: `https://registry.pdpp.dev/connectors/${SLUG}`, + version, + }; +} + +function digestOf(bytes: string): string { + return `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +} + +interface Version { + readonly entry: ConnectorCatalogEntry; + readonly profileBytes: string; + readonly sidecar: string; +} + +/** A version whose sidecar names `sourceId` as its Core source; the profile itself embeds nothing. */ +function versionWith(version: string, sourceId?: string): Version { + const profile = profileFor(version); + const declaration = + sourceId === undefined + ? withFixtureSourceDeclaration(profile).source_declaration + : snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest(profile, { + connectorImplementationId: `https://implementations.example/connectors/${SLUG}`, + publisherId: "https://publishers.example/registry-journey", + sourceId, + }); + const profileBytes = JSON.stringify(profile, null, 2); + return { + entry: { + config_digest: CONFIG_DIGEST, + connector_id: SLUG, + connector_key: SLUG, + digest: digestOf(profileBytes), + tier: "supported", + version, + }, + profileBytes, + sidecar: JSON.stringify(declaration, null, 2), + }; +} + +function writeLayout(root: string, version: Version): void { + mkdirSync(join(root, "profile"), { recursive: true }); + mkdirSync(join(root, "dist"), { recursive: true }); + writeFileSync(join(root, "profile", "collection-profile.json"), version.profileBytes); + writeFileSync(join(root, "dist", "collection-profile.mjs"), "export {};\n"); + writeFileSync(join(root, "provenance.json"), "{}\n"); + writeFileSync(join(root, "source-declaration.json"), version.sidecar); +} + +function retainedSourceId(stored: Record | null): string | undefined { + const declaration = stored?.source_declaration as { source?: { id?: unknown } } | undefined; + return typeof declaration?.source?.id === "string" ? declaration.source.id : undefined; +} + +/** The production registration wiring, optionally failing once on a chosen call to exercise rollback. */ +function productionService(dataDir: string, versions: readonly Version[], failOnCall?: number) { + const store = createFileConnectorInstallStore(dataDir); + let calls = 0; + const service = createConnectorInstallService({ + catalogLoader: async () => versions.map((version) => version.entry), + dataDir, + installArtifact: (root, entry) => { + const version = versions.find((candidate) => candidate.entry.digest === entry.digest); + if (!version) { + throw new Error("no fixture version for the requested digest"); + } + writeLayout(root, version); + }, + registerManifest: async (manifest) => { + calls += 1; + if (failOnCall === calls) { + throw new Error("registration refused (test)"); + } + return await installedConnectorManifestRegistration.registerManifest(manifest); + }, + repairManifestDerivedState: installedConnectorManifestRegistration.repairManifestDerivedState, + store, + }); + return { service, store }; +} + +async function journey(): Promise { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-install-registry-")); + try { + const v1 = versionWith("1.0.0"); + const v2 = versionWith("1.1.0"); + const first = productionService(dataDir, [v1, v2]); + const installed = await first.service.install(SLUG, v1.entry.digest); + const sourceAfterInstall = retainedSourceId((await getConnectorManifest(SLUG)) as Record | null); + assert.ok(sourceAfterInstall, "the registry retains a source for the installed connector"); + assert.equal( + installed.manifest.source_declaration !== undefined, + true, + "the install record carries the verified declaration" + ); + + // Same-digest reuse: the registry keeps the source and the record keeps its sidecar digest. + const reused = await first.service.install(SLUG, v1.entry.digest); + assert.equal(reused.sourceDeclarationSha256, installed.sourceDeclarationSha256); + assert.equal( + retainedSourceId((await getConnectorManifest(SLUG)) as Record | null), + sourceAfterInstall + ); + + // Restart: a new store and a new service read the same record and republish the same source. + const restarted = productionService(dataDir, [v1, v2]); + const inspected = await inspectActiveConnector(restarted.store, SLUG); + assert.equal(inspected.status, "active"); + await restarted.service.install(SLUG, v1.entry.digest); + assert.equal( + retainedSourceId((await getConnectorManifest(SLUG)) as Record | null), + sourceAfterInstall + ); + + // A new version whose declaration rebinds the retained source is refused by the registry, and the + // previous install stays active with its own source. + const rebinding = versionWith("1.2.0", OTHER_SOURCE); + const third = productionService(dataDir, [v1, rebinding]); + await assert.rejects(() => third.service.install(SLUG, rebinding.entry.digest), ALREADY_BOUND); + const active = await third.store.getActive(SLUG); + assert.equal(active?.digest, v1.entry.digest, "the previous install stays active"); + assert.equal( + retainedSourceId((await getConnectorManifest(SLUG)) as Record | null), + sourceAfterInstall + ); + + // Tampered bytes while stopped: refused on read, and the registry keeps the verified source. + writeFileSync(join(installed.root, "source-declaration.json"), '{"source":"tampered"}\n'); + const tampered = await inspectActiveConnector(createFileConnectorInstallStore(dataDir), SLUG); + assert.equal(tampered.status, "invalid"); + assert.equal( + retainedSourceId((await getConnectorManifest(SLUG)) as Record | null), + sourceAfterInstall + ); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +} + +test("SQLite: the verified declaration reaches the registry and survives reuse, restart and tamper", async () => { + initDb(":memory:"); + try { + await journey(); + } finally { + closeDb(); + } +}); + +test("PostgreSQL: the verified declaration reaches the registry and survives reuse, restart and tamper (skipped: PDPP_TEST_POSTGRES_URL unset)", { + skip: !process.env.PDPP_TEST_POSTGRES_URL, +}, async () => { + const url = process.env.PDPP_TEST_POSTGRES_URL; + assert.ok(url); + await withTemporaryPostgresDatabase( + { + closeConnections: closePostgresStorage, + connectionString: url, + databaseName: `pdpp_test_install_registry_${Date.now().toString(36)}_${Math.random().toString(16).slice(2, 10)}_1`, + templateName: null, + }, + async (databaseUrl) => { + initDb(":memory:"); + try { + await initPostgresStorage({ backend: "postgres", databaseUrl }); + await journey(); + } finally { + await closePostgresStorage(); + closeDb(); + } + } + ); +}); diff --git a/reference-implementation/test/connector-install-source-declaration.test.ts b/reference-implementation/test/connector-install-source-declaration.test.ts new file mode 100644 index 000000000..b3139d189 --- /dev/null +++ b/reference-implementation/test/connector-install-source-declaration.test.ts @@ -0,0 +1,577 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * The verified Core source declaration reaches registration through the install record. + * + * `normalizeCoreInstallLayout` moves the verified `source-declaration.json` into the install root. The + * install record must bind that file's digest, and registration must read the declaration from the + * verified record, not from the profile alone. These tests drive the real install service with a + * trusted verified-layout artifact through its installation seam (`installArtifact`, which stands in for + * the pinned OCI verifier); they do not exercise signatures. Registration here is a collector; the real + * registry and activation journey is covered by connector-install-registry-integration.test.ts. + */ + +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import fs, { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { syncBuiltinESMExports } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { + type ConnectorCatalogEntry, + type ConnectorInstallStore, + createConnectorInstallService, + createFileConnectorInstallStore, + inspectActiveConnector, +} from "../server/connector-install/index.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; +import { readCollectionProfileFixture } from "./helpers/installed-collection-profiles.ts"; + +const CONFIG_DIGEST = `sha256:${"c".repeat(64)}`; +const SLUG = "fixture-oci-custom"; +const INTEGRITY_REFUSAL = /source declaration bytes failed integrity verification/; +const SIDECAR_REFUSAL = /source declaration|Invalid SourceDeclaration/i; +const CONFLICT_REFUSAL = /conflicts with the verified source declaration/; +const SIDECAR_INVALID = /Verified source declaration is invalid/; +const PROFILE_TOO_LARGE = /Connector profile exceeds the 1 MiB RI limit/; +const REGISTRATION_REFUSED = /registration refused/; +const PINNED_SIDECAR_MISSING = /Verified source declaration is missing/; +const PINNED_SIDECAR_MISMATCH = /does not match installer-core verification/; +const MIB = 1024 * 1024; + +interface Layout { + readonly entry: ConnectorCatalogEntry; + readonly install?: (root: string) => void; + /** + * Set for a pinned install: the source-declaration bytes the pinned installer core verified. The installer + * reports their digest, and the sidecar on disk (`sidecar`) may then differ or be absent. + */ + readonly pinnedDeclaration?: string; + readonly profileBytes: string; + readonly sidecar: string | null; +} + +/** A signed-shape profile (streams and capabilities from the github fixture), re-keyed to a custom slug. */ +function profileFixture(version = "1.0.0"): Record { + return { + ...readCollectionProfileFixture("github"), + connector_id: SLUG, + connector_key: SLUG, + manifest_uri: `https://registry.pdpp.dev/connectors/${SLUG}`, + version, + }; +} + +function declarationOf(manifest: Record): Record { + const declared = withFixtureSourceDeclaration(manifest) as Record; + return declared.source_declaration as Record; +} + +function entryFor(bytes: string): ConnectorCatalogEntry { + return { + config_digest: CONFIG_DIGEST, + connector_id: SLUG, + connector_key: SLUG, + digest: `sha256:${createHash("sha256").update(bytes).digest("hex")}`, + tier: "supported", + version: "1.0.0", + }; +} + +/** A layout whose catalog entry digests the profile bytes it installs, so the two can never disagree. */ +function layoutOf( + profile: Record, + sidecar: string | null = JSON.stringify(declarationOf(profile), null, 2) +): Layout { + const profileBytes = JSON.stringify(profile, null, 2); + return { entry: entryFor(profileBytes), profileBytes, sidecar }; +} + +/** Writes the layout that `normalizeCoreInstallLayout` produces: profile, dist, provenance, sidecar. */ +function writeVerifiedLayout(root: string, profileBytes: string, sidecarBytes: string | null): void { + mkdirSync(join(root, "profile"), { recursive: true }); + mkdirSync(join(root, "dist"), { recursive: true }); + writeFileSync(join(root, "profile", "collection-profile.json"), profileBytes); + writeFileSync(join(root, "dist", "collection-profile.mjs"), "export {};\n"); + writeFileSync(join(root, "provenance.json"), "{}\n"); + if (sidecarBytes !== null) { + writeFileSync(join(root, "source-declaration.json"), sidecarBytes); + } +} + +/** The real install service over a durable store, with a collector in place of the registry. */ +function serviceOver( + dataDir: string, + store: ConnectorInstallStore, + layouts: readonly Layout[], + options: { readonly failRegistrationOn?: number } = {} +) { + const registered: Record[] = []; + const service = createConnectorInstallService({ + catalogLoader: async () => layouts.map((layout) => layout.entry), + dataDir, + installArtifact: (root, entry) => { + const layout = layouts.find((candidate) => candidate.entry.digest === entry.digest); + if (!layout) { + throw new Error("no fixture layout for the requested digest"); + } + if (layout.install) { + layout.install(root); + } else { + writeVerifiedLayout(root, layout.profileBytes, layout.sidecar); + } + return layout.pinnedDeclaration === undefined + ? undefined + : { configDigest: CONFIG_DIGEST, sourceDeclarationSha256: digestOfText(layout.pinnedDeclaration) }; + }, + registerManifest: (manifest) => { + registered.push(manifest); + if (options.failRegistrationOn === registered.length) { + return Promise.reject(new Error("registration refused (test)")); + } + return Promise.resolve(); + }, + store, + }); + return { registered, service }; +} + +function harness(dataDir: string, layout: Layout) { + const store = createFileConnectorInstallStore(dataDir); + const { registered, service } = serviceOver(dataDir, store, [layout]); + return { entry: layout.entry, registered, service, store }; +} + +function sidecarOf(declaration: Record): string { + return JSON.stringify(declaration, null, 2); +} + +function digestOfText(bytes: string): string { + return `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +} + +/** + * Fails only the first lstat of the staged `source-declaration.json` with EIO. Every other filesystem call + * is the real one. The builtin export is swapped through the CJS object and resynced; callers restore it. + */ +function injectFirstSidecarStatEio(): { readonly faults: () => number; readonly restore: () => void } { + const realLstat = fs.lstatSync; + let faults = 0; + fs.lstatSync = ((path: fs.PathLike, options?: fs.StatOptions) => { + if (faults === 0 && String(path).endsWith("/next/source-declaration.json")) { + faults += 1; + throw Object.assign(new Error(`EIO: i/o error, lstat '${String(path)}'`), { + code: "EIO", + errno: -5, + path: String(path), + syscall: "lstat", + }); + } + return realLstat(path, options as never); + }) as typeof fs.lstatSync; + syncBuiltinESMExports(); + return { + faults: () => faults, + restore: () => { + fs.lstatSync = realLstat; + syncBuiltinESMExports(); + }, + }; +} + +test("the verified source declaration reaches registration from the install record", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-")); + try { + const profile = profileFixture(); + const declaration = declarationOf(profile); + const { entry, registered, service, store } = harness(dataDir, layoutOf(profile, sidecarOf(declaration))); + + const record = await service.install(entry.connector_id, entry.digest); + + assert.equal(registered.length, 1); + assert.notEqual(registered[0]?.source_declaration, undefined, "registration received no source declaration"); + assert.deepEqual(registered[0]?.source_declaration, declaration); + assert.deepEqual(record.manifest.source_declaration, declaration); + const inspected = await inspectActiveConnector(store, SLUG); + assert.equal(inspected.status, "active"); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a modified source declaration sidecar is refused on read verification", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-tamper-")); + try { + const profile = profileFixture(); + const { entry, service, store } = harness(dataDir, layoutOf(profile, sidecarOf(declarationOf(profile)))); + const record = await service.install(entry.connector_id, entry.digest); + + writeFileSync(join(record.root, "source-declaration.json"), '{"source":"changed"}\n'); + + const inspected = await inspectActiveConnector(store, SLUG); + assert.ok(inspected.status === "invalid"); + assert.match(inspected.reason, INTEGRITY_REFUSAL); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a missing source declaration sidecar is refused on read verification", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-missing-")); + try { + const profile = profileFixture(); + const { entry, service, store } = harness(dataDir, layoutOf(profile, sidecarOf(declarationOf(profile)))); + const record = await service.install(entry.connector_id, entry.digest); + + rmSync(join(record.root, "source-declaration.json")); + + const inspected = await inspectActiveConnector(store, SLUG); + assert.ok(inspected.status === "invalid"); + assert.match(inspected.reason, INTEGRITY_REFUSAL); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a symlinked source declaration sidecar is refused before publication", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-symlink-")); + const outside = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-outside-")); + try { + const profile = profileFixture(); + const profileBytes = JSON.stringify(profile, null, 2); + const declarationPath = join(outside, "declaration.json"); + writeFileSync(declarationPath, sidecarOf(declarationOf(profile))); + const layout: Layout = { + entry: entryFor(profileBytes), + install: (root) => { + writeVerifiedLayout(root, profileBytes, null); + symlinkSync(declarationPath, join(root, "source-declaration.json")); + }, + profileBytes, + sidecar: null, + }; + const { entry, registered, service, store } = harness(dataDir, layout); + + await assert.rejects(() => service.install(entry.connector_id, entry.digest), SIDECAR_REFUSAL); + assert.equal(registered.length, 0); + assert.equal((await store.getActive(SLUG)) ?? null, null); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + rmSync(outside, { force: true, recursive: true }); + } +}); + +test("a profile that embeds a conflicting source declaration is refused before publication", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-conflict-")); + try { + // The catalog digests the embedded profile that is actually installed (version 2.0.0), so the refusal + // is the intended conflict between the embedded declaration and the sidecar, not a digest mismatch. + const sidecarDeclaration = declarationOf(profileFixture()); + const embedded = withFixtureSourceDeclaration({ ...profileFixture("2.0.0"), version: "2.0.0" }); + const layout = layoutOf(embedded, sidecarOf(sidecarDeclaration)); + const { entry, registered, service, store } = harness(dataDir, layout); + + await assert.rejects(() => service.install(entry.connector_id, entry.digest), CONFLICT_REFUSAL); + assert.equal(registered.length, 0); + assert.equal((await store.getActive(SLUG)) ?? null, null); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("same-digest reuse re-registers the verified declaration and keeps the recorded sidecar digest", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-reuse-")); + try { + const profile = profileFixture(); + const declaration = declarationOf(profile); + const { entry, registered, service } = harness(dataDir, layoutOf(profile, sidecarOf(declaration))); + const first = await service.install(entry.connector_id, entry.digest); + assert.equal(typeof first.sourceDeclarationSha256, "string", "the install record binds the sidecar digest"); + + const second = await service.install(entry.connector_id, entry.digest); + + assert.equal(registered.length, 2, "reuse re-registers the verified manifest"); + assert.deepEqual(registered[1]?.source_declaration, declaration); + assert.equal(second.sourceDeclarationSha256, first.sourceDeclarationSha256); + assert.equal(second.manifestSha256, first.manifestSha256, "the raw profile bytes and hash are unchanged"); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("restart: a new store instance and a new service read the same declaration from the record", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-restart-")); + try { + const profile = profileFixture(); + const declaration = declarationOf(profile); + const layout = layoutOf(profile, sidecarOf(declaration)); + const first = harness(dataDir, layout); + const installed = await first.service.install(SLUG, layout.entry.digest); + + const restartedStore = createFileConnectorInstallStore(dataDir); + const inspected = await inspectActiveConnector(restartedStore, SLUG); + assert.equal(inspected.status, "active"); + assert.equal(inspected.record.sourceDeclarationSha256, installed.sourceDeclarationSha256); + assert.deepEqual(inspected.record.manifest.source_declaration, declaration); + + const restarted = serviceOver(dataDir, restartedStore, [layout]); + const reinstalled = await restarted.service.install(SLUG, layout.entry.digest); + assert.deepEqual(restarted.registered[0]?.source_declaration, declaration); + assert.equal(reinstalled.sourceDeclarationSha256, installed.sourceDeclarationSha256); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("restart: a sidecar tampered while stopped is refused and no tampered bytes are registered", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-restart-tamper-")); + try { + const profile = profileFixture(); + const declaration = declarationOf(profile); + const layout = layoutOf(profile, sidecarOf(declaration)); + const first = harness(dataDir, layout); + const installed = await first.service.install(SLUG, layout.entry.digest); + writeFileSync(join(installed.root, "source-declaration.json"), '{"source":"tampered"}\n'); + + const restartedStore = createFileConnectorInstallStore(dataDir); + const inspected = await inspectActiveConnector(restartedStore, SLUG); + assert.equal(inspected.status, "invalid"); + assert.match(inspected.reason, INTEGRITY_REFUSAL); + + const restarted = serviceOver(dataDir, restartedStore, [layout]); + await restarted.service.install(SLUG, layout.entry.digest); + assert.ok(restarted.registered.length >= 1); + for (const manifest of restarted.registered) { + assert.deepEqual(manifest.source_declaration, declaration, "only the verified declaration is registered"); + } + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("restart: a missing sidecar is refused by a new store instance", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-restart-missing-")); + try { + const profile = profileFixture(); + const layout = layoutOf(profile, sidecarOf(declarationOf(profile))); + const installed = await harness(dataDir, layout).service.install(SLUG, layout.entry.digest); + rmSync(join(installed.root, "source-declaration.json")); + + const inspected = await inspectActiveConnector(createFileConnectorInstallStore(dataDir), SLUG); + assert.equal(inspected.status, "invalid"); + assert.match(inspected.reason, INTEGRITY_REFUSAL); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a legacy record without the sidecar digest does not trust a newly supplied sidecar", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-legacy-")); + try { + const profile = profileFixture(); + const layout = layoutOf(profile, null); + const first = harness(dataDir, layout); + const installed = await first.service.install(SLUG, layout.entry.digest); + assert.equal(installed.sourceDeclarationSha256, undefined); + + // Simulate a record written before the sidecar digest existed, then supply a sidecar afterwards. + const statePath = join(dataDir, "connector-install-state.json"); + const state = JSON.parse(readFileSync(statePath, "utf8")) as Record>; + const record = state[SLUG]; + assert.ok(record, "the install state holds the installed connector"); + Reflect.deleteProperty(record, "sourceDeclarationSha256"); + writeFileSync(statePath, `${JSON.stringify(state, null, 2)}\n`); + writeFileSync(join(installed.root, "source-declaration.json"), sidecarOf(declarationOf(profileFixture()))); + + const inspected = await inspectActiveConnector(createFileConnectorInstallStore(dataDir), SLUG); + assert.equal(inspected.status, "active"); + assert.equal(inspected.record.manifest.source_declaration, undefined, "the unrecorded sidecar is not trusted"); + assert.equal(inspected.record.manifestSha256, installed.manifestSha256); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("an oversize sidecar is refused before publication and nothing is registered", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-oversize-")); + try { + const profile = profileFixture(); + const padded = `${sidecarOf(declarationOf(profile))}${" ".repeat(MIB)}`; + const { entry, registered, service, store } = harness(dataDir, layoutOf(profile, padded)); + + await assert.rejects(() => service.install(entry.connector_id, entry.digest), SIDECAR_INVALID); + assert.equal(registered.length, 0); + assert.equal((await store.getActive(SLUG)) ?? null, null); + assert.equal(existsSync(join(dataDir, "connectors", SLUG, entry.digest)), false, "the staged root is removed"); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("an oversize profile is refused before publication and nothing is registered", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-profile-oversize-")); + try { + const profile = { ...profileFixture(), padding: "x".repeat(MIB) }; + const { entry, registered, service, store } = harness( + dataDir, + layoutOf(profile, sidecarOf(declarationOf(profile))) + ); + + await assert.rejects(() => service.install(entry.connector_id, entry.digest), PROFILE_TOO_LARGE); + assert.equal(registered.length, 0); + assert.equal((await store.getActive(SLUG)) ?? null, null); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a sidecar that is valid JSON but not a valid Core declaration is refused and nothing is registered", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-shape-")); + try { + const profile = profileFixture(); + const invalid = { ...declarationOf(profile), protocol_version: "9.9.9" }; + const { entry, registered, service, store } = harness(dataDir, layoutOf(profile, sidecarOf(invalid))); + + await assert.rejects(() => service.install(entry.connector_id, entry.digest), SIDECAR_REFUSAL); + assert.equal(registered.length, 0); + assert.equal((await store.getActive(SLUG)) ?? null, null); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a failed registration of a new version rolls back to the previous good install", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-rollback-")); + try { + const v1 = layoutOf(profileFixture("1.0.0")); + const v2 = layoutOf(profileFixture("1.1.0")); + const store = createFileConnectorInstallStore(dataDir); + const good = serviceOver(dataDir, store, [v1]); + const previous = await good.service.install(SLUG, v1.entry.digest); + + const failing = serviceOver(dataDir, store, [v1, v2], { failRegistrationOn: 1 }); + await assert.rejects(() => failing.service.install(SLUG, v2.entry.digest), REGISTRATION_REFUSED); + + const active = await store.getActive(SLUG); + assert.equal(active?.digest, previous.digest, "the previous good install stays active"); + assert.equal(active?.sourceDeclarationSha256, previous.sourceDeclarationSha256); + assert.equal(existsSync(join(dataDir, "connectors", SLUG, v2.entry.digest)), false, "the failed root is removed"); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("an unreadable sidecar fails the install instead of being recorded as absent", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-eio-")); + try { + const v1 = layoutOf(profileFixture("1.0.0")); + const v2 = layoutOf(profileFixture("1.1.0")); + const store = createFileConnectorInstallStore(dataDir); + const previous = await serviceOver(dataDir, store, [v1]).service.install(SLUG, v1.entry.digest); + + const failing = serviceOver(dataDir, store, [v1, v2]); + const fault = injectFirstSidecarStatEio(); + try { + await assert.rejects(() => failing.service.install(SLUG, v2.entry.digest), { code: "EIO" }); + } finally { + fault.restore(); + } + + assert.equal(fault.faults(), 1, "the fault fired on the staged sidecar"); + assert.equal(failing.registered.length, 0, "nothing is registered from the unreadable install"); + const active = await store.getActive(SLUG); + assert.equal(active?.digest, previous.digest, "the previous good install stays active"); + assert.equal(active?.sourceDeclarationSha256, previous.sourceDeclarationSha256); + assert.equal(existsSync(join(dataDir, "connectors", SLUG, v2.entry.digest)), false, "the staged root is removed"); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a pinned install records the Core-verified sidecar digest and registers the verified declaration", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-pinned-")); + try { + const profile = profileFixture(); + const sidecar = sidecarOf(declarationOf(profile)); + const layout: Layout = { ...layoutOf(profile, sidecar), pinnedDeclaration: sidecar }; + const { entry, registered, service } = harness(dataDir, layout); + + const record = await service.install(SLUG, entry.digest); + + assert.equal(record.sourceDeclarationSha256, digestOfText(sidecar)); + assert.deepEqual(registered[0]?.source_declaration, declarationOf(profile)); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a pinned install whose verified sidecar is missing after normalization is refused, not recorded as legacy", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-pinned-missing-")); + try { + const profile = profileFixture(); + const verified = sidecarOf(declarationOf(profile)); + const layout: Layout = { ...layoutOf(profile, null), pinnedDeclaration: verified }; + const { entry, registered, service, store } = harness(dataDir, layout); + + await assert.rejects(() => service.install(SLUG, entry.digest), PINNED_SIDECAR_MISSING); + assert.equal(registered.length, 0); + assert.equal((await store.getActive(SLUG)) ?? null, null); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a pinned install whose sidecar bytes differ from the verified digest is refused", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-pinned-mismatch-")); + try { + const profile = profileFixture(); + const verified = sidecarOf(declarationOf(profile)); + const layout: Layout = { ...layoutOf(profile, '{"source":"other"}\n'), pinnedDeclaration: verified }; + const { entry, registered, service, store } = harness(dataDir, layout); + + await assert.rejects(() => service.install(SLUG, entry.digest), PINNED_SIDECAR_MISMATCH); + assert.equal(registered.length, 0); + assert.equal((await store.getActive(SLUG)) ?? null, null); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a transient EIO on a pinned install's sidecar refuses the install and registers nothing", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-pinned-eio-")); + try { + const profile = profileFixture(); + const sidecar = sidecarOf(declarationOf(profile)); + const layout: Layout = { ...layoutOf(profile, sidecar), pinnedDeclaration: sidecar }; + const { entry, registered, service, store } = harness(dataDir, layout); + const fault = injectFirstSidecarStatEio(); + try { + await assert.rejects(() => service.install(SLUG, entry.digest), { code: "EIO" }); + } finally { + fault.restore(); + } + + assert.equal(fault.faults(), 1); + assert.equal(registered.length, 0); + assert.equal((await store.getActive(SLUG)) ?? null, null); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + +test("a legacy install with no sidecar stays legacy and records no sidecar digest", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-oci-declaration-legacy-absent-")); + try { + const profile = profileFixture(); + const { entry, registered, service } = harness(dataDir, layoutOf(profile, null)); + + const record = await service.install(SLUG, entry.digest); + + assert.equal(record.sourceDeclarationSha256, undefined); + assert.equal(registered[0]?.source_declaration, undefined); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); diff --git a/reference-implementation/test/connector-install-sql-persistence.test.ts b/reference-implementation/test/connector-install-sql-persistence.test.ts new file mode 100644 index 000000000..df7ddccda --- /dev/null +++ b/reference-implementation/test/connector-install-sql-persistence.test.ts @@ -0,0 +1,284 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * The digest-bearing install record through the production SQL store. + * + * `createConnectorInstallStore()` (no preload directory) persists `connector_installs.record_json`. This journey + * installs a pinned fixture through the production registration wiring, reads the record back through fresh store + * instances, and reads the activation through the lazy path (`getConnectorActivation`, which imports the legacy + * install row into `connector_activations`). It runs on SQLite always and on PostgreSQL 18 when + * PDPP_TEST_POSTGRES_URL is set. The OCI artifact is a trusted fixture that reports a verified source declaration + * digest; signatures are not exercised. Compare connector-install-registry-integration.test.ts, which uses the + * file install store. + */ + +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; +import { getConnectorManifest } from "../server/auth.ts"; +import { getConnectorActivation } from "../server/connector-install/activation-authority.ts"; +import { + type ConnectorCatalogEntry, + type ConnectorInstallRecord, + createConnectorInstallService, + createConnectorInstallStore, + inspectActiveConnector, +} from "../server/connector-install/index.ts"; +import { installedConnectorManifestRegistration } from "../server/connector-install/manifest-registration.ts"; +import { closeDb, getDb, initDb } from "../server/db.ts"; +import { + closePostgresStorage, + initPostgresStorage, + isPostgresStorageBackend, + postgresQuery, +} from "../server/postgres-storage.ts"; +import { snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest } from "../server/source-declaration-legacy-collection.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; +import { readCollectionProfileFixture } from "./helpers/installed-collection-profiles.ts"; +import { withTemporaryPostgresDatabase } from "./helpers/postgres-temp-database.ts"; + +const SLUG = "fixture-sql-persistence"; +const CONFIG_DIGEST = `sha256:${"e".repeat(64)}`; +const OTHER_SOURCE = "https://sources.example/other/sql-persistence"; +const INTEGRITY_REFUSAL = /source declaration bytes failed integrity verification/; +const CONFLICT_REFUSAL = /conflicts with the verified source declaration/; + +function digestOf(bytes: string): string { + return `sha256:${createHash("sha256").update(bytes).digest("hex")}`; +} + +function profileFor(version: string): Record { + return { + ...readCollectionProfileFixture("github"), + connector_id: SLUG, + connector_key: SLUG, + manifest_uri: `https://registry.pdpp.dev/connectors/${SLUG}`, + version, + }; +} + +function declarationOf(profile: Record): Record { + return withFixtureSourceDeclaration(profile).source_declaration as Record; +} + +interface Version { + readonly entry: ConnectorCatalogEntry; + readonly profileBytes: string; + /** The verified sidecar bytes the pinned installer core reports, and the install writes. */ + readonly sidecar: string; +} + +/** + * A version. `embedded` is written into the profile itself; it is absent unless a test needs a profile that + * carries a declaration (equal, or conflicting with the sidecar). + */ +function versionOf(version: string, embedded?: unknown): Version { + const base = profileFor(version); + const profile = embedded === undefined ? base : { ...base, source_declaration: embedded }; + const profileBytes = JSON.stringify(profile, null, 2); + return { + entry: { + config_digest: CONFIG_DIGEST, + connector_id: SLUG, + connector_key: SLUG, + digest: digestOf(profileBytes), + tier: "supported", + version, + }, + profileBytes, + sidecar: JSON.stringify(declarationOf(base), null, 2), + }; +} + +function writeLayout(root: string, version: Version): void { + mkdirSync(join(root, "profile"), { recursive: true }); + mkdirSync(join(root, "dist"), { recursive: true }); + writeFileSync(join(root, "profile", "collection-profile.json"), version.profileBytes); + writeFileSync(join(root, "dist", "collection-profile.mjs"), "export {};\n"); + writeFileSync(join(root, "provenance.json"), "{}\n"); + writeFileSync(join(root, "source-declaration.json"), version.sidecar); +} + +/** The production store and registration wiring over the owned data directory. */ +function serviceOver(dataDir: string, versions: readonly Version[]) { + const registered: Record[] = []; + const service = createConnectorInstallService({ + catalogLoader: async () => versions.map((version) => version.entry), + dataDir, + installArtifact: (root, entry) => { + const version = versions.find((candidate) => candidate.entry.digest === entry.digest); + if (!version) { + throw new Error("no fixture version for the requested digest"); + } + writeLayout(root, version); + return { + configDigest: CONFIG_DIGEST, + sourceDeclarationSha256: digestOf(version.sidecar), + }; + }, + registerManifest: async (manifest) => { + registered.push(manifest); + return await installedConnectorManifestRegistration.registerManifest(manifest); + }, + repairManifestDerivedState: installedConnectorManifestRegistration.repairManifestDerivedState, + store: createConnectorInstallStore(), + }); + return { registered, service }; +} + +async function storedRecord(): Promise { + const json = isPostgresStorageBackend() + ? ( + await postgresQuery<{ record_json: string }>( + "SELECT record_json FROM connector_installs WHERE connector_id=$1", + [SLUG] + ) + ).rows[0]?.record_json + : getDb() + .prepare("SELECT record_json FROM connector_installs WHERE connector_id=?") + .get<{ record_json: string }>(SLUG)?.record_json; + return json === undefined ? null : (JSON.parse(json) as ConnectorInstallRecord); +} + +async function retainedSourceId(): Promise { + const declaration = ( + (await getConnectorManifest(SLUG)) as { + source_declaration?: { source?: { id?: unknown } }; + } | null + )?.source_declaration; + return declaration?.source?.id; +} + +/** Runs `run` with PDPP_DATA_DIR owned by the test and no preload directory, restoring both afterwards. */ +async function withOwnedDataDir(run: (dataDir: string) => Promise): Promise { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-install-sql-")); + const previousDataDir = process.env.PDPP_DATA_DIR; + const previousPreload = process.env.PDPP_CONNECTOR_PRELOAD_DIR; + process.env.PDPP_DATA_DIR = dataDir; + delete process.env.PDPP_CONNECTOR_PRELOAD_DIR; + try { + await run(dataDir); + } finally { + if (previousDataDir === undefined) { + delete process.env.PDPP_DATA_DIR; + } else { + process.env.PDPP_DATA_DIR = previousDataDir; + } + if (previousPreload === undefined) { + delete process.env.PDPP_CONNECTOR_PRELOAD_DIR; + } else { + process.env.PDPP_CONNECTOR_PRELOAD_DIR = previousPreload; + } + rmSync(dataDir, { force: true, recursive: true }); + } +} + +async function journey(dataDir: string): Promise { + const v1 = versionOf("1.0.0"); + const declaration = declarationOf(profileFor("1.0.0")); + const first = serviceOver(dataDir, [v1]); + + // Install through the production store; the record carries the Core-verified sidecar digest. + const installed = await first.service.install(SLUG, v1.entry.digest); + assert.equal(installed.sourceDeclarationSha256, digestOf(v1.sidecar)); + + // A fresh read from SQL: the digest, the raw profile hash and the verified declaration survive the round trip. + const persisted = await storedRecord(); + assert.ok(persisted, "the SQL install store holds the record"); + assert.equal(persisted.sourceDeclarationSha256, digestOf(v1.sidecar), "the stored record keeps the sidecar digest"); + assert.equal(persisted.manifestSha256, digestOf(v1.profileBytes), "the raw profile hash is unchanged"); + assert.deepEqual(persisted.manifest.source_declaration, declaration); + const inspected = await inspectActiveConnector(createConnectorInstallStore(), SLUG); + assert.equal(inspected.status, "active"); + assert.equal(inspected.record.sourceDeclarationSha256, digestOf(v1.sidecar)); + + // Lazy activation: the first read imports the install row into connector_activations. + const activation = await getConnectorActivation(SLUG); + assert.ok(activation, "the activation is imported on first read"); + assert.equal(activation.state, "active", "the registry and the installed artifact agree"); + assert.equal(activation.record.sourceDeclarationSha256, digestOf(v1.sidecar)); + assert.deepEqual(activation.record.manifest.source_declaration, declaration); + assert.equal((await getConnectorActivation(SLUG))?.activationId, activation.activationId, "the import is stable"); + + // Same-digest reuse keeps the record's digest and the retained registry source. + const sourceAfterInstall = await retainedSourceId(); + assert.ok(sourceAfterInstall, "the registry retains a source"); + const reused = await first.service.install(SLUG, v1.entry.digest); + assert.equal(reused.sourceDeclarationSha256, digestOf(v1.sidecar)); + assert.equal((await storedRecord())?.sourceDeclarationSha256, digestOf(v1.sidecar)); + assert.equal(await retainedSourceId(), sourceAfterInstall); + + // Tampered while stopped: the read is refused, and a repair registers only the verified declaration. + const root = join(dataDir, "connectors", SLUG, v1.entry.digest); + writeFileSync(join(root, "source-declaration.json"), '{"source":"tampered"}\n'); + const tampered = await inspectActiveConnector(createConnectorInstallStore(), SLUG); + assert.equal(tampered.status, "invalid"); + assert.match(tampered.reason, INTEGRITY_REFUSAL); + + const repaired = serviceOver(dataDir, [v1]); + const repairedRecord = await repaired.service.install(SLUG, v1.entry.digest); + assert.equal(repaired.registered.length, 1, "the repair registers once"); + for (const manifest of repaired.registered) { + assert.deepEqual(manifest.source_declaration, declaration, "only the verified declaration is registered"); + } + assert.equal(repairedRecord.sourceDeclarationSha256, digestOf(v1.sidecar)); + assert.equal(await retainedSourceId(), sourceAfterInstall, "the registry keeps the verified source"); + + // A profile whose embedded declaration conflicts with the sidecar is refused and changes nothing. + const conflicting = versionOf( + "1.1.0", + snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest(profileFor("1.1.0"), { + connectorImplementationId: `https://implementations.example/connectors/${SLUG}`, + publisherId: "https://publishers.example/sql-persistence", + sourceId: OTHER_SOURCE, + }) + ); + const rejecting = serviceOver(dataDir, [v1, conflicting]); + await assert.rejects(() => rejecting.service.install(SLUG, conflicting.entry.digest), CONFLICT_REFUSAL); + assert.equal(rejecting.registered.length, 0, "the conflicting profile is never registered"); + assert.equal((await storedRecord())?.digest, v1.entry.digest, "the previous install stays active"); + + // A profile that embeds an equal declaration is accepted and records its own sidecar digest. + const equal = versionOf("1.2.0", declarationOf(profileFor("1.2.0"))); + const accepted = await serviceOver(dataDir, [equal]).service.install(SLUG, equal.entry.digest); + assert.equal(accepted.sourceDeclarationSha256, digestOf(equal.sidecar)); + assert.equal((await storedRecord())?.digest, equal.entry.digest); +} + +test("SQLite: a pinned install's source declaration digest survives the SQL store, lazy activation and repair", async () => { + initDb(":memory:"); + try { + await withOwnedDataDir(journey); + } finally { + closeDb(); + } +}); + +test("PostgreSQL: a pinned install's source declaration digest survives the SQL store, lazy activation and repair (skipped: PDPP_TEST_POSTGRES_URL unset)", { + skip: !process.env.PDPP_TEST_POSTGRES_URL, +}, async () => { + const url = process.env.PDPP_TEST_POSTGRES_URL; + assert.ok(url); + await withTemporaryPostgresDatabase( + { + closeConnections: closePostgresStorage, + connectionString: url, + databaseName: `pdpp_test_install_sql_${Date.now().toString(36)}_${Math.random().toString(16).slice(2, 10)}_1`, + templateName: null, + }, + async (databaseUrl) => { + initDb(":memory:"); + try { + await initPostgresStorage({ backend: "postgres", databaseUrl }); + await withOwnedDataDir(journey); + } finally { + await closePostgresStorage(); + closeDb(); + } + } + ); +}); diff --git a/reference-implementation/test/connector-install.test.ts b/reference-implementation/test/connector-install.test.ts index 97e44e2a6..cf1a387f8 100644 --- a/reference-implementation/test/connector-install.test.ts +++ b/reference-implementation/test/connector-install.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; import { existsSync, mkdirSync, @@ -21,6 +22,7 @@ import { dirname, join, resolve } from "node:path"; import { createRequire } from "node:module"; import { pathToFileURL } from "node:url"; import test from "node:test"; +import { validateConnectorManifest } from "../server/connector-manifest-validation.ts"; import { type ConnectorCatalogEntry, @@ -62,6 +64,7 @@ const RE_SYMBOLIC_LINK = /symbolic-link/; const RE_UPDATE_FAILED = /crashed while staging update/; const RE_UPDATE_TARGET_MISSING = /no verified update target/; const RE_REGISTRATION_FAILED = /manifest registration failed/; +const RE_LOCAL_EXPAND_INVALID = /query\.expand.*must match a same-stream relationships\[\] entry/; const require = createRequire(import.meta.url); async function waitFor(condition: () => boolean, timeoutMs = 5000): Promise { const started = Date.now(); @@ -86,6 +89,43 @@ const entry: ConnectorCatalogEntry = { version: "1.0.0", }; +test("adding a developer source rejects invalid query.expand before publishing local source state", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-local-profile-validation-")); + const sourceRoot = join(dataDir, "candidate"); + try { + const profile = JSON.parse( + readFileSync(new URL("./fixtures/collection-profiles/github.json", import.meta.url), "utf8") + ) as Record; + // The authoritative validator accepts the unmodified profile. This test + // changes only the semantic contract that shallow local checks miss. + validateConnectorManifest(profile); + assert.ok(Array.isArray(profile.streams)); + const firstStream = profile.streams[0] as Record; + firstStream.query = { expand: [{ name: "missing_relationship" }] }; + assert.throws(() => validateConnectorManifest(profile), RE_LOCAL_EXPAND_INVALID); + + mkdirSync(join(sourceRoot, "profile"), { recursive: true }); + mkdirSync(join(sourceRoot, "dist")); + writeFileSync(join(sourceRoot, "profile", "collection-profile.json"), JSON.stringify(profile)); + writeFileSync(join(sourceRoot, "dist", "collection-profile.mjs"), "export {};\n"); + const localSourceStore = createFileLocalConnectorSourceStore(dataDir); + const service = createConnectorInstallService({ + dataDir, + localSourceStore, + registerManifest: async (manifest) => { + validateConnectorManifest(manifest); + }, + store: createFileConnectorInstallStore(dataDir), + }); + const { addLocalSource } = service; + assert.ok(addLocalSource); + await assert.rejects(() => addLocalSource(sourceRoot), RE_LOCAL_EXPAND_INVALID); + assert.deepEqual(await localSourceStore.list(), [], "a rejected profile must not be published as a local source"); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + test("pinned catalog schema accepts binding features and rejects unknown binding fields", () => { const catalog = { catalog_version: "1.0", @@ -148,16 +188,16 @@ test("owner local-source state lives in PDPP_DATA_DIR even when PDPP_CONNECTOR_P try { mkdirSync(join(sourceRoot, "profile")); mkdirSync(join(sourceRoot, "dist")); + const profile = JSON.parse( + readFileSync(new URL("./fixtures/collection-profiles/github.json", import.meta.url), "utf8") + ) as Record; + profile.connector_id = "https://vendor.example/connectors/preload-split-fixture"; + profile.manifest_uri = profile.connector_id; + profile.connector_key = "preload-split-fixture"; + profile.version = "0.1.0"; writeFileSync( join(sourceRoot, "profile", "collection-profile.json"), - JSON.stringify({ - connector_id: "https://registry.pdpp.dev/connectors/preload-split-fixture", - connector_key: "preload-split-fixture", - display_name: "Preload split fixture", - runtime_requirements: { bindings: { network: { required: true } } }, - streams: [{ name: "items" }], - version: "0.1.0", - }) + JSON.stringify(profile) ); writeFileSync(join(sourceRoot, "dist", "collection-profile.mjs"), "export default {};\n"); @@ -583,6 +623,74 @@ test("a second store handle cannot create a source while uninstall owns the data } }); +const DUPLICATE_BINDING_PROFILES = { + escaped: + '{"connector_id":"github","version":"1.0.0","runtime_requirements":{"bindings":{"network":{"required":true},"n\\u0065twork":{"required":false}}}}', + nested: + '{"connector_id":"github","version":"1.0.0","runtime_requirements":{"bindings":{"network":{"required":true,"required":false}}}}', + plain: + '{"connector_id":"github","version":"1.0.0","runtime_requirements":{"bindings":{"network":{"required":true},"network":{"required":false}}}}', +} as const; +const RE_PROFILE_REPEATS_KEY = /Connector profile repeats an object key\./; + +for (const [name, profileText] of Object.entries(DUPLICATE_BINDING_PROFILES)) { + test(`an installed profile with a repeated object key (${name}) is refused, as the HTTP and local paths refuse it`, async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-connector-install-")); + try { + const store = createFileConnectorInstallStore(dataDir); + const registered: unknown[] = []; + const service = createConnectorInstallService({ + catalogLoader: async () => [entry], + dataDir, + installArtifact: (root) => { + writeFixture(root); + writeFileSync(join(root, "profile", "collection-profile.json"), profileText); + }, + registerManifest: (manifest) => { + registered.push(manifest); + return Promise.resolve(); + }, + store, + }); + await assert.rejects(service.install("github", digest), RE_PROFILE_REPEATS_KEY); + assert.equal(await store.getActive("github"), null, "nothing became active"); + assert.deepEqual(registered, [], "nothing was registered, so no last-wins value reached the registry"); + assert.equal(existsSync(join(dataDir, "connectors", "github", digest)), false, "the published root is removed"); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } + }); +} + +test("an active record whose profile bytes repeat an object key is not runnable", async () => { + const dataDir = mkdtempSync(join(tmpdir(), "pdpp-connector-install-")); + try { + const store = createFileConnectorInstallStore(dataDir); + const service = createConnectorInstallService({ + catalogLoader: async () => [entry], + dataDir, + installArtifact: (root) => { + writeFixture(root); + }, + registerManifest: () => Promise.resolve(), + store, + }); + const active = await service.install("github", digest); + // Re-record the altered profile under its own hash, so only the repeated key is wrong. + const profile = join(active.root, "profile", "collection-profile.json"); + writeFileSync(profile, DUPLICATE_BINDING_PROFILES.plain); + const manifestSha256 = `sha256:${createHash("sha256").update(readFileSync(profile)).digest("hex")}`; + await store.activate({ ...active, manifestSha256 }); + + const inspected = await inspectActiveConnector(store, "github"); + assert.equal(inspected.status, "invalid"); + assert.match(inspected.status === "invalid" ? inspected.reason : "", RE_PROFILE_REPEATS_KEY); + assert.equal(await resolveActiveConnectorPath(store, "github"), null); + } finally { + rmSync(dataDir, { force: true, recursive: true }); + } +}); + test("a corrupted active entrypoint fails closed", async () => { const dataDir = mkdtempSync(join(tmpdir(), "pdpp-connector-install-")); try { diff --git a/reference-implementation/test/connector-manifest-validation.test.ts b/reference-implementation/test/connector-manifest-validation.test.ts index 5684de7d6..43195cdd1 100644 --- a/reference-implementation/test/connector-manifest-validation.test.ts +++ b/reference-implementation/test/connector-manifest-validation.test.ts @@ -39,7 +39,9 @@ const TOP_LEVEL_REGEX_27 = /capabilities\.proven\.local_collector=true requires runtime_requirements\.bindings\.filesystem/; const TOP_LEVEL_REGEX_28 = /credential_capture field 'password' is a secret field .* with no label/; const TOP_LEVEL_REGEX_29 = /credential_capture field 'password' is a secret field with zero env aliases/; -const PROFILE_VERSION_PATTERN = /protocol_version must be one of 0\.1\.0, 0\.2\.0, received 0\.3\.0/; +// The unknown-version message is a fixed rule. It never echoes the manifest's own value. +const PROFILE_VERSION_PATTERN = /protocol_version must be one of 0\.1\.0, 0\.2\.0$/; +const PROFILE_VERSION_MESSAGE = "protocol_version must be one of 0.1.0, 0.2.0"; const PROFILE_0_2_0_CAPABILITY_PATTERN = /protocol_capabilities DETAIL_COVERAGE_KEY_SETS require protocol_version 0\.2\.0/; const UNKNOWN_CAPABILITY_PATTERN = /protocol_capabilities must be an array of known capability names/; @@ -227,6 +229,11 @@ test("validateConnectorManifest enforces the Collection Profile declaration at r validateConnectorManifest({ ...manifestWithStream(), ...keySets, protocol_version: "0.2.0" }) ); assert.doesNotThrow(() => validateConnectorManifest({ ...manifestWithStream(), protocol_capabilities: ["BLOB"] })); + assert.throws( + () => validateConnectorManifest({ ...manifestWithStream(), protocol_version: "0.3.0" }), + (error: unknown) => (error as Error).message === PROFILE_VERSION_MESSAGE, + "an unknown version is rejected with the fixed rule, not an echo of the manifest value" + ); const rejected: [Record, RegExp][] = [ [{ protocol_version: "0.3.0" }, PROFILE_VERSION_PATTERN], [{ ...keySets, protocol_version: "0.1.0" }, PROFILE_0_2_0_CAPABILITY_PATTERN], diff --git a/reference-implementation/test/connector-registry-manifest-derivation.test.ts b/reference-implementation/test/connector-registry-manifest-derivation.test.ts index 540fb3f7c..cb309dc4e 100644 --- a/reference-implementation/test/connector-registry-manifest-derivation.test.ts +++ b/reference-implementation/test/connector-registry-manifest-derivation.test.ts @@ -69,6 +69,7 @@ import { readPolyfillManifests } from "@pdpp/polyfill-connectors/manifests"; import { seedableConnectorsFromManifests } from "../cli/commands/seed.ts"; import { SUPPORTED_SEED_CONNECTOR_KEYS } from "../connectors/seed/index.ts"; +import { validateLegacySourceBindings } from "../scripts/legacy-source-binding-validation.ts"; import type { ConnectorManifestLike } from "../server/connection-setup-plan.ts"; import { buildConnectionSetupPlan, @@ -79,6 +80,7 @@ import { canonicalConnectorKey, isConnectorKey } from "../server/connector-key.t import { BROWSER_BOUND_KEYS, FIRST_PARTY_CONNECTOR_KEYS, + LEGACY_FIRST_PARTY_SOURCE_ENTRIES, LEGACY_LOCAL_ALIASES, LOCAL_COLLECTOR_PROVEN_KEYS, NATIVE_CONNECTOR_KEYS, @@ -86,6 +88,12 @@ import { STATIC_SECRET_LIVE_PROVEN_KEYS, } from "../server/generated/connector-registry.generated.ts"; +const GENERATED_BINDINGS_LINE = /^export const LEGACY_FIRST_PARTY_SOURCE_ENTRIES:[^\n]*$/m; +const MAPPING_CHANGED = /Pinned historical source mapping changed/; +const DUPLICATE_BINDINGS = /duplicate keys or source IDs/; +const INVALID_BINDING_KEY = /Invalid pinned connector_key/; +const INVALID_BINDING_SOURCE = /Invalid pinned legacy connector_id/; + const trackedRegistry = { BROWSER_BOUND_KEYS, FIRST_PARTY_CONNECTOR_KEYS, @@ -173,6 +181,98 @@ test("the generator throws (does not silently drop the claim) when a manifest de } }); +test("historical source bindings stay pinned to real package identities, independent of fixture overrides", () => { + const scratchDir = mkdtempSync(join(tmpdir(), "connector-registry-binding-pin-")); + try { + for (const { file, manifest } of readPolyfillManifests()) { + writeFileSync( + join(scratchDir, file), + JSON.stringify({ ...(manifest as Record), manifest_uri: "https://provenance.example/changed" }) + ); + } + const scratchPath = join(scratchDir, "generated.ts"); + execFileSync("node", ["--experimental-strip-types", generatorScript, scratchPath], { + cwd: riRoot, + env: { ...process.env, PDPP_POLYFILL_MANIFESTS_DIR: scratchDir }, + stdio: "pipe", + }); + assert.equal(readFileSync(scratchPath, "utf8"), readFileSync(trackedRegistryPath, "utf8")); + + const firstManifestFile = readdirSync(scratchDir).find((file) => file.endsWith(".json")); + assert.ok(firstManifestFile, "the override fixture contains a manifest"); + const firstManifestPath = join(scratchDir, firstManifestFile); + const changed = JSON.parse(readFileSync(firstManifestPath, "utf8")) as Record; + changed.connector_id = "https://registry.pdpp.dev/connectors/repointed"; + writeFileSync(firstManifestPath, JSON.stringify(changed)); + execFileSync("node", ["--experimental-strip-types", generatorScript, scratchPath], { + cwd: riRoot, + env: { ...process.env, PDPP_POLYFILL_MANIFESTS_DIR: scratchDir }, + stdio: "pipe", + }); + const overridden = readFileSync(scratchPath, "utf8"); + const generatedBindings = (source: string): string => { + const match = source.match(GENERATED_BINDINGS_LINE); + assert.ok(match, "the generator emits historical binding data"); + return match[0]; + }; + assert.equal(generatedBindings(overridden), generatedBindings(readFileSync(trackedRegistryPath, "utf8"))); + + const overrideKey = "fixture-override-probe"; + writeFileSync( + join(scratchDir, `${overrideKey}.json`), + JSON.stringify({ + connector_id: "https://fixture.example/connectors/fixture-override-probe", + connector_key: overrideKey, + manifest_uri: "https://provenance.example/override-probe", + runtime_requirements: { bindings: { browser: { required: true } } }, + }) + ); + execFileSync("node", ["--experimental-strip-types", generatorScript, scratchPath], { + cwd: riRoot, + env: { ...process.env, PDPP_POLYFILL_MANIFESTS_DIR: scratchDir }, + stdio: "pipe", + }); + const withOverrideProbe = readFileSync(scratchPath, "utf8"); + assert.match(withOverrideProbe, new RegExp(`BROWSER_BOUND_KEYS:[^\\n]*${overrideKey}`)); + assert.equal(generatedBindings(withOverrideProbe), generatedBindings(readFileSync(trackedRegistryPath, "utf8"))); + } finally { + rmSync(scratchDir, { force: true, recursive: true }); + } +}); + +test("historical binding validation rejects reassignment, additions, duplicates, and malformed identities", () => { + const approved = LEGACY_FIRST_PARTY_SOURCE_ENTRIES; + const [first] = approved; + const last = approved.at(-1); + assert.ok(first); + assert.ok(last); + const replaceFirst = (entry: readonly [string, string]): readonly (readonly [string, string])[] => [ + entry, + ...approved.slice(1), + ]; + assert.throws( + () => validateLegacySourceBindings(replaceFirst(["amazon", "https://other.example/source"])), + MAPPING_CHANGED + ); + assert.throws( + () => validateLegacySourceBindings([...approved, ["new-connector", "https://other.example/source"] as const]), + MAPPING_CHANGED + ); + assert.throws(() => validateLegacySourceBindings([...approved, first]), DUPLICATE_BINDINGS); + assert.throws( + () => validateLegacySourceBindings([...approved, ["other-key", first[1]] as const]), + DUPLICATE_BINDINGS + ); + assert.throws( + () => validateLegacySourceBindings([...approved.slice(0, -1), ["bad/key", last[1]] as const]), + INVALID_BINDING_KEY + ); + assert.throws( + () => validateLegacySourceBindings([...approved.slice(0, -1), [last[0], "https://"] as const]), + INVALID_BINDING_SOURCE + ); +}); + test("every generated registry entry is a real, non-empty connector key with no duplicates", () => { for (const [name, value] of Object.entries(trackedRegistry)) { if (Array.isArray(value)) { diff --git a/reference-implementation/test/connector-source-identity-boundary.test.ts b/reference-implementation/test/connector-source-identity-boundary.test.ts new file mode 100644 index 000000000..371777fc4 --- /dev/null +++ b/reference-implementation/test/connector-source-identity-boundary.test.ts @@ -0,0 +1,1079 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Connector source identity at the registration boundary. + * + * A connector's Core source identity is the `source.id` of the SourceDeclaration + * the runtime retains for its connector key. Resolution order, on both write + * paths (`registerConnector` and the installer's `publishConnectorActivation`): + * + * 1. a retained declaration keeps its `source.id`; provenance cannot change it; + * 2. otherwise an explicit declaration is validated and used; + * 3. otherwise a first-party key takes the reviewed legacy source identity from + * the runtime-owned map (connector-legacy-source-map.ts), independent of the + * incoming provenance (`manifest_uri`) and alias (`connector_id`); + * 4. otherwise the registration is refused before any write. + * + * Covers both write paths. + */ + +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import test from "node:test"; +import { executeAsPolyfillConnectorRegister } from "../operations/as-polyfill-connector-register/index.ts"; +import { + __setRegisterConnectorPhaseHookForTest, + connectorDeclarationDigest, + getAcceptedDeclarationDigest, + getConnectorManifest, + normalizeConnectorManifestForStorage, + registerConnector, +} from "../server/auth.ts"; +import { + getConnectorActivation, + publishConnectorActivation, +} from "../server/connector-install/activation-authority.ts"; +import type { ConnectorInstallRecord } from "../server/connector-install/index.ts"; +import { + LEGACY_FIRST_PARTY_ENTRIES, + LEGACY_FIRST_PARTY_KEYS, + legacyFirstPartySourceIdFor, +} from "../server/connector-legacy-source-map.ts"; +import { validateConnectorManifest } from "../server/connector-manifest-validation.ts"; +import { closeDb, initDb } from "../server/db.ts"; +import { readPolyfillManifests } from "../server/polyfill-connectors-runtime.ts"; +import { closePostgresStorage, initPostgresStorage } from "../server/postgres-storage.ts"; +import { snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest } from "../server/source-declaration-legacy-collection.ts"; +import { withTemporaryPostgresDatabase } from "./helpers/postgres-temp-database.ts"; + +const PUBLISHER = "https://publishers.example/accepted"; +const DOC_ONE = "https://docs.example/manifests/one.json"; +const DOC_TWO = "https://docs.example/manifests/two.json"; +const EXPLICIT_SOURCE = "https://sources.example/explicit/identity-boundary"; +const OTHER_SOURCE = "https://sources.example/other/identity-boundary"; + +const INVALID_REQUEST = "invalid_request"; +const ALREADY_BOUND = /already bound to source/; +const CANONICAL_KEY_MESSAGE = /canonical key/; +const INVALID_SOURCE_DECLARATION = /Invalid SourceDeclaration/; +const KEY_MATCH_MESSAGE = /connector_id must match connector_key/; +const NO_SOURCE_MESSAGE = /no source declaration/; +const URL_KEY_MESSAGE = /is a URL with no connector_key/; + +/** The reviewed historical source identity of a shipped first-party connector. */ +function firstPartySource(slug: string): string { + return `https://registry.pdpp.dev/connectors/${slug}`; +} + +function streamFor(name: string) { + return { + name, + primary_key: ["id"], + schema: { + properties: { id: { type: "string" } }, + required: ["id"], + type: "object", + }, + selection: { fields: true, resources: true }, + semantics: "append_only", + }; +} + +function legacyManifest(overrides: Record): Record { + return { + capabilities: { human_interaction: [] }, + display_name: "Identity boundary", + protocol_version: "0.1.0", + streams: [streamFor("items")], + version: "1.0.0", + ...overrides, + }; +} + +/** A fully valid Core declaration, produced by the existing legacy projection. */ +function explicitDeclaration(connectorKey: string, sourceId: string, streams = [streamFor("items")]) { + return snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest( + legacyManifest({ + connector_id: connectorKey, + connector_key: connectorKey, + streams, + }), + { + connectorImplementationId: `https://implementations.example/connectors/${connectorKey}`, + publisherId: PUBLISHER, + sourceId, + } + ); +} + +function retainedSourceId(stored: Record | null): string | undefined { + const declaration = stored?.source_declaration as { source?: { id?: unknown } } | undefined; + return typeof declaration?.source?.id === "string" ? declaration.source.id : undefined; +} + +async function storedFor(connectorKey: string): Promise | null> { + return ((await getConnectorManifest(connectorKey)) as Record | null) ?? null; +} + +async function assertRejectedWithoutWrite( + connectorKey: string, + attempt: () => Promise, + messagePattern: RegExp +): Promise { + await assert.rejects(attempt, (error: unknown) => { + const err = error as { code?: unknown; message?: unknown }; + assert.equal(err.code, INVALID_REQUEST); + assert.match(String(err.message), messagePattern); + return true; + }); + assert.equal(await storedFor(connectorKey), null, "a rejected registration must not persist a manifest"); +} + +const SCENARIOS: Array<{ name: string; run: () => Promise | void }> = []; + +function scenario(name: string, run: () => Promise | void): void { + SCENARIOS.push({ name, run }); +} + +const PROVENANCE_VARIANTS: [string, Record][] = [ + ["provenance U1", { manifest_uri: DOC_ONE }], + ["provenance U2", { manifest_uri: DOC_TWO }], + ["no provenance", {}], +]; + +// Fresh first registration, unmapped custom key: provenance must not choose the source. +for (const [label, provenance] of PROVENANCE_VARIANTS) { + scenario(`a fresh custom key without a declaration is refused before any write (${label})`, async () => { + const key = "srcid-fresh-custom"; + await assertRejectedWithoutWrite( + key, + () => + registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + ...provenance, + }), + { + skipDerivedRepair: true, + } + ), + NO_SOURCE_MESSAGE + ); + }); +} + +// Fresh key-only registration (no connector_id), unmapped custom key: refused, no fallback. +scenario("a fresh key-only custom manifest without a declaration is refused before any write", async () => { + const key = "srcid-fresh-keyonly"; + await assertRejectedWithoutWrite( + key, + () => registerConnector(legacyManifest({ connector_key: key, manifest_uri: DOC_ONE }), { skipDerivedRepair: true }), + NO_SOURCE_MESSAGE + ); +}); + +// Unmapped registry-URL alias: a URL connector_id must not become the operational key or the source. +scenario("an unmapped registry URL connector_id is refused and never becomes a source", async () => { + const url = "https://registry.pdpp.dev/connectors/srcid-fresh-url"; + await assertRejectedWithoutWrite( + url, + () => registerConnector(legacyManifest({ connector_id: url, manifest_uri: DOC_ONE }), { skipDerivedRepair: true }), + URL_KEY_MESSAGE + ); +}); + +// Causal guard: an explicit, valid declaration does not make an unknown URL an operational key. +scenario("a URL-only connector_id with an explicit valid declaration is refused before any write", async () => { + const url = "https://sources.example/connectors/srcid-explicit-url"; + const key = "srcid-explicit-url"; + await assertRejectedWithoutWrite( + url, + () => + registerConnector( + legacyManifest({ + connector_id: url, + manifest_uri: DOC_ONE, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ), + URL_KEY_MESSAGE + ); + assert.equal(await storedFor(key), null, "the declared slug must not be registered either"); +}); + +// Known first-party key: the reviewed source identity wins over any incoming provenance. +for (const [label, provenance] of PROVENANCE_VARIANTS) { + scenario(`a first-party key takes its reviewed source identity regardless of provenance (${label})`, async () => { + const key = "amazon"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + ...provenance, + }), + { + skipDerivedRepair: true, + } + ); + assert.equal(retainedSourceId(await storedFor(key)), firstPartySource("amazon")); + }); +} + +// Known registry-URL alias with no connector_key resolves to its key and the same reviewed source. +scenario("a known registry URL alias resolves to its key and the reviewed source", async () => { + await registerConnector( + legacyManifest({ + connector_id: firstPartySource("gmail"), + manifest_uri: DOC_ONE, + }), + { skipDerivedRepair: true } + ); + const stored = await storedFor("gmail"); + assert.equal(stored?.connector_key, "gmail"); + assert.equal(retainedSourceId(stored), firstPartySource("gmail")); +}); + +// Known legacy local alias with its canonical key and no declaration resolves to the reviewed source. +scenario("a known legacy local alias with its canonical key takes the reviewed source", async () => { + await registerConnector( + legacyManifest({ + connector_id: "claude_code", + connector_key: "claude-code", + }), + { + skipDerivedRepair: true, + } + ); + assert.equal(retainedSourceId(await storedFor("claude-code")), firstPartySource("claude-code")); +}); + +// Control: an explicit validated declaration is the supported route for a first-party key too. +scenario("an explicit declaration wins over the reviewed mapping on a first registration", async () => { + const key = "amazon"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); +}); + +// Retained identity: a provenance-only update keeps the retained source (no fresh synthesis involved). +scenario("a provenance-only update keeps the retained source identity", async () => { + const key = "srcid-prov"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_TWO, + version: "1.1.0", + }), + { skipDerivedRepair: true } + ); + assert.equal( + retainedSourceId(await storedFor(key)), + EXPLICIT_SOURCE, + "manifest_uri must not replace the retained source" + ); +}); + +// Retained identity: a key-only update keeps the retained declaration. +scenario("a key-only update keeps the retained declaration", async () => { + const key = "srcid-keyonly"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + await registerConnector(legacyManifest({ connector_key: key, version: "1.1.0" }), { skipDerivedRepair: true }); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); +}); + +// Retained identity: an explicit different source identity cannot replace the retained one. +scenario("an explicit different source identity cannot replace the retained one", async () => { + const key = "srcid-replace"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + await assert.rejects( + () => + registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + source_declaration: explicitDeclaration(key, OTHER_SOURCE), + version: "1.1.0", + }), + { skipDerivedRepair: true } + ), + (error: unknown) => { + const err = error as { code?: unknown; message?: unknown }; + assert.equal(err.code, INVALID_REQUEST); + assert.match(String(err.message), ALREADY_BOUND); + return true; + } + ); + assert.equal( + retainedSourceId(await storedFor(key)), + EXPLICIT_SOURCE, + "the retained source must survive the rejected update" + ); +}); + +// Fail-before (explicit invalid): an explicit declaration is validated before it is retained. +scenario("an invalid explicit SourceDeclaration is rejected before persistence", async () => { + const key = "srcid-invalid"; + const invalid = { + ...explicitDeclaration(key, EXPLICIT_SOURCE), + protocol_version: "9.9.9", + }; + await assertRejectedWithoutWrite( + key, + () => + registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + source_declaration: invalid, + }), + { skipDerivedRepair: true } + ), + INVALID_SOURCE_DECLARATION + ); +}); + +// Control: a valid explicit declaration keeps its identity when provenance changes. +scenario("a valid explicit declaration keeps its source identity when provenance changes", async () => { + const key = "srcid-explicit"; + const declaration = explicitDeclaration(key, EXPLICIT_SOURCE); + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + source_declaration: declaration, + }), + { skipDerivedRepair: true } + ); + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_TWO, + source_declaration: declaration, + version: "1.1.0", + }), + { skipDerivedRepair: true } + ); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); +}); + +// Control: a legitimate schema update keeps the source and yields a new declaration version. +scenario("a schema update keeps the source identity and yields a new declaration version", async () => { + const key = "srcid-schema"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + const first = (await storedFor(key))?.source_declaration as { + declaration_version: string; + }; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + streams: [streamFor("items"), streamFor("more")], + version: "1.1.0", + }), + { skipDerivedRepair: true } + ); + const second = (await storedFor(key))?.source_declaration as { + declaration_version: string; + }; + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); + assert.notEqual(second.declaration_version, first.declaration_version); +}); + +// Control: the normalizer keeps a retained identity when given explicit context. +scenario("the normalizer keeps a retained identity when given it as context", () => { + const key = "srcid-context"; + const retained = explicitDeclaration(key, EXPLICIT_SOURCE); + const { storedManifest } = normalizeConnectorManifestForStorage( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_TWO, + }), + { retainedSourceDeclaration: retained } + ); + assert.equal(retainedSourceId(storedManifest), EXPLICIT_SOURCE); +}); + +// Control: an alias with a different explicit key is rejected (existing validator guard). +scenario("a known alias with a different explicit key is rejected", async () => { + await assertRejectedWithoutWrite( + "codex", + () => + registerConnector( + legacyManifest({ + connector_id: "claude_code", + connector_key: "codex", + }), + { + skipDerivedRepair: true, + } + ), + CANONICAL_KEY_MESSAGE + ); +}); + +// Control: an explicit key with an unmapped, unequal custom URI is rejected (existing validator guard). +scenario("an explicit key with an unmapped unequal custom URI is rejected", async () => { + await assertRejectedWithoutWrite( + "srcid-custom", + () => + registerConnector( + legacyManifest({ + connector_id: "https://example.org/manifest.json", + connector_key: "srcid-custom", + }), + { + skipDerivedRepair: true, + } + ), + KEY_MATCH_MESSAGE + ); +}); + +// Control: a known alias with its canonical explicit key is accepted when the source comes from an explicit declaration. +scenario("a known alias with its canonical explicit key is accepted", async () => { + await registerConnector( + legacyManifest({ + connector_id: "claude_code", + connector_key: "claude-code", + source_declaration: explicitDeclaration("claude-code", EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + const stored = await storedFor("claude-code"); + assert.equal(stored?.connector_key, "claude-code"); + assert.equal(retainedSourceId(stored), EXPLICIT_SOURCE); +}); + +// Control: an idempotent re-registration of the same content keeps everything. +scenario("an idempotent re-registration keeps the retained identity", async () => { + const key = "srcid-idem"; + const manifest = legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }); + await registerConnector(manifest, { skipDerivedRepair: true }); + await registerConnector(manifest, { skipDerivedRepair: true }); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); +}); + +// Concurrency: two registrations that name different explicit sources cannot both commit. +scenario("concurrent registrations of different explicit sources commit exactly one", async () => { + const key = "srcid-race"; + const results = await Promise.allSettled([ + registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ), + registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, OTHER_SOURCE), + }), + { skipDerivedRepair: true } + ), + ]); + const fulfilled = results.filter((result) => result.status === "fulfilled"); + assert.equal(fulfilled.length, 1, "exactly one registration may commit"); + const winner = retainedSourceId(await storedFor(key)); + assert.ok(winner === EXPLICIT_SOURCE || winner === OTHER_SOURCE); +}); + +interface Deferred { + readonly promise: Promise; + readonly resolve: () => void; +} + +function deferred(): Deferred { + let resolve: () => void = () => undefined; + const promise = new Promise((done) => { + resolve = done; + }); + return { promise, resolve }; +} + +interface ParkedWriter { + readonly arrived: Deferred; + readonly released: Deferred; +} + +/** + * Parks each registration of `connectorKey` at the stale-read point (after it observed the retained identity, + * before it persists) until the test releases it. Parking is keyed by the source the writer intends. + */ +function parkRegistrationsAtStaleRead( + connectorKey: string, + sourceIds: readonly string[] +): { + readonly gate: (sourceId: string) => ParkedWriter; + readonly release: (sourceId: string) => void; + readonly reset: () => void; +} { + const gates = new Map( + sourceIds.map((sourceId) => [sourceId, { arrived: deferred(), released: deferred() }]) + ); + const gate = (sourceId: string): ParkedWriter => { + const found = gates.get(sourceId); + if (!found) { + throw new Error(`no parked writer for ${sourceId}`); + } + return found; + }; + __setRegisterConnectorPhaseHookForTest(async (point, context) => { + if (point !== "before-manifest-persisted" || context.connectorId !== connectorKey) { + return; + } + const sourceId = retainedSourceId(context.manifest as Record); + const writer = sourceId === undefined ? undefined : gates.get(sourceId); + if (!writer) { + return; + } + writer.arrived.resolve(); + await writer.released.promise; + }); + return { + gate, + release: (sourceId) => gate(sourceId).released.resolve(), + reset: () => { + __setRegisterConnectorPhaseHookForTest(null); + for (const writer of gates.values()) { + writer.released.resolve(); + } + }, + }; +} + +/** + * Resolves when the writer reaches its stale-read point. Fails if the writer settles first: then the + * interleaving was not replayed. Racing (not awaiting) keeps a late settlement from becoming an unhandled rejection. + */ +function parkedBefore(arrived: Promise, writer: Promise): Promise { + return Promise.race([ + arrived, + writer.then(() => { + throw new Error("a writer settled before the stale-read point: the interleaving was not replayed"); + }), + ]); +} + +function explicitRegistration(key: string, sourceId: string, extra: Record = {}) { + return registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, sourceId), + ...extra, + }), + { skipDerivedRepair: true } + ); +} + +// Causal replay of the unlocked interleaving (source-identity-boundary-r5 model, Locked = FALSE): both writers read +// "no retained identity" before either persists. The commit-time check must keep the first commit and refuse the +// second, leaving the registry row byte-identical. +scenario( + "two writers that both read no retained identity: the commit-time check keeps the first and refuses the second", + async () => { + const key = "srcid-race-replay"; + const park = parkRegistrationsAtStaleRead(key, [EXPLICIT_SOURCE, OTHER_SOURCE]); + const writerA = explicitRegistration(key, EXPLICIT_SOURCE); + const writerB = explicitRegistration(key, OTHER_SOURCE); + try { + await Promise.all([ + parkedBefore(park.gate(EXPLICIT_SOURCE).arrived.promise, writerA), + parkedBefore(park.gate(OTHER_SOURCE).arrived.promise, writerB), + ]); + park.release(EXPLICIT_SOURCE); + await writerA; + const afterA = JSON.stringify(await storedFor(key)); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); + + park.release(OTHER_SOURCE); + await assert.rejects(writerB, (error: unknown) => { + const err = error as { code?: unknown; message?: unknown }; + assert.equal(err.code, INVALID_REQUEST); + assert.match(String(err.message), ALREADY_BOUND); + return true; + }); + assert.equal(JSON.stringify(await storedFor(key)), afterA, "the refused writer must not change the registry row"); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE, "the first writer's source survives"); + assert.equal(await getConnectorActivation(key), null, "no activation is written by a registration"); + } finally { + park.reset(); + await Promise.allSettled([writerA, writerB]); + } + } +); + +// Installer publish races a registration that read no identity. The registration resumes last and must be refused. +scenario( + "a registration that read no identity before an installer publish is refused and the installer source survives", + async () => { + const key = "srcid-race-install"; + const park = parkRegistrationsAtStaleRead(key, [EXPLICIT_SOURCE]); + const registration = explicitRegistration(key, EXPLICIT_SOURCE); + try { + await parkedBefore(park.gate(EXPLICIT_SOURCE).arrived.promise, registration); + await publishConnectorActivation( + installRecord( + key, + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, OTHER_SOURCE), + }) + ) + ); + park.release(EXPLICIT_SOURCE); + await assert.rejects(registration, (error: unknown) => { + const err = error as { code?: unknown; message?: unknown }; + assert.equal(err.code, INVALID_REQUEST); + assert.match(String(err.message), ALREADY_BOUND); + return true; + }); + assert.equal(retainedSourceId(await storedFor(key)), OTHER_SOURCE, "the installer's source is retained"); + const activation = await getConnectorActivation(key); + assert.equal(retainedSourceId(activation?.canonicalManifest ?? null), OTHER_SOURCE); + } finally { + park.reset(); + await Promise.allSettled([registration]); + } + } +); + +// A seeded retained identity that differs from the reviewed map must survive a schema/provenance update through the writer. +scenario("a retained custom source for a mapped key survives a schema update without a declaration", async () => { + const key = "amazon"; + await explicitRegistration(key, EXPLICIT_SOURCE); + const before = (await storedFor(key))?.source_declaration as { declaration_version: string } | undefined; + assert.notEqual(EXPLICIT_SOURCE, firstPartySource("amazon"), "the seed must differ from the reviewed map"); + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_TWO, + streams: [streamFor("items"), streamFor("more")], + version: "1.1.0", + }), + { skipDerivedRepair: true } + ); + const stored = await storedFor(key); + const after = stored?.source_declaration as { + declaration_version: string; + publisher: { id: string }; + streams: { name: string }[]; + }; + assert.equal(retainedSourceId(stored), EXPLICIT_SOURCE, "the retained custom source, not the map, is kept"); + assert.equal(after.publisher.id, PUBLISHER, "the retained publisher is kept"); + assert.deepEqual( + after.streams.map((stream) => stream.name), + ["items", "more"], + "the new schema is the one stored" + ); + assert.equal(stored?.version, "1.1.0"); + assert.notEqual(after.declaration_version, before?.declaration_version); + assert.equal(await getConnectorActivation(key), null); +}); + +// Pure lookup: prototype names and unmapped names never resolve, and no key can be repointed through the exports. +scenario("prototype and unmapped names never resolve to a reviewed source", async () => { + for (const name of ["constructor", "__proto__", "toString", "hasOwnProperty", "valueOf", "srcid-not-mapped"]) { + assert.equal(legacyFirstPartySourceIdFor(name), undefined, `${name} must not resolve`); + } + await assertRejectedWithoutWrite( + "constructor", + () => + registerConnector(legacyManifest({ connector_id: "constructor", connector_key: "constructor" }), { + skipDerivedRepair: true, + }), + NO_SOURCE_MESSAGE + ); +}); + +// Mutation control: the exported map views are frozen; a mutation attempt is refused and the lookup is unchanged. +scenario("exported key, entry and tuple views cannot be mutated to repoint a lookup", () => { + assert.equal(Object.isFrozen(LEGACY_FIRST_PARTY_KEYS), true); + assert.equal(Object.isFrozen(LEGACY_FIRST_PARTY_ENTRIES), true); + const [firstKey, firstSource] = LEGACY_FIRST_PARTY_ENTRIES[0] ?? []; + assert.ok(firstKey && firstSource); + const tuple = LEGACY_FIRST_PARTY_ENTRIES[0] as unknown as object; + assert.equal(Reflect.set(tuple, 1, OTHER_SOURCE), false, "a frozen entry tuple refuses assignment"); + assert.equal(Reflect.set(LEGACY_FIRST_PARTY_ENTRIES as unknown as object, 0, [firstKey, OTHER_SOURCE]), false); + assert.equal(Reflect.set(LEGACY_FIRST_PARTY_KEYS as unknown as object, 0, "constructor"), false); + assert.equal(legacyFirstPartySourceIdFor(firstKey), firstSource, "the lookup is unchanged"); + assert.equal(legacyFirstPartySourceIdFor("amazon"), firstPartySource("amazon")); +}); + +// Null and undefined mean "unspecified" (documented normalizer behavior), never a request for a map fallback. +async function assertAbsenceDeclaration(declaration: null | undefined): Promise { + await registerConnector( + legacyManifest({ connector_id: "amazon", connector_key: "amazon", source_declaration: declaration }), + { skipDerivedRepair: true } + ); + assert.equal(retainedSourceId(await storedFor("amazon")), firstPartySource("amazon")); + await assertRejectedWithoutWrite( + "srcid-null-unmapped", + () => + registerConnector( + legacyManifest({ + connector_id: "srcid-null-unmapped", + connector_key: "srcid-null-unmapped", + source_declaration: declaration, + }), + { skipDerivedRepair: true } + ), + NO_SOURCE_MESSAGE + ); +} + +scenario( + "a null or undefined declaration is absence: mapped keys take the map, unmapped keys refuse, retained keys keep", + async () => { + await assertAbsenceDeclaration(null); + await assertAbsenceDeclaration(undefined); + const key = "srcid-null-retained"; + await explicitRegistration(key, EXPLICIT_SOURCE); + await registerConnector( + legacyManifest({ connector_id: key, connector_key: key, source_declaration: null, version: "1.1.0" }), + { skipDerivedRepair: true } + ); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); + } +); + +// Malformed nonnull declarations refuse before any write, and never fall back to the reviewed map or provenance. +for (const [label, malformed] of [ + ["an empty object", {}], + ["a string", "declaration"], + ["a number", 42], + ["a boolean", true], + ["an array", []], +] as const) { + scenario(`${label} as a declaration is refused before write on a mapped key (no map fallback)`, async () => { + await assertRejectedWithoutWrite( + "amazon", + () => + registerConnector( + legacyManifest({ connector_id: "amazon", connector_key: "amazon", source_declaration: malformed }), + { + skipDerivedRepair: true, + } + ), + INVALID_SOURCE_DECLARATION + ); + }); +} + +/** The POST /connectors operation, wired with the same dependencies the server mounts. */ +function postRegister(manifest: Record) { + return executeAsPolyfillConnectorRegister( + { manifest }, + { + declarationDigest: connectorDeclarationDigest, + getAcceptedDeclarationDigest, + getConnectorManifest, + normalizeManifestForStorage: normalizeConnectorManifestForStorage, + registerConnector: (stored: Record) => registerConnector(stored, { skipDerivedRepair: true }), + validateManifest: (stored: Record) => validateConnectorManifest(stored), + } + ); +} + +// POST control: an identical re-POST of a retained connector is idempotent and keeps its source. +scenario("an identical POST to a retained connector is idempotent and keeps its source", async () => { + const key = "srcid-post-idem"; + const manifest = legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }); + await registerConnector(manifest, { skipDerivedRepair: true }); + const outcome = await postRegister(manifest); + assert.equal(outcome.outcome, "success", JSON.stringify(outcome)); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); +}); + +// POST control: a first-party identical re-POST with no declaration is idempotent under the reviewed source. +scenario("an identical POST of a first-party connector without a declaration is idempotent", async () => { + const manifest = legacyManifest({ + connector_id: "amazon", + connector_key: "amazon", + manifest_uri: DOC_ONE, + }); + const first = await postRegister(manifest); + assert.equal(first.outcome, "success", JSON.stringify(first)); + const second = await postRegister(manifest); + assert.equal(second.outcome, "success", JSON.stringify(second)); + assert.equal(retainedSourceId(await storedFor("amazon")), firstPartySource("amazon")); +}); + +// POST: a version update without a declaration keeps the retained source, whatever provenance it names. +scenario("a POST version update without a declaration keeps the retained source", async () => { + const key = "srcid-post-update"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + const outcome = await postRegister( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_TWO, + version: "1.1.0", + }) + ); + assert.equal(outcome.outcome, "success", JSON.stringify(outcome)); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); +}); + +// POST by a known registry alias: an identical re-POST compares against the retained identity of its canonical key. +// A lookup under the raw alias misses the retained row, takes the reviewed fresh-path source, and reports +// equivocation for content the server already accepted. +scenario( + "an identical POST by a known registry alias to a retained connector is idempotent and keeps its source", + async () => { + const key = "amazon"; + const alias = `https://registry.pdpp.dev/connectors/${key}`; + // The retained declaration is the one a re-projection of this provenance produces (implementation id = manifest_uri), + // so an identical re-POST can be compared by digest. + const retained = snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest( + legacyManifest({ connector_id: key, connector_key: key }), + { connectorImplementationId: alias, publisherId: PUBLISHER, sourceId: EXPLICIT_SOURCE } + ); + await registerConnector( + legacyManifest({ connector_id: key, connector_key: key, manifest_uri: alias, source_declaration: retained }), + { skipDerivedRepair: true } + ); + const outcome = await postRegister(legacyManifest({ connector_id: alias, manifest_uri: alias })); + assert.equal(outcome.outcome, "success", JSON.stringify(outcome)); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE); + } +); + +/** An installer record whose manifest is the given one; only the identity fields matter here. */ +function installRecord(connectorKey: string, manifest: Record): ConnectorInstallRecord { + const hex = (digit: string) => `sha256:${digit.repeat(64)}`; + return { + activatedAt: "2026-11-02T00:00:00.000Z", + bindings: {}, + configDigest: hex("0"), + connectorId: connectorKey, + digest: hex("1"), + entrypointPath: "/app/entrypoint.js", + entrypointSha256: hex("2"), + manifest, + manifestPath: "/app/manifest.json", + manifestSha256: hex("3"), + provenancePath: "/app/provenance.json", + provenanceSha256: hex("4"), + registry: "ghcr.io", + repository: `ghcr.io/example/connectors/${connectorKey}`, + root: `/var/lib/pdpp/${connectorKey}`, + tier: null, + version: "1.0.0", + }; +} + +// Installer: a retained source survives a provenance-only install. +scenario("an installer publish keeps the retained source identity across a provenance change", async () => { + const key = "srcid-install-prov"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + await publishConnectorActivation( + installRecord( + key, + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_TWO, + }) + ) + ); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE, "the registry keeps the retained source"); + const activation = await getConnectorActivation(key); + assert.equal( + retainedSourceId(activation?.canonicalManifest ?? null), + EXPLICIT_SOURCE, + "the activation records the same source" + ); +}); + +// Installer: an explicit different identity is rejected before any write. +scenario("an installer publish cannot rebind a retained source to an explicit different one", async () => { + const key = "srcid-install-replace"; + await registerConnector( + legacyManifest({ + connector_id: key, + connector_key: key, + source_declaration: explicitDeclaration(key, EXPLICIT_SOURCE), + }), + { skipDerivedRepair: true } + ); + await assert.rejects( + () => + publishConnectorActivation( + installRecord( + key, + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + source_declaration: explicitDeclaration(key, OTHER_SOURCE), + }) + ) + ), + (error: unknown) => { + const err = error as { code?: unknown; message?: unknown }; + assert.equal(err.code, INVALID_REQUEST); + assert.match(String(err.message), ALREADY_BOUND); + return true; + } + ); + assert.equal(retainedSourceId(await storedFor(key)), EXPLICIT_SOURCE, "the registry keeps the retained source"); + assert.equal(await getConnectorActivation(key), null, "no activation is written for a rejected publish"); +}); + +// Installer, fresh: an unmapped custom key with provenance is refused before any registry or activation write. +scenario("an installer publish refuses a fresh custom key without a declaration", async () => { + const key = "srcid-install-fresh"; + await assert.rejects( + () => + publishConnectorActivation( + installRecord( + key, + legacyManifest({ + connector_id: key, + connector_key: key, + manifest_uri: DOC_ONE, + }) + ) + ), + (error: unknown) => { + const err = error as { code?: unknown; message?: unknown }; + assert.equal(err.code, INVALID_REQUEST); + assert.match(String(err.message), NO_SOURCE_MESSAGE); + return true; + } + ); + assert.equal(await storedFor(key), null, "no registry row is written"); + assert.equal(await getConnectorActivation(key), null, "no activation is written"); +}); + +// The reviewed map is the proof for fresh first-party registrations: it must equal the pinned package +// identities exactly. A package change that adds, removes or re-points a shipped key fails here, so the +// map is reviewed rather than extended silently. +test("the reviewed legacy source map equals the pinned first-party package identities", () => { + const pinned = readPolyfillManifests(); + const byKey = new Map(pinned.map((entry) => [entry.manifest.connector_key, entry.manifest])); + assert.equal(pinned.length, 45, "the pinned package ships 45 first-party manifests"); + assert.deepEqual([...LEGACY_FIRST_PARTY_KEYS].sort(), [...byKey.keys()].sort()); + const canonicalPairs = [...LEGACY_FIRST_PARTY_ENTRIES].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)); + const mappingPin = createHash("sha256").update(JSON.stringify(canonicalPairs), "utf8").digest("hex"); + assert.equal(mappingPin, "7e8f726725b7a14705571339c84a52d8f05888ffc0996b77cce3d4ae13dae9e6"); + for (const [key, sourceId] of LEGACY_FIRST_PARTY_ENTRIES) { + const manifest = byKey.get(key); + assert.ok(manifest, `pinned manifest for ${key}`); + assert.equal(manifest.connector_id, sourceId, `${key}: connector_id is the reviewed source`); + assert.equal(manifest.manifest_uri, sourceId, `${key}: manifest_uri is the reviewed source`); + assert.equal(manifest.source_declaration, undefined, `${key}: the pinned manifest declares no source`); + } +}); + +for (const entry of SCENARIOS) { + test(`SQLite: ${entry.name}`, async () => { + initDb(":memory:"); + try { + await entry.run(); + } finally { + closeDb(); + } + }); + + test(`PostgreSQL: ${entry.name} (skipped: PDPP_TEST_POSTGRES_URL unset)`, { + skip: !process.env.PDPP_TEST_POSTGRES_URL, + }, async () => { + const url = process.env.PDPP_TEST_POSTGRES_URL; + assert.ok(url); + await withTemporaryPostgresDatabase( + { + closeConnections: closePostgresStorage, + connectionString: url, + databaseName: `pdpp_test_srcid_${Date.now().toString(36)}_${Math.random().toString(16).slice(2, 10)}_1`, + templateName: null, + }, + async (databaseUrl) => { + initDb(":memory:"); + try { + await initPostgresStorage({ backend: "postgres", databaseUrl }); + await entry.run(); + } finally { + await closePostgresStorage(); + closeDb(); + } + } + ); + }); +} diff --git a/reference-implementation/test/control-actions.test.ts b/reference-implementation/test/control-actions.test.ts index 4283db924..1bb6fc190 100644 --- a/reference-implementation/test/control-actions.test.ts +++ b/reference-implementation/test/control-actions.test.ts @@ -392,7 +392,7 @@ async function emitSyntheticRun({ actor_id: connectorId, actor_type: "runtime", data: { - bindings: { filesystem: {}, interactive: {}, network: {} }, + bindings: { browser: {}, filesystem: {}, network: {} }, boot_epoch: _epoch.boot_epoch, collection_mode: "incremental", connector_instance_id: connectorInstanceId, @@ -478,7 +478,7 @@ async function emitSyntheticStartedRun({ actor_id: connectorId, actor_type: "runtime", data: { - bindings: { filesystem: {}, interactive: {}, network: {} }, + bindings: { browser: {}, filesystem: {}, network: {} }, boot_epoch: _epoch.boot_epoch, collection_mode: "incremental", connection_id: connectorInstanceId, @@ -1051,7 +1051,7 @@ test("controller startup reconciles abandoned controller-managed runs after rest actor_id: connectorId, actor_type: "runtime", data: { - bindings: { filesystem: {}, interactive: {}, network: {} }, + bindings: { browser: {}, filesystem: {}, network: {} }, // Synthetic "prior incarnation" stamp — this test seeds the // pre-crash state of an interrupted run. // diff --git a/reference-implementation/test/detail-coverage-shortfall-severity.test.ts b/reference-implementation/test/detail-coverage-shortfall-severity.test.ts index ea19e9967..2b5c22572 100644 --- a/reference-implementation/test/detail-coverage-shortfall-severity.test.ts +++ b/reference-implementation/test/detail-coverage-shortfall-severity.test.ts @@ -43,12 +43,12 @@ import { join } from "node:path"; // carrying a durable DETAIL_GAP. import test from "node:test"; import { type RuntimeRunConnectorResult, runConnector } from "../runtime/index.ts"; - import { getDb } from "../server/db.ts"; import { startServer } from "../server/index.ts"; import { createRequestConnectorInstanceStore } from "../server/request-store-factories.ts"; import { getDefaultConnectorDetailGapStore } from "../server/stores/connector-detail-gap-store.ts"; import { admitOwnerRunConnection } from "../server/stores/connector-instance-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; // Hoisted: the coverage-shortfall explanation the runtime authors, and the // connection-scoped denominator the sibling-isolation oracle asserts on. @@ -188,49 +188,56 @@ async function issueOwnerToken(asUrl: string): Promise { // Connector-agnostic shape of the live failure: one parent state_stream // ("holdings") whose key fans out to two child detail streams. -const MANIFEST = { - connector_id: "coverage-shortfall-regression", - display_name: "Coverage Shortfall Regression", - manifest_uri: "https://registry.pdpp.dev/connectors/coverage-shortfall-regression", - protocol_version: "0.1.0", - streams: [ - { - name: "holdings", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - { - name: "activity", - primary_key: ["id"], - relationships: [{ cardinality: "has_one", foreign_key: "holding_id", name: "holding", stream: "holdings" }], - schema: { - properties: { holding_id: { type: "string" }, id: { type: "string" } }, - required: ["id"], - type: "object", +const MANIFEST = withFixtureSourceDeclaration( + { + connector_id: "coverage-shortfall-regression", + display_name: "Coverage Shortfall Regression", + manifest_uri: "https://registry.pdpp.dev/connectors/coverage-shortfall-regression", + protocol_version: "0.1.0", + streams: [ + { + name: "holdings", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - { - coverage_strategy: "parent_detail_accounting", - name: "valuations", - parent_streams: ["holdings", "activity"], - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", -}; + { + name: "activity", + primary_key: ["id"], + relationships: [{ cardinality: "has_one", foreign_key: "holding_id", name: "holding", stream: "holdings" }], + schema: { + properties: { holding_id: { type: "string" }, id: { type: "string" } }, + required: ["id"], + type: "object", + }, + selection: { fields: true, resources: true }, + semantics: "append_only", + }, + { + coverage_strategy: "parent_detail_accounting", + name: "valuations", + parent_streams: ["holdings", "activity"], + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "append_only", + }, + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/coverage-shortfall-regression" } +); /** - * The same connector, declaring DETAIL_COVERAGE_KEY_SETS. Only a connector that - * declares it gets evidence-based crediting of hydrated keys; MANIFEST keeps - * the crediting it had before the capability existed. The capability is new in - * Collection Profile 0.2.0, so the manifest declares that version (Section 3.1). + * The same connector, declaring DETAIL_COVERAGE_KEY_SETS. The manifest + * declaration alone makes a hydrated key need an accepted detail record for it: + * the runtime derives that requirement from the manifest, not from START. The + * canned connectors below never see the extension in START, so they follow the + * base hydrated_keys fallback. MANIFEST, which does not declare the extension, + * credits a hydrated key by assertion. The runtime's manifest validation + * requires protocol_version 0.2.0 for a manifest that declares the extension. + * That is a package rule, not canonical Collection Profile text. */ const KEY_SETS_MANIFEST = { ...MANIFEST, @@ -1273,8 +1280,9 @@ test("a 0.1.0 manifest keeps legacy crediting and cannot declare a 0.2.0 capabil // Pre-spawn gate (Sections 3.4 and 4): an unknown capability name fails closed, // and a known required capability this runtime does not implement fails before -// any connector code runs. DETAIL_COVERAGE_KEY_SETS is advisory and -// STREAM_EVIDENCE is implemented, so both pass. BLOB is covered separately. +// any connector code runs. DETAIL_COVERAGE_KEY_SETS is advisory, and OBSERVATION +// and STREAM_EVIDENCE are implemented, so all three pass. BLOB is required and +// unimplemented, so it is refused here before spawn. test("the pre-spawn gate rejects unknown and unsupported required capabilities and admits implemented ones", async (t) => { const server = await typedStartServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); t.after(() => closeServer(server)); @@ -1291,7 +1299,7 @@ test("the pre-spawn gate rejects unknown and unsupported required capabilities a ["DETAIL_COVERAGE_KEY_SETS", UNKNOWN_CAPABILITY_PATTERN, null], [null, UNKNOWN_CAPABILITY_PATTERN, null], [["OBSERVATION"], null, null], - [["DETAIL_COVERAGE_KEY_SETS", "OBSERVATION", "BLOB"], null, null], + [["DETAIL_COVERAGE_KEY_SETS", "OBSERVATION", "BLOB"], UNSUPPORTED_CAPABILITY_PATTERN, "BLOB"], [["DETAIL_COVERAGE_KEY_SETS", "STREAM_EVIDENCE"], null, null], ]; for (const [protocolCapabilities, rejection, unsupported] of cases) { @@ -1317,16 +1325,17 @@ test("the pre-spawn gate rejects unknown and unsupported required capabilities a } }); -// START.protocol_capabilities lists what this runtime implements (Section -// 5.1): it accepts empty_keys/unchanged_keys and it validates STREAM_EVIDENCE. -// It also accepts OBSERVATION (Section 5.10). It does not implement BLOB, so it -// does not list it. +// START.protocol_capabilities lists what this runtime implements (Section 5.1): +// OBSERVATION (Section 5.10) and STREAM_EVIDENCE (Section 5.7). It does not list +// the advisory DETAIL_COVERAGE_KEY_SETS, because it does not implement the +// extension's crediting, so a connector falls back to hydrated_keys. BLOB is +// unimplemented and never listed. test("START advertises the protocol capabilities this runtime implements, to every connector", async (t) => { const server = await typedStartServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); t.after(() => closeServer(server)); const ownerToken = await registerAndAuthorize(server); - for (const manifest of [MANIFEST, KEY_SETS_MANIFEST]) { - // biome-ignore lint/performance/noAwaitInLoops: each manifest is one isolated run. + await [MANIFEST, KEY_SETS_MANIFEST].reduce(async (previous, manifest) => { + await previous; const { start, thrown } = await runCapturingStart( server, ownerToken, @@ -1334,33 +1343,40 @@ test("START advertises the protocol capabilities this runtime implements, to eve holdingsRun([holdingsCoverage({ empty_keys: ["K"], required_keys: ["K"] })]) ); assert.equal(thrown, null); - assert.deepEqual(start?.protocol_capabilities, ["DETAIL_COVERAGE_KEY_SETS", "OBSERVATION", "STREAM_EVIDENCE"]); - } + assert.deepEqual(start?.protocol_capabilities, ["OBSERVATION", "STREAM_EVIDENCE"]); + }, Promise.resolve()); }); -// BLOB is a runtime-specific extension (Collection Profile Section 7) that this -// runtime does not implement. chatgpt and anthropic declare it but emit BLOB -// only for an item over the inline cap, so a declaring connector still starts -// here. A run that emits no BLOB completes; a run that emits one fails at that -// line, with the same error as before the capability gate existed. -test("a 0.1.0 manifest declaring BLOB starts; it completes without BLOB and fails at a BLOB line", async (t) => { +// BLOB is required by a manifest that declares it, and this runtime does not +// implement it. The gate refuses such a manifest before spawn, in 0.1.0 and 0.2.0, +// whether or not the connector would emit a BLOB line. This replaces the former +// spawn waiver. A connector that did not declare BLOB and emits one is rejected +// at that line, as an unknown message type. +const BLOB_REFUSED_PATTERN = /requires protocol capabilities this runtime does not support: BLOB$/; + +test("a manifest declaring BLOB is refused before spawn at 0.1.0 and 0.2.0, even when no BLOB would be emitted", async (t) => { const server = await typedStartServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); t.after(() => closeServer(server)); const ownerToken = await registerAndAuthorize(server); - const blobManifest = { ...MANIFEST, protocol_capabilities: ["BLOB"], version: "1.2.0" }; - assert.equal(isAdmitted(await registerManifest(server, blobManifest)), true, "the BLOB manifest is admitted"); + const blobManifest010 = { ...MANIFEST, protocol_capabilities: ["BLOB"], version: "1.2.0" }; + const blobManifest020 = { ...KEY_SETS_MANIFEST, protocol_capabilities: ["BLOB"], version: "1.2.1" }; + await [blobManifest010, blobManifest020].reduce(async (previous, blobManifest) => { + await previous; + const refused = await runCapturingStart( + server, + ownerToken, + blobManifest, + holdingsRun([holdingsCoverage({ empty_keys: ["K"], required_keys: ["K"] })]) + ); + assert.equal(refused.start, null, `${blobManifest.protocol_version} BLOB manifest never reaches the connector`); + assert.match(String((refused.thrown as Error | null)?.message), BLOB_REFUSED_PATTERN); + }, Promise.resolve()); +}); - const clean = await runCapturingStart( - server, - ownerToken, - blobManifest, - holdingsRun([holdingsCoverage({ empty_keys: ["K"], required_keys: ["K"] })]) - ); - assert.equal(clean.thrown, null); - assert.ok(clean.start, "the connector received START"); - assert.equal(clean.start?.protocol_capabilities instanceof Array, true); - assert.equal((clean.start?.protocol_capabilities as string[]).includes("BLOB"), false, "START does not offer BLOB"); - assert.equal(clean.result?.status, "succeeded"); +test("a connector that emits BLOB without declaring it is rejected at that line as an unknown message", async (t) => { + const server = await typedStartServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); + t.after(() => closeServer(server)); + const ownerToken = await registerAndAuthorize(server); const blobLine = { file: "blob-1.json", @@ -1371,7 +1387,7 @@ test("a 0.1.0 manifest declaring BLOB starts; it completes without BLOB and fail stream: "holdings", type: "BLOB", }; - const withBlob = await runCapturingStart(server, ownerToken, blobManifest, [blobLine, ...holdingsRun([])]); + const withBlob = await runCapturingStart(server, ownerToken, MANIFEST, [blobLine, ...holdingsRun([])]); assert.ok(withBlob.start, "the connector received START"); const failure = withBlob.thrown instanceof Error ? withBlob.thrown.message : JSON.stringify(withBlob.result); assert.match(failure, UNKNOWN_BLOB_MESSAGE_PATTERN); diff --git a/reference-implementation/test/event-spine.test.ts b/reference-implementation/test/event-spine.test.ts index 40e60dd04..373b7cf49 100644 --- a/reference-implementation/test/event-spine.test.ts +++ b/reference-implementation/test/event-spine.test.ts @@ -30,6 +30,7 @@ import { startServer as startServerUntyped } from "../server/index.ts"; import { ingestRecord as ingestRecordUntyped } from "../server/records.ts"; import { createRequestConnectorInstanceStore } from "../server/request-store-factories.ts"; import { admitOwnerRunConnection } from "../server/stores/connector-instance-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; // Real ingest resolves the acting owner subject from the request's bearer // token (`getOwnerTokenSubjectId` in server/index.ts), independent of @@ -2096,7 +2097,6 @@ test("event spine", async (t) => { assert.deepEqual(startedEvent.data.bindings, { browser: {}, filesystem: {}, - interactive: {}, network: {}, }); assert.deepEqual(startedEvent.data.scope, { @@ -2190,8 +2190,11 @@ rl.on('line', (line) => { }); const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; - const manifest = { + const manifest = withFixtureSourceDeclaration({ connector_id: "event-spine-multi-stream-checkpoint-test", + connector_key: "event-spine-multi-stream-checkpoint-test", + display_name: "Event Spine Multi-Stream Checkpoint Test", + protocol_version: "0.1.0", streams: [ { name: "items", @@ -2204,6 +2207,8 @@ rl.on('line', (line) => { required: ["id"], type: "object", }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", }, { name: "other_items", @@ -2216,22 +2221,11 @@ rl.on('line', (line) => { required: ["id"], type: "object", }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", }, ], version: "0.1.0", - }; - Object.assign(manifest, { - source_declaration: { - declaration_version: "event-spine-multi-stream-checkpoint-test.v1", - display: { name: "Event Spine Multi-Stream Checkpoint Test" }, - protocol_version: "0.1.0", - publisher: { id: "https://pdpp.dev/reference-implementation/tests" }, - source: { - id: "https://registry.pdpp.dev/connectors/event-spine-multi-stream-checkpoint-test", - kind: "connector", - }, - streams: manifest.streams, - }, }); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-event-spine-multi-stream-commit-")); @@ -2333,8 +2327,10 @@ rl.on('line', (line) => { rsPort: 0, }); const asUrl = `http://localhost:${server.asPort}`; - const manifest = { - connector_id: "https://registry.pdpp.dev/connectors/event-spine-partial-checkpoint-failure-test", + const manifest = withFixtureSourceDeclaration({ + connector_id: "event-spine-partial-checkpoint-failure-test", + connector_key: "event-spine-partial-checkpoint-failure-test", + display_name: "Event Spine Partial Checkpoint Failure Test", protocol_version: "0.1.0", streams: [ { @@ -2348,6 +2344,8 @@ rl.on('line', (line) => { required: ["id"], type: "object", }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", }, { name: "other_items", @@ -2360,22 +2358,11 @@ rl.on('line', (line) => { required: ["id"], type: "object", }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", }, ], version: "0.1.0", - }; - Object.assign(manifest, { - source_declaration: { - declaration_version: "event-spine-partial-checkpoint-failure-test.v1", - display: { name: "Event Spine Partial Checkpoint Failure Test" }, - protocol_version: "0.1.0", - publisher: { id: "https://pdpp.dev/reference-implementation/tests" }, - source: { - id: manifest.connector_id, - kind: "connector", - }, - streams: manifest.streams, - }, }); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-event-spine-partial-checkpoint-failure-")); @@ -4730,7 +4717,15 @@ rl.on('line', (line) => { collectionMode: "full_refresh", connectorId: spotifyManifest.connector_id, connectorPath, - manifest: spotifyManifest, + // The connector emits an otp INTERACTION. The seed declares only + // `credentials`, so this run's manifest also declares `otp`. + manifest: { + ...spotifyManifest, + capabilities: { + ...((spotifyManifest.capabilities as Record | undefined) ?? {}), + human_interaction: ["credentials", "otp"], + }, + }, onInteraction: async () => new Promise(() => { /* intentionally waits for the page to advance */ diff --git a/reference-implementation/test/helpers/checkpoint-dependency-multi-parent-scenario.ts b/reference-implementation/test/helpers/checkpoint-dependency-multi-parent-scenario.ts index a82acddca..0838692e8 100644 --- a/reference-implementation/test/helpers/checkpoint-dependency-multi-parent-scenario.ts +++ b/reference-implementation/test/helpers/checkpoint-dependency-multi-parent-scenario.ts @@ -18,6 +18,7 @@ import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import type { runConnector as RunConnectorType } from "../../runtime/index.ts"; +import { withFixtureSourceDeclaration } from "./fixture-source-declaration.ts"; type RunConnectorFn = typeof RunConnectorType; @@ -91,39 +92,42 @@ export function fakeAdmitRunConnection( } export function multiParentManifest(connectorId: string) { - return { - capabilities: { human_interaction: [] }, - connector_id: connectorId, - display_name: connectorId, - manifest_uri: `https://sources.example/${connectorId}`, - protocol_version: "0.1.0", - streams: [ - { - name: "holdings", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - name: "activity", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - coverage_strategy: "parent_detail_accounting", - name: "valuations", - parent_streams: ["holdings", "activity"], - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "0.1.0", - }; + return withFixtureSourceDeclaration( + { + capabilities: { human_interaction: [] }, + connector_id: connectorId, + display_name: connectorId, + manifest_uri: `https://sources.example/${connectorId}`, + protocol_version: "0.1.0", + streams: [ + { + name: "holdings", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + name: "activity", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + coverage_strategy: "parent_detail_accounting", + name: "valuations", + parent_streams: ["holdings", "activity"], + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "0.1.0", + }, + { sourceId: `https://sources.example/${connectorId}` } + ); } function writeConnectorStub(tmpDir: string, script: string): string { @@ -175,7 +179,7 @@ export async function runMultiParentScenario({ process.stdout.write(JSON.stringify({ type: 'RECORD', stream: 'holdings', key: 'H', data: { id: 'H' }, emitted_at: new Date().toISOString() }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'RECORD', stream: 'activity', key: 'A', data: { id: 'A' }, emitted_at: new Date().toISOString() }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'DETAIL_COVERAGE', reference_only: true, stream: 'valuations', state_stream: 'holdings', required_keys: ['H'], hydrated_keys: [] }) + '\\n'); - process.stdout.write(JSON.stringify({ type: 'DETAIL_COVERAGE', reference_only: true, stream: 'valuations', state_stream: 'activity', required_keys: ['A'], hydrated_keys: [], empty_keys: ['A'] }) + '\\n'); + process.stdout.write(JSON.stringify({ type: 'DETAIL_COVERAGE', reference_only: true, stream: 'valuations', state_stream: 'activity', required_keys: ['A'], hydrated_keys: ['A'] }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'STATE', stream: 'holdings', cursor: { page: 'holdings-1' } }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'STATE', stream: 'activity', cursor: { page: 'activity-1' } }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'DONE', status: 'succeeded', records_emitted: 2 }) + '\\n'); @@ -229,13 +233,19 @@ export async function runMultiParentScenario({ } /** - * Hydrated-key crediting depends on the connector's own manifest declaration. - * The connector lists `H` as hydrated for the holdings boundary but emits no - * valuations record for it, and reports `A` as verified empty for activity. - * With DETAIL_COVERAGE_KEY_SETS declared, the bare `H` listing is not evidence - * and holdings stays put. Without it, the connector keeps the crediting it had - * before the capability existed and both cursors commit. Shared by the SQLite - * and Postgres conformance files so both backends run one scenario. + * DETAIL_COVERAGE_KEY_SETS credit depends on the manifest declaration, and the + * runtime omits the capability from START, so every connector here falls back + * to base-profile `hydrated_keys` on the wire. The two modes differ only in the + * manifest declaration: + * - Declaring connector: a hydrated key credits only with an accepted detail + * record for it. Activity `A` has one (`valuations` id `A`, the primary key + * of its own parent boundary), so activity commits. Holdings `H` has no + * record, so its hydration credits nothing and holdings stays put. + * - Nondeclaring connector: hydration credits by assertion, so `H` credits and + * both parents commit. + * Raw `empty_keys` and `unchanged_keys` are never emitted, so neither mode + * credits a key through them. Both backend suites run the same wire exchange + * and assert the same committed state. */ export async function runHydratedKeyCreditingScenario({ asUrl, @@ -254,24 +264,30 @@ export async function runHydratedKeyCreditingScenario({ connectorId: string; declaresKeySets: boolean; }): Promise { - // DETAIL_COVERAGE_KEY_SETS is new in Collection Profile 0.2.0, so a manifest - // that declares it declares that version (profile Section 3.1). - const manifest = { - ...multiParentManifest(connectorId), - protocol_capabilities: declaresKeySets ? ["DETAIL_COVERAGE_KEY_SETS"] : [], - protocol_version: declaresKeySets ? "0.2.0" : "0.1.0", - }; + // DETAIL_COVERAGE_KEY_SETS is an extension of the connector-protocol package, + // not a canonical profile capability. The runtime's manifest validation + // requires protocol_version 0.2.0 for a manifest that declares it, so a + // declaring manifest uses that version. + const manifest = withFixtureSourceDeclaration( + { + ...multiParentManifest(connectorId), + protocol_capabilities: declaresKeySets ? ["DETAIL_COVERAGE_KEY_SETS"] : [], + protocol_version: declaresKeySets ? "0.2.0" : "0.1.0", + }, + { sourceId: `https://sources.example/${connectorId}` } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-checkpoint-profile-hydrated-credit-")); const connectorPath = writeConnectorStub( tmpDir, ` process.stdout.write(JSON.stringify({ type: 'RECORD', stream: 'holdings', key: 'H', data: { id: 'H' }, emitted_at: new Date().toISOString() }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'RECORD', stream: 'activity', key: 'A', data: { id: 'A' }, emitted_at: new Date().toISOString() }) + '\\n'); + process.stdout.write(JSON.stringify({ type: 'RECORD', stream: 'valuations', key: 'A', data: { id: 'A' }, emitted_at: new Date().toISOString() }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'DETAIL_COVERAGE', reference_only: true, stream: 'valuations', state_stream: 'holdings', required_keys: ['H'], hydrated_keys: ['H'] }) + '\\n'); - process.stdout.write(JSON.stringify({ type: 'DETAIL_COVERAGE', reference_only: true, stream: 'valuations', state_stream: 'activity', required_keys: ['A'], hydrated_keys: [], empty_keys: ['A'] }) + '\\n'); + process.stdout.write(JSON.stringify({ type: 'DETAIL_COVERAGE', reference_only: true, stream: 'valuations', state_stream: 'activity', required_keys: ['A'], hydrated_keys: ['A'] }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'STATE', stream: 'holdings', cursor: { page: 'holdings-1' } }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'STATE', stream: 'activity', cursor: { page: 'activity-1' } }) + '\\n'); - process.stdout.write(JSON.stringify({ type: 'DONE', status: 'succeeded', records_emitted: 2 }) + '\\n'); + process.stdout.write(JSON.stringify({ type: 'DONE', status: 'succeeded', records_emitted: 3 }) + '\\n'); rl.close(); process.exit(0); ` @@ -301,7 +317,17 @@ export async function runHydratedKeyCreditingScenario({ assert.equal(result.status, "succeeded"); assert.equal(result.checkpoint_summary?.state_streams_staged, 2); - assert.equal(result.checkpoint_summary?.state_streams_committed, declaresKeySets ? 1 : 2); + // The runtime omits DETAIL_COVERAGE_KEY_SETS from START, so both modes send + // the same base-fallback hydration. Only the manifest declaration differs. + // A declaring connector keeps its accepted-record requirement: activity is + // proven by its valuations record, holdings' bare hydration is not. + assert.equal( + result.checkpoint_summary?.state_streams_committed, + declaresKeySets ? 1 : 2, + declaresKeySets + ? "a declaring connector commits only the parent whose hydrated key has an accepted record" + : "a nondeclaring connector's hydrated declarations credit by assertion, so both parents commit" + ); const stateResp = await fetch(`${rsUrl}/v1/state/${encodeURIComponent(manifest.connector_id)}`, { headers: { Authorization: `Bearer ${ownerToken}` }, @@ -328,39 +354,42 @@ export async function runHydratedKeyCreditingScenario({ // ----------------------------------------------------------------------- export function staticParentManifest(connectorId: string) { - return { - capabilities: { human_interaction: [] }, - connector_id: connectorId, - display_name: connectorId, - manifest_uri: `https://sources.example/${connectorId}`, - protocol_version: "0.1.0", - streams: [ - { - name: "messages", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - name: "unrelated_channel", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - coverage_strategy: "checkpoint_window", - name: "reactions", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - state_stream: "messages", - }, - ], - version: "0.1.0", - }; + return withFixtureSourceDeclaration( + { + capabilities: { human_interaction: [] }, + connector_id: connectorId, + display_name: connectorId, + manifest_uri: `https://sources.example/${connectorId}`, + protocol_version: "0.1.0", + streams: [ + { + name: "messages", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + name: "unrelated_channel", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + coverage_strategy: "checkpoint_window", + name: "reactions", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + state_stream: "messages", + }, + ], + version: "0.1.0", + }, + { sourceId: `https://sources.example/${connectorId}` } + ); } /** @@ -459,39 +488,42 @@ export async function runStaticParentEmitsCoverageScenario({ } export function undeclaredParentManifest(connectorId: string) { - return { - capabilities: { human_interaction: [] }, - connector_id: connectorId, - display_name: connectorId, - manifest_uri: `https://sources.example/${connectorId}`, - protocol_version: "0.1.0", - streams: [ - { - name: "messages", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - name: "unrelated_parent", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - coverage_strategy: "parent_detail_accounting", - name: "attachments", - parent_streams: ["messages"], - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "0.1.0", - }; + return withFixtureSourceDeclaration( + { + capabilities: { human_interaction: [] }, + connector_id: connectorId, + display_name: connectorId, + manifest_uri: `https://sources.example/${connectorId}`, + protocol_version: "0.1.0", + streams: [ + { + name: "messages", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + name: "unrelated_parent", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + coverage_strategy: "parent_detail_accounting", + name: "attachments", + parent_streams: ["messages"], + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "0.1.0", + }, + { sourceId: `https://sources.example/${connectorId}` } + ); } /** @@ -572,39 +604,42 @@ export async function runUndeclaredParentScenario({ } export function subsetParentManifest(connectorId: string) { - return { - capabilities: { human_interaction: [] }, - connector_id: connectorId, - display_name: connectorId, - manifest_uri: `https://sources.example/${connectorId}`, - protocol_version: "0.1.0", - streams: [ - { - name: "a", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - name: "b", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - coverage_strategy: "parent_detail_accounting", - name: "detail", - parent_streams: ["a", "b"], - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "0.1.0", - }; + return withFixtureSourceDeclaration( + { + capabilities: { human_interaction: [] }, + connector_id: connectorId, + display_name: connectorId, + manifest_uri: `https://sources.example/${connectorId}`, + protocol_version: "0.1.0", + streams: [ + { + name: "a", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + name: "b", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + coverage_strategy: "parent_detail_accounting", + name: "detail", + parent_streams: ["a", "b"], + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "0.1.0", + }, + { sourceId: `https://sources.example/${connectorId}` } + ); } /** @@ -637,7 +672,7 @@ export async function runSubsetParentCoverageScenario({ ` process.stdout.write(JSON.stringify({ type: 'RECORD', stream: 'a', key: 'a-1', data: { id: 'a-1' }, emitted_at: new Date().toISOString() }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'RECORD', stream: 'b', key: 'b-1', data: { id: 'b-1' }, emitted_at: new Date().toISOString() }) + '\\n'); - process.stdout.write(JSON.stringify({ type: 'DETAIL_COVERAGE', reference_only: true, stream: 'detail', state_stream: 'a', required_keys: ['a-1'], hydrated_keys: [], empty_keys: ['a-1'] }) + '\\n'); + process.stdout.write(JSON.stringify({ type: 'DETAIL_COVERAGE', reference_only: true, stream: 'detail', state_stream: 'a', required_keys: ['a-1'], hydrated_keys: ['a-1'] }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'STATE', stream: 'a', cursor: { page: 'a-1' } }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'STATE', stream: 'b', cursor: { page: 'b-1' } }) + '\\n'); process.stdout.write(JSON.stringify({ type: 'DONE', status: 'succeeded', records_emitted: 2 }) + '\\n'); diff --git a/reference-implementation/test/helpers/connector-state-scheduler-conformance.ts b/reference-implementation/test/helpers/connector-state-scheduler-conformance.ts index de8f96c4c..7faa10e10 100644 --- a/reference-implementation/test/helpers/connector-state-scheduler-conformance.ts +++ b/reference-implementation/test/helpers/connector-state-scheduler-conformance.ts @@ -246,8 +246,8 @@ interface SchedulerDriver { type TestFn = (name: string, fn: () => Promise) => void; -export const CONNECTOR_A = "https://test.pdpp.dev/connectors/conformance-a"; -export const CONNECTOR_B = "https://test.pdpp.dev/connectors/conformance-b"; +export const CONNECTOR_A = "conformance-a"; +export const CONNECTOR_B = "conformance-b"; export const STREAM_X = "stream_x"; export const STREAM_Y = "stream_y"; diff --git a/reference-implementation/test/helpers/fixture-source-declaration.ts b/reference-implementation/test/helpers/fixture-source-declaration.ts index 98d9d12c7..aca0d97c4 100644 --- a/reference-implementation/test/helpers/fixture-source-declaration.ts +++ b/reference-implementation/test/helpers/fixture-source-declaration.ts @@ -7,11 +7,10 @@ const FIXTURE_PUBLISHER = "https://publishers.pdpp.test/fixtures"; /** * Test-only. Gives a fixture manifest an explicit, validated source declaration computed from its own - * final content, so registration uses the fixture's authored source identity instead of inferring one from - * the manifest's `connector_id` or `manifest_uri`. The default identity is the fixture's own - * (`https://fixtures.pdpp.test/sources/`), not a production identity; pass `sourceId` to pin another. - * Always recomputed from the content it is given, so a manifest derived by spreading another fixture never - * keeps a declaration for content it no longer has. + * final content, so a first registration of a fixture key with no reviewed legacy mapping goes through the + * Core route. The source identity is the fixture's own (`https://fixtures.pdpp.test/sources/`), not a + * production identity. Always recomputed from the content it is given, so a manifest derived by spreading + * another fixture never keeps a declaration for content it no longer has. */ export function withFixtureSourceDeclaration(manifest: T, options: { sourceId?: string } = {}): T { const content = Object.fromEntries( diff --git a/reference-implementation/test/helpers/legacy-upgrade.ts b/reference-implementation/test/helpers/legacy-upgrade.ts index 33882defe..dbf7eea5b 100644 --- a/reference-implementation/test/helpers/legacy-upgrade.ts +++ b/reference-implementation/test/helpers/legacy-upgrade.ts @@ -8,7 +8,7 @@ * `package/manifests/*.json` bundle entries. An upgraded database holds them * the way c82718d4's registerConnector stored them: the row id is the * manifest's `connector_key`, and the JSON went through - * `normalizeConnectorManifestForStorage`. Only the OCI fetch is replaced in + * the pre-retention declaration projection (`legacyStoredManifest`). Only the OCI fetch is replaced in * the scenarios; the install service, owner routes, registration, state, * ingest and read paths are the production ones. */ @@ -19,7 +19,7 @@ import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; -import { normalizeConnectorManifestForStorage, registerConnector } from "../../server/auth.ts"; +import { registerConnector } from "../../server/auth.ts"; import { type ConnectorCatalogEntry, createConnectorInstallService, @@ -28,6 +28,7 @@ import { import { getDb } from "../../server/db.ts"; import { startServer } from "../../server/index.ts"; import { postgresQuery } from "../../server/postgres-storage.ts"; +import { snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest } from "../../server/source-declaration-legacy-collection.ts"; import { createPostgresConnectorInstanceStore, createSqliteConnectorInstanceStore, @@ -52,9 +53,28 @@ export function legacyManifest(key: string): Record { return JSON.parse(readFileSync(join(HERE, "..", "fixtures", "legacy-upgrade", `${key}.json`), "utf8")); } -/** The manifest JSON c82718d4 left in `connectors.manifest`. */ +/** Publisher the pre-retention projection stamped on every legacy declaration (server/auth.ts). */ +const LEGACY_PROJECTION_PUBLISHER_ID = "https://pdpp.dev/reference-implementation"; + +/** + * The manifest JSON c82718d4 left in `connectors.manifest`: the O-era manifest plus the + * declaration base projected from its provenance (`source.id = manifest_uri`). Built here + * rather than through the current normalizer, whose fresh-path rule refuses keys with no + * reviewed mapping, because an upgraded row was written by the earlier projection. + */ export function legacyStoredManifest(key: string): Record { - return normalizeConnectorManifestForStorage(legacyManifest(key)).storedManifest; + const manifest = legacyManifest(key); + // The O-era projection stored connector_id/connector_key as the canonical key and kept the URL as manifest_uri. + const stored: Record = { ...manifest, connector_id: key, connector_key: key }; + const provenance = typeof manifest.manifest_uri === "string" ? manifest.manifest_uri : String(manifest.connector_id); + return { + ...stored, + source_declaration: snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest(stored, { + connectorImplementationId: provenance, + publisherId: LEGACY_PROJECTION_PUBLISHER_ID, + sourceId: provenance, + }), + }; } type InstanceStore = ReturnType | ReturnType; diff --git a/reference-implementation/test/helpers/production-store-connector-state-scheduler-driver.ts b/reference-implementation/test/helpers/production-store-connector-state-scheduler-driver.ts index ddf73f4ad..0cb9c7597 100644 --- a/reference-implementation/test/helpers/production-store-connector-state-scheduler-driver.ts +++ b/reference-implementation/test/helpers/production-store-connector-state-scheduler-driver.ts @@ -83,8 +83,9 @@ interface ActiveRunRecord { const HARNESS_MANIFESTS = [ { connector_id: CONNECTOR_A, + connector_key: CONNECTOR_A, display_name: "Conformance Connector A", - manifest_uri: `https://sources.example/${CONNECTOR_A}`, + manifest_uri: "https://sources.example/https://test.pdpp.dev/connectors/conformance-a", protocol_version: "0.1.0", runtime_requirements: { bindings: { network: { required: true } } }, streams: [ @@ -107,8 +108,9 @@ const HARNESS_MANIFESTS = [ }, { connector_id: CONNECTOR_B, + connector_key: CONNECTOR_B, display_name: "Conformance Connector B", - manifest_uri: `https://sources.example/${CONNECTOR_B}`, + manifest_uri: "https://sources.example/https://test.pdpp.dev/connectors/conformance-b", protocol_version: "0.1.0", runtime_requirements: { bindings: { network: { required: true } } }, streams: [ @@ -274,7 +276,14 @@ export function createProductionStoreConnectorStateSchedulerDriver() { initDb(); for (const manifest of HARNESS_MANIFESTS) { // biome-ignore lint/performance/noAwaitInLoops: ordered test setup is intentionally sequential - await registerConnector(withFixtureSourceDeclaration(manifest, { sourceId: manifest.manifest_uri })); + await registerConnector( + withFixtureSourceDeclaration(manifest, { + sourceId: + manifest.connector_id === CONNECTOR_A + ? "https://sources.example/https://test.pdpp.dev/connectors/conformance-a" + : "https://sources.example/https://test.pdpp.dev/connectors/conformance-b", + }) + ); } stateStore = createSqliteConnectorStateStore(); schedulerStore = createSqliteSchedulerStore(); diff --git a/reference-implementation/test/helpers/ri-zero-connector-knowledge-data-load-scan.ts b/reference-implementation/test/helpers/ri-zero-connector-knowledge-data-load-scan.ts index 17c2a2ea3..c32f14e4a 100644 --- a/reference-implementation/test/helpers/ri-zero-connector-knowledge-data-load-scan.ts +++ b/reference-implementation/test/helpers/ri-zero-connector-knowledge-data-load-scan.ts @@ -150,7 +150,7 @@ const MANIFEST_ROOTS = [ * external state read at request time, never RI-committed source or * RI-committed JSON, so it was never a candidate for this allowlist either * (see `reference-implementation/server/version-disposition.ts`'s own module - * doc comment). The canonical Collection Profile observation schema is + * doc comment). The canonical Collection Profile observation schema is now * registered for its exact runtime loader. It describes protocol shapes, not * provider policy; the content check still rejects connector identity in it. * Additional entries require the same review as the generic read sites below. @@ -159,6 +159,9 @@ const SANCTIONED_POLICY_RESOURCES: ReadonlyMap> = ne ["reference-implementation/runtime/observations.ts", new Set([ "reference-implementation/runtime/observation.schema.json", ])], + ["reference-implementation/runtime/binding-declarations.ts", new Set([ + "reference-implementation/runtime/connector-manifest.schema.json", + ])], ]); const POLYFILL_MANIFEST_READ_SITE = 'reference-implementation/server/polyfill-manifest-reconcile.ts::readManifestJson::readFile(path, "utf8")'; diff --git a/reference-implementation/test/helpers/sqlite-connector-state-scheduler-driver.ts b/reference-implementation/test/helpers/sqlite-connector-state-scheduler-driver.ts index b49d73373..75f4aa7c6 100644 --- a/reference-implementation/test/helpers/sqlite-connector-state-scheduler-driver.ts +++ b/reference-implementation/test/helpers/sqlite-connector-state-scheduler-driver.ts @@ -79,8 +79,9 @@ interface ActiveDbRow { const HARNESS_MANIFESTS = [ { connector_id: CONNECTOR_A, + connector_key: CONNECTOR_A, display_name: "Conformance Connector A", - manifest_uri: `https://sources.example/${CONNECTOR_A}`, + manifest_uri: "https://sources.example/https://test.pdpp.dev/connectors/conformance-a", protocol_version: "0.1.0", runtime_requirements: { bindings: { network: { required: true } } }, streams: [ @@ -103,8 +104,9 @@ const HARNESS_MANIFESTS = [ }, { connector_id: CONNECTOR_B, + connector_key: CONNECTOR_B, display_name: "Conformance Connector B", - manifest_uri: `https://sources.example/${CONNECTOR_B}`, + manifest_uri: "https://sources.example/https://test.pdpp.dev/connectors/conformance-b", protocol_version: "0.1.0", runtime_requirements: { bindings: { network: { required: true } } }, streams: [ @@ -268,7 +270,14 @@ export function createSqliteConnectorStateSchedulerDriver() { initDb(); for (const manifest of HARNESS_MANIFESTS) { // biome-ignore lint/performance/noAwaitInLoops: ordered setup is intentionally sequential because each iteration advances shared test state. - await registerConnector(withFixtureSourceDeclaration(manifest, { sourceId: manifest.manifest_uri })); + await registerConnector( + withFixtureSourceDeclaration(manifest, { + sourceId: + manifest.connector_id === CONNECTOR_A + ? "https://sources.example/https://test.pdpp.dev/connectors/conformance-a" + : "https://sources.example/https://test.pdpp.dev/connectors/conformance-b", + }) + ); } // Establish this incarnation's boot epoch before any run is // announced: the spine refuses an unstamped `run.started`, and the diff --git a/reference-implementation/test/introspection-manifest-fail-closed.test.ts b/reference-implementation/test/introspection-manifest-fail-closed.test.ts index 73731d6fd..462e3b98e 100644 --- a/reference-implementation/test/introspection-manifest-fail-closed.test.ts +++ b/reference-implementation/test/introspection-manifest-fail-closed.test.ts @@ -40,6 +40,7 @@ const MANIFEST = { primary_key: ["id"], schema: { properties: { id: { type: "string" } }, type: "object" }, selection: { fields: true, resources: true }, + semantics: "mutable_state", }, ], }, @@ -49,6 +50,7 @@ const MANIFEST = { primary_key: ["id"], schema: { properties: { id: { type: "string" } }, type: "object" }, selection: { fields: true, resources: true }, + semantics: "mutable_state", }, ], version: "1.0.0", diff --git a/reference-implementation/test/legacy-upgrade-preservation.test.ts b/reference-implementation/test/legacy-upgrade-preservation.test.ts index 32e5f1f17..172ea2a42 100644 --- a/reference-implementation/test/legacy-upgrade-preservation.test.ts +++ b/reference-implementation/test/legacy-upgrade-preservation.test.ts @@ -64,11 +64,10 @@ test("a developer-local source preserves the underscore legacy id apple_contacts const dataDir = join(dir, "data"); mkdirSync(join(sourceRoot, "profile"), { recursive: true }); mkdirSync(join(sourceRoot, "dist"), { recursive: true }); - // The profile is the O-era manifest unchanged (connector_key apple_contacts). - writeFileSync( - join(sourceRoot, "profile", "collection-profile.json"), - JSON.stringify(legacyManifest("apple_contacts")) - ); + // The profile is the shipped O-era manifest, unmodified. This is an upgrade: the retained historical + // registration seeded below already carries the source identity, and add/select must not change it. + // A fresh, unmapped local source with the same key would still need an explicit source_declaration. + writeFileSync(join(sourceRoot, "profile", "collection-profile.json"), JSON.stringify(legacyManifest("apple_contacts"))); writeFileSync(join(sourceRoot, "dist", "collection-profile.mjs"), "export default {};\n"); const localStore = createFileLocalConnectorSourceStore(dataDir); const installStore = createFileConnectorInstallStore(dataDir); diff --git a/reference-implementation/test/manifest-runtime-requirements.test.ts b/reference-implementation/test/manifest-runtime-requirements.test.ts index 0c388c7be..e8167495d 100644 --- a/reference-implementation/test/manifest-runtime-requirements.test.ts +++ b/reference-implementation/test/manifest-runtime-requirements.test.ts @@ -11,7 +11,13 @@ import assert from "node:assert/strict"; import test from "node:test"; +import { BindingDeclarationError } from "../runtime/binding-declarations.ts"; +import { getConnectorManifest } from "../server/auth.ts"; +import { getDb } from "../server/db.ts"; import { startServer } from "../server/index.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; + +const DUPLICATE_OBJECT_KEY = /repeats object key/; type StartedServer = Awaited>; @@ -76,8 +82,9 @@ async function registerConnectorManifest(asUrl: string, manifest: unknown): Prom } function makeManifest(runtimeRequirements: unknown) { - return { - connector_id: "https://registry.pdpp.test/connectors/runtime-requirements-fixture", + return withFixtureSourceDeclaration({ + connector_id: "runtime-requirements-fixture", + connector_key: "runtime-requirements-fixture", display_name: "Runtime requirements fixture", protocol_version: "0.1.0", runtime_requirements: runtimeRequirements, @@ -99,7 +106,7 @@ function makeManifest(runtimeRequirements: unknown) { }, ], version: "0.1.0", - }; + }); } test("valid browser runtime binding is accepted", async () => { @@ -122,7 +129,7 @@ test("unsupported runtime binding is rejected", async () => { assert.ok(hasErrorBody(body)); assert.equal(body.error.code, "invalid_request"); // biome-ignore lint/performance/useTopLevelRegex: localized test assertion preserves its explicit contract. - assert.match(body.error.message, /unsupported keys: toaster/u); + assert.match(body.error.message, /toaster must have required property 'kind'/u); }); }); @@ -136,6 +143,77 @@ test("runtime binding required flag must be boolean", async () => { assert.ok(hasErrorBody(body)); assert.equal(body.error.code, "invalid_request"); // biome-ignore lint/performance/useTopLevelRegex: localized test assertion preserves its explicit contract. - assert.match(body.error.message, /browser\.required must be a boolean/u); + assert.match(body.error.message, /browser\.required must be boolean/u); + }); +}); + +const RAW_KEY_CONNECTOR = "raw-binding-keys"; + +/** + * Valid registration for the raw-key tests: the operational key is a slug and + * the source identity is a distinct absolute URI. A URI-only `connector_id` + * registers with a source-declaration error (sourceId must be an absolute URI) + * or stores a URL connector_key that the read path rejects, so neither is a + * usable control. + */ +function makeRawBindingKeysManifest() { + return withFixtureSourceDeclaration({ + ...makeManifest({ bindings: { network: { required: false } } }), + connector_id: RAW_KEY_CONNECTOR, + connector_key: RAW_KEY_CONNECTOR, + manifest_uri: `https://registry.pdpp.test/connectors/${RAW_KEY_CONNECTOR}`, + }); +} + +test("registration rejects duplicate binding keys in the raw HTTP body before persistence", async () => { + await withHarness(async ({ asUrl }) => { + const manifest = makeRawBindingKeysManifest(); + const valid = JSON.stringify(manifest); + const duplicate = valid.replace( + '"network":{"required":false}', + '"network":{"required":true},"network":{"required":false}' + ); + assert.notEqual(duplicate, valid, "the request must retain both raw object keys"); + const response = await fetch(`${asUrl}/connectors`, { + body: duplicate, + headers: { "Content-Type": "application/json" }, + method: "POST", + }); + assert.equal(response.status, 400); + assert.match(await response.text(), DUPLICATE_OBJECT_KEY); + assert.equal(await getConnectorManifest(manifest.connector_id), null, "a rejected declaration is not persisted"); + const baseline = await registerConnectorManifest(asUrl, manifest); + assert.equal(baseline.status, 201, "the same declaration without duplicate keys must register"); + }); +}); + +test("stored manifest reads reject duplicate raw keys before JSON parsing loses them", async () => { + await withHarness(async ({ asUrl }) => { + const manifest = makeRawBindingKeysManifest(); + const registered = await registerConnectorManifest(asUrl, manifest); + assert.equal(registered.status, 201); + try { + assert.ok(await getConnectorManifest(manifest.connector_id)); + } catch (error: unknown) { + assert.fail( + `valid stored fixture must load: ${error instanceof Error ? String(error.cause ?? error) : String(error)}` + ); + } + const valid = JSON.stringify(manifest); + const duplicate = valid.replace( + '"network":{"required":false}', + '"network":{"required":true},"network":{"required":false}' + ); + assert.notEqual(duplicate, valid); + getDb().prepare("UPDATE connectors SET manifest = ? WHERE connector_id = ?").run(duplicate, manifest.connector_id); + await assert.rejects( + () => getConnectorManifest(manifest.connector_id), + (error: unknown) => { + assert.ok(error instanceof Error); + assert.ok(error.cause instanceof BindingDeclarationError); + assert.equal(error.cause.code, "duplicate_json_key"); + return true; + } + ); }); }); diff --git a/reference-implementation/test/manifest-validation-runtime-refresh.test.ts b/reference-implementation/test/manifest-validation-runtime-refresh.test.ts index eb16a9c26..5e988f638 100644 --- a/reference-implementation/test/manifest-validation-runtime-refresh.test.ts +++ b/reference-implementation/test/manifest-validation-runtime-refresh.test.ts @@ -8,10 +8,8 @@ // bindings + external tools and its refresh policy; a loosened check silently // admits a malformed manifest at registration. // -// NOTE (pinned quirk): validateRuntimeRequirements returns EARLY when -// `runtime_requirements.bindings` is absent, so `external_tools` is only validated -// when `bindings` is also declared. These tests include `bindings` for the -// external-tools cases and pin the early-return behavior explicitly. +// Binding and external-tool declarations are validated independently, including +// external tools in a runtime_requirements object without a bindings member. // // Mutation surface: // validateRuntimeRequirements -- object-shape guards, RUNTIME_REQUIREMENT_BINDINGS @@ -54,7 +52,7 @@ test("validateRuntimeRequirements: absent runtime_requirements / bindings is a p assert.doesNotThrow(() => validateRuntimeRequirements({}, CODE)); assert.doesNotThrow( () => validateRuntimeRequirements({ runtime_requirements: {} }, CODE), - "no bindings -> early return pass" + "an absent bindings member does not add a binding requirement" ); }); @@ -67,14 +65,14 @@ test("validateRuntimeRequirements: a recognized binding with a boolean required test("validateRuntimeRequirements: an unsupported binding key is rejected", () => { expectReject( () => validateRuntimeRequirements({ runtime_requirements: { bindings: { teleport: {} } } }, CODE), - "unsupported keys: teleport" + "teleport must have required property 'required'" ); }); test("validateRuntimeRequirements: a non-boolean `required` is rejected", () => { expectReject( () => validateRuntimeRequirements({ runtime_requirements: { bindings: { network: { required: "yes" } } } }, CODE), - "required must be a boolean" + "network.required must be boolean" ); }); @@ -82,28 +80,37 @@ test("validateRuntimeRequirements: non-object runtime_requirements or bindings i expectReject(() => validateRuntimeRequirements({ runtime_requirements: [] }, CODE), "must be an object"); expectReject( () => validateRuntimeRequirements({ runtime_requirements: { bindings: [] } }, CODE), - "bindings must be an object" + "bindings must be object" ); }); // --------------------------------------------------------------------------- -// validateRuntimeRequirements — external_tools (requires bindings present) +// validateRuntimeRequirements — external_tools (independent of bindings) // --------------------------------------------------------------------------- const withBindings = { bindings: { network: { required: true } } }; -test("validateRuntimeRequirements: external_tools is only validated when bindings is declared (pinned quirk)", () => { - // WITHOUT bindings, a clearly-invalid external_tools entry is NOT validated - // (early return) — this pins the current behavior so a refactor that reorders - // the checks is caught. - assert.doesNotThrow( +test("validateRuntimeRequirements: external_tools is validated even when bindings is absent", () => { + // Formerly an early return skipped external_tools when bindings was absent. + // The coupling is removed: the invalid entry is rejected either way. + expectReject( () => validateRuntimeRequirements({ runtime_requirements: { external_tools: [{ bogus: 1 }] } }, CODE), - "no bindings -> external_tools not reached" + "has unsupported keys: bogus" ); - // WITH bindings, the same invalid entry IS rejected. expectReject(() => validateRuntimeRequirements({ runtime_requirements: { ...withBindings, external_tools: [{ bogus: 1 }] } }, CODE) ); + // Control: a valid external_tools entry without bindings still passes. + assert.doesNotThrow(() => + validateRuntimeRequirements( + { + runtime_requirements: { + external_tools: [{ license: "MIT", name: "git", purpose: "vcs" }], + }, + }, + CODE + ) + ); }); test("validateRuntimeRequirements: external_tools requires name/license/purpose non-empty strings", () => { diff --git a/reference-implementation/test/owner-connection-intent.test.ts b/reference-implementation/test/owner-connection-intent.test.ts index a112c509e..483e9b152 100644 --- a/reference-implementation/test/owner-connection-intent.test.ts +++ b/reference-implementation/test/owner-connection-intent.test.ts @@ -44,6 +44,7 @@ import { COLLECTOR_PROTOCOL_VERSION } from "../server/collector-protocol.ts"; import { type ConnectorManifestLike, classifyConnectorIntentModality } from "../server/connection-setup-plan.ts"; import { canonicalConnectorKey } from "../server/connector-key.ts"; import { startServer } from "../server/index.ts"; +import { snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest } from "../server/source-declaration-legacy-collection.ts"; import { createSqliteConnectorInstanceStore } from "../server/stores/connector-instance-store.ts"; import { TEST_PRE_REGISTERED_PUBLIC_CLIENTS } from "./fixtures/demo-clients.ts"; @@ -285,27 +286,23 @@ function loadPackageManifest(name: string): PackageManifest { return entry.manifest as PackageManifest; } +/** + * Test-only explicit attribution. The declaration is the production legacy projection of the manifest's + * own content (streams keep only the Core members), not a copy of connector-only stream fields, which + * Core refuses. Source and publisher are fixture identities, never a production source. + */ function withExplicitTestSourceDeclaration(manifest: PackageManifest): PackageManifest { if (manifest.source_declaration && typeof manifest.source_declaration === "object") { return manifest; } const connectorKey = canonicalConnectorKey(manifest.connector_id) ?? manifest.connector_id; - const streams = Array.isArray(manifest.streams) - ? manifest.streams.map((stream) => ({ - ...stream, - ...(stream.semantics === "append" ? { semantics: "append_only" } : {}), - })) - : []; return { ...manifest, - source_declaration: { - declaration_version: `owner-connection-intent-test:${connectorKey}:v1`, - display: { name: typeof manifest.display_name === "string" ? manifest.display_name : connectorKey }, - protocol_version: manifest.protocol_version, - publisher: { id: "https://pdpp.dev/reference-implementation/tests" }, - source: { id: `https://sources.example/connectors/${encodeURIComponent(connectorKey)}`, kind: "connector" }, - streams, - }, + source_declaration: snapshotContentAddressedSourceDeclarationFromLegacyConnectorManifest(manifest, { + connectorImplementationId: `https://implementations.example/connectors/${encodeURIComponent(connectorKey)}`, + publisherId: "https://pdpp.dev/reference-implementation/tests", + sourceId: `https://sources.example/connectors/${encodeURIComponent(connectorKey)}`, + }), }; } diff --git a/reference-implementation/test/owner-connector-templates.test.ts b/reference-implementation/test/owner-connector-templates.test.ts index 641ad5bad..0da2fa29a 100644 --- a/reference-implementation/test/owner-connector-templates.test.ts +++ b/reference-implementation/test/owner-connector-templates.test.ts @@ -19,6 +19,7 @@ import { canonicalConnectorKey } from "../server/connector-key.ts"; import { startServer } from "../server/index.ts"; import { createSqliteConnectorInstanceStore } from "../server/stores/connector-instance-store.ts"; import { TEST_PRE_REGISTERED_PUBLIC_CLIENTS } from "./fixtures/demo-clients.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; const OWNER_SUBJECT_ID = "owner_local"; const NOW = "2026-06-01T00:00:00.000Z"; @@ -379,7 +380,7 @@ test("owner-template setup plan offers a catalog-installed browser connector by acmeManifest.display_name = "Acme Shop"; acmeManifest.setup = undefined; acmeManifest.capabilities = { ...asRecord(acmeManifest.capabilities), public_listing: { tier: "preview" } }; - await registerConnector(asUrl, acmeManifest); + await registerConnector(asUrl, withFixtureSourceDeclaration(acmeManifest)); const ownerToken = await issueOwnerToken(asUrl); const { status, body } = await fetchJson(`${rsUrl}/v1/owner/connector-templates`, { diff --git a/reference-implementation/test/ri-zero-connector-knowledge-conformance.test.ts b/reference-implementation/test/ri-zero-connector-knowledge-conformance.test.ts index 9217b0814..4245d27da 100644 --- a/reference-implementation/test/ri-zero-connector-knowledge-conformance.test.ts +++ b/reference-implementation/test/ri-zero-connector-knowledge-conformance.test.ts @@ -99,6 +99,37 @@ test("the pinned observation schema has one exact loader and cannot carry connec } }); +test("the pinned connector manifest schema has one exact loader and cannot carry connector identity", () => { + const fixtureRoot = mkdtempSync(join(tmpdir(), "ri-manifest-schema-resource-")); + const sourceRel = "reference-implementation/runtime/binding-declarations.ts"; + const sourceFile = join(fixtureRoot, sourceRel); + const schemaFile = join(dirname(sourceFile), "connector-manifest.schema.json"); + const source = + 'import manifestSchema from "./connector-manifest.schema.json" with { type: "json" };\nexport default manifestSchema;\n'; + try { + mkdirSync(dirname(sourceFile), { recursive: true }); + writeFileSync(sourceFile, source); + writeFileSync(schemaFile, JSON.stringify({ enum: ["browser", "network", "filesystem"] })); + assert.deepEqual(scanFileDataLoads(sourceFile, sourceRel, fixtureRoot), [], + "the canonical manifest schema is a generic resource, not connector-specific policy"); + + writeFileSync(schemaFile, JSON.stringify({ nested: { enum: ["synthetic_provider_identity"] } })); + const injected = scanFileDataLoads(sourceFile, sourceRel, fixtureRoot, new Set(["synthetic_provider_identity"])); + assert.ok(injected.some((violation) => violation.rule === "hardcoded-connector-literal-in-ri-owned-json"), + "sanctioning the manifest schema path must retain the provider-content check"); + + writeFileSync(schemaFile, JSON.stringify({ enum: ["browser"] })); + const otherRel = "reference-implementation/runtime/other-manifest-loader.ts"; + const otherFile = join(fixtureRoot, otherRel); + writeFileSync(otherFile, source); + assert.ok(scanFileDataLoads(otherFile, otherRel, fixtureRoot) + .some((violation) => violation.rule === "unsanctioned-policy-resource-path"), + "a different loader does not inherit the manifest schema exception"); + } finally { + rmSync(fixtureRoot, { force: true, recursive: true }); + } +}); + test("manifest-derived connector identity set is non-trivial (sanity check on the derivation itself)", () => { const keys = manifestDerivedConnectorKeys({ repoRoot }); // A hardcoded lower-bound count would itself be exactly the kind of diff --git a/reference-implementation/test/run-interaction-control.test.ts b/reference-implementation/test/run-interaction-control.test.ts index 886d86c96..e22e6b14e 100644 --- a/reference-implementation/test/run-interaction-control.test.ts +++ b/reference-implementation/test/run-interaction-control.test.ts @@ -28,6 +28,19 @@ import { canonicalConnectorKey } from "../server/connector-key.ts"; import { startServer } from "../server/index.ts"; import { resolveCredentialFreeFixtureRunEnv } from "./helpers/credential-free-run-fixture.ts"; +// Test scaffolding only. The shipped seed declares `credentials` alone, while these +// tests drive connectors that also emit otp and manual_action. Declaring them here +// keeps the shipped seed unchanged and the undeclared-kind refusal intact elsewhere. +function withTestHumanInteraction>(manifest: T): T { + return { + ...manifest, + capabilities: { + ...((manifest.capabilities as Record | undefined) ?? {}), + human_interaction: ["credentials", "otp", "manual_action"], + }, + } as T; +} + const REGEXP_1 = /text\/html/; const REGEXP_2 = /Pending interaction/; const REGEXP_3 = /Send success/; @@ -264,8 +277,8 @@ async function withHarness( rsPort: 0, })) as ClosableServer; const asUrl = `http://localhost:${server.asPort}`; - const spotifyManifest: SpotifyManifest = JSON.parse( - readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") + const spotifyManifest: SpotifyManifest = withTestHumanInteraction( + JSON.parse(readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8")) ); try { await registerConnector(asUrl, spotifyManifest); @@ -286,8 +299,8 @@ async function withCustomHarness(connectorPath: string, fn: (ctx: HarnessContext rsPort: 0, })) as ClosableServer; const asUrl = `http://localhost:${server.asPort}`; - const spotifyManifest: SpotifyManifest = JSON.parse( - readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") + const spotifyManifest: SpotifyManifest = withTestHumanInteraction( + JSON.parse(readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8")) ); try { await registerConnector(asUrl, spotifyManifest); diff --git a/reference-implementation/test/run-interaction-stream-routes.test.ts b/reference-implementation/test/run-interaction-stream-routes.test.ts index 36cf67755..697310d12 100644 --- a/reference-implementation/test/run-interaction-stream-routes.test.ts +++ b/reference-implementation/test/run-interaction-stream-routes.test.ts @@ -48,6 +48,20 @@ import { normalizeReferenceWireViewportPayload } from "../server/streaming/proto import { createStreamingSessionStore } from "../server/streaming/sessions.ts"; import { resolveCredentialFreeFixtureRunEnv } from "./helpers/credential-free-run-fixture.ts"; +// Test scaffolding only. The shipped seed declares `credentials` alone, while these +// tests drive connectors that also emit otp and manual_action. Declaring them here +// keeps the shipped seed unchanged and the undeclared-kind refusal intact elsewhere. +function withTestHumanInteraction>(manifest: T): T { + return { + ...manifest, + capabilities: { + ...((manifest.capabilities as Record | undefined) ?? {}), + human_interaction: ["credentials", "otp", "manual_action"], + }, + } as T; +} + + const REGEXP_1 = /retry/i; const REGEXP_2 = /retry/i; const REGEXP_3 = /text\/event-stream/; @@ -895,9 +909,7 @@ async function withHarness(options: HarnessOptions | null, fn: (ctx: HarnessCont }); const asUrl = `http://localhost:${server.asPort}`; const manifestName = harnessOptions.manifestName || "spotify"; - const spotifyManifest: SpotifyManifest = JSON.parse( - readFileSync(join(REFERENCE_IMPL_DIR, `fixtures/seed-manifests/${manifestName}.json`), "utf8") - ); + const spotifyManifest: SpotifyManifest = withTestHumanInteraction(JSON.parse(readFileSync(join(REFERENCE_IMPL_DIR, `fixtures/seed-manifests/${manifestName}.json`), "utf8"))); try { await registerConnector(asUrl, spotifyManifest); await fn({ asUrl, companions, server, spotifyManifest }); @@ -3536,9 +3548,7 @@ test("SSE delivers multiple frames and acks each, even when ack rejects", async }); try { const asUrl = `http://localhost:${server.asPort}`; - const spotifyManifest: SpotifyManifest = JSON.parse( - readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") - ); + const spotifyManifest: SpotifyManifest = withTestHumanInteraction(JSON.parse(readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8"))); await registerConnector(asUrl, spotifyManifest); const started = await startRun(asUrl, spotifyManifest.connector_id); const pending = await waitForPendingInteraction(asUrl, started.run_id); @@ -3644,7 +3654,7 @@ test("mint fails closed with 503 streaming_companion_unavailable when no compani }); try { const asUrl = `http://localhost:${server.asPort}`; - const spotifyManifest = JSON.parse(readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8")); + const spotifyManifest = withTestHumanInteraction(JSON.parse(readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8"))); await registerConnector(asUrl, spotifyManifest); const started = await startRun(asUrl, spotifyManifest.connector_id); const pending = await waitForPendingInteraction(asUrl, started.run_id); diff --git a/reference-implementation/test/runtime-requirements-validation.test.ts b/reference-implementation/test/runtime-requirements-validation.test.ts index 653ae3669..722cecee4 100644 --- a/reference-implementation/test/runtime-requirements-validation.test.ts +++ b/reference-implementation/test/runtime-requirements-validation.test.ts @@ -74,7 +74,7 @@ test("validateRuntimeRequirements: returns when runtime_requirements is absent", test("validateRuntimeRequirements: accepts valid bindings", () => { assert.equal( validateRuntimeRequirements( - { runtime_requirements: { bindings: { browser: { required: true }, network: {} } } }, + { runtime_requirements: { bindings: { browser: { required: true }, network: { required: false } } } }, CODE ), undefined @@ -95,18 +95,18 @@ test("validateRuntimeRequirements: accepts a valid external tool (with bindings test("validateRuntimeRequirements: rejects non-object requirements or bindings", () => { assertRejects({ runtime_requirements: "x" }, "runtime_requirements must be an object"); - assertRejects({ runtime_requirements: { bindings: "x" } }, "runtime_requirements.bindings must be an object"); + assertRejects({ runtime_requirements: { bindings: "x" } }, "runtime_requirements.bindings must be object"); }); test("validateRuntimeRequirements: rejects an unsupported binding key", () => { - assertRejects({ runtime_requirements: { bindings: { gpu: {} } } }, "bindings has unsupported keys: gpu"); + assertRejects({ runtime_requirements: { bindings: { gpu: {} } } }, "gpu must have required property 'required'"); }); test("validateRuntimeRequirements: rejects a non-object binding value and a non-boolean required", () => { - assertRejects({ runtime_requirements: { bindings: { browser: "x" } } }, "bindings.browser must be an object"); + assertRejects({ runtime_requirements: { bindings: { browser: "x" } } }, "bindings.browser must be object"); assertRejects( { runtime_requirements: { bindings: { browser: { required: "yes" } } } }, - "bindings.browser.required must be a boolean" + "bindings.browser.required must be boolean" ); }); @@ -269,13 +269,11 @@ test("validateRuntimeRequirements: rejects an unsupported local_paths.paths entr // --- short-circuit contract ------------------------------------------------- -test("validateRuntimeRequirements: external_tools is NOT validated when bindings is absent (early return)", () => { - // No bindings => the function returns before ever reaching external_tools, so - // an otherwise-invalid external_tools value is not rejected here. - assert.equal( - validateRuntimeRequirements({ runtime_requirements: { external_tools: "not-an-array" } }, CODE), - undefined, - "absent bindings short-circuits external_tools validation" +test("validateRuntimeRequirements: external_tools is validated when bindings is absent", () => { + // Formerly an early return skipped external_tools when bindings was absent. + assertRejects( + { runtime_requirements: { external_tools: "not-an-array" } }, + "runtime_requirements.external_tools must be an array" ); }); diff --git a/reference-implementation/test/runtime-stream-collection-failed-commit.test.ts b/reference-implementation/test/runtime-stream-collection-failed-commit.test.ts index d9ef97859..e23b55beb 100644 --- a/reference-implementation/test/runtime-stream-collection-failed-commit.test.ts +++ b/reference-implementation/test/runtime-stream-collection-failed-commit.test.ts @@ -12,6 +12,7 @@ import { runConnector } from "../runtime/index.ts"; import { startServer as startServerUntyped } from "../server/index.ts"; import { createRequestConnectorInstanceStore } from "../server/request-store-factories.ts"; import { admitOwnerRunConnection } from "../server/stores/connector-instance-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; // Regression coverage for the live runtime path (reference-implementation/runtime/index.ts) // used by every real connector run, as distinct from the standalone @@ -141,29 +142,32 @@ function fakeAdmitRunConnection( // shape that discriminates it: one sibling that fully succeeds, one stream // that reports its own certified failure. function testManifest(connectorId: string) { - return { - connector_id: connectorId, - display_name: "Runtime stream isolation test", - manifest_uri: `https://registry.pdpp.dev/connectors/${connectorId}`, - protocol_version: "0.1.0", - streams: [ - { - name: "sibling_a", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - name: "sibling_b", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "1.0.0", - }; + return withFixtureSourceDeclaration( + { + connector_id: connectorId, + display_name: "Runtime stream isolation test", + manifest_uri: `https://registry.pdpp.dev/connectors/${connectorId}`, + protocol_version: "0.1.0", + streams: [ + { + name: "sibling_a", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + name: "sibling_b", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "1.0.0", + }, + { sourceId: `https://registry.pdpp.dev/connectors/${connectorId}` } + ); } type RuntimeManifest = NonNullable[0]["manifest"]> & { @@ -173,18 +177,21 @@ type RuntimeManifest = NonNullable[0]["manifest" type RuntimeManifestStream = NonNullable[number]; function completeManifest(connectorId: string, streams: RuntimeManifestStream[]): RuntimeManifest { - return { - connector_id: connectorId, - display_name: "Runtime stream isolation test", - manifest_uri: `https://registry.pdpp.dev/connectors/${connectorId}`, - protocol_version: "0.1.0", - streams: streams.map((stream) => ({ - selection: { fields: true, resources: true }, - semantics: "mutable_state", - ...stream, - })), - version: "1.0.0", - }; + return withFixtureSourceDeclaration( + { + connector_id: connectorId, + display_name: "Runtime stream isolation test", + manifest_uri: `https://registry.pdpp.dev/connectors/${connectorId}`, + protocol_version: "0.1.0", + streams: streams.map((stream) => ({ + selection: { fields: true, resources: true }, + semantics: "mutable_state", + ...stream, + })), + version: "1.0.0", + }, + { sourceId: `https://registry.pdpp.dev/connectors/${connectorId}` } + ); } function writeConnectorStub(tmpDir: string, script: string): string { @@ -871,10 +878,9 @@ rl.on('line', (line) => { type: string; }; assert.equal(replayStart.type, "START"); - assert.ok( - replayStart.protocol_capabilities?.includes("DETAIL_COVERAGE_KEY_SETS"), - "START must advertise support before connectors emit empty_keys/unchanged_keys" - ); + // START lists exactly what this runtime implements. DETAIL_COVERAGE_KEY_SETS is + // advisory and not advertised, so this connector reports hydrated keys only. + assert.deepEqual(replayStart.protocol_capabilities, ["OBSERVATION", "STREAM_EVIDENCE"]); const replayStartState = replayStart.state; assert.ok( !("sibling_b" in (replayStartState ?? {})), diff --git a/reference-implementation/test/scheduler-static-secret-injection.test.ts b/reference-implementation/test/scheduler-static-secret-injection.test.ts index 4b7166945..2c15737b7 100644 --- a/reference-implementation/test/scheduler-static-secret-injection.test.ts +++ b/reference-implementation/test/scheduler-static-secret-injection.test.ts @@ -61,6 +61,7 @@ import type { RunRecord } from "../runtime/scheduler-domain-types.ts"; import { closeDb, getDb } from "../server/db.ts"; import { startServer } from "../server/index.ts"; import { createSqliteConnectorInstanceStore } from "../server/stores/connector-instance-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; const BACKGROUND_SAFE_MANIFEST = { capabilities: { @@ -246,33 +247,36 @@ async function registerFixtureConnectorInstance(options: { } function minimalNestedAuthManifest(connectorKey: string): Record { - return { - capabilities: { - auth: { kind: "custom", required: [["NESTED_PRIMARY_TOKEN", "NESTED_SECONDARY_TOKEN"]] }, - refresh_policy: { - background_safe: true, - interaction_posture: "none", - rationale: "Test fixture for connector environment policy coverage.", - recommended_interval_seconds: 60, - recommended_mode: "automatic", + return withFixtureSourceDeclaration( + { + capabilities: { + auth: { kind: "custom", required: [["NESTED_PRIMARY_TOKEN", "NESTED_SECONDARY_TOKEN"]] }, + refresh_policy: { + background_safe: true, + interaction_posture: "none", + rationale: "Test fixture for connector environment policy coverage.", + recommended_interval_seconds: 60, + recommended_mode: "automatic", + }, }, + connector_id: connectorKey, + connector_key: connectorKey, + display_name: connectorKey, + manifest_uri: `https://registry.pdpp.dev/connectors/${connectorKey}`, + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "0.1.0", }, - connector_id: connectorKey, - connector_key: connectorKey, - display_name: connectorKey, - manifest_uri: `https://registry.pdpp.dev/connectors/${connectorKey}`, - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "0.1.0", - }; + { sourceId: `https://registry.pdpp.dev/connectors/${connectorKey}` } + ); } test("connector platform env keeps cross-host execution inputs without ambient secrets", () => { @@ -1024,33 +1028,39 @@ test("connector stderr progress and retained diagnostics redact secret and proxy }); try { - await registerManifest(`http://localhost:${server.asPort}`, { - ...BACKGROUND_SAFE_MANIFEST, - capabilities: { - refresh_policy: { - background_safe: true, - interaction_posture: "none", - rationale: "Test fixture for stderr redaction coverage.", - recommended_interval_seconds: 60, - recommended_mode: "automatic", - }, - }, - connector_id: "stderr-server-test", - connector_key: "stderr-server-test", - display_name: "Stderr server test", - manifest_uri: "https://registry.pdpp.dev/connectors/stderr-server-test", - protocol_version: "0.1.0", - streams: [ + await registerManifest( + `http://localhost:${server.asPort}`, + withFixtureSourceDeclaration( { - name: "items", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", + ...BACKGROUND_SAFE_MANIFEST, + capabilities: { + refresh_policy: { + background_safe: true, + interaction_posture: "none", + rationale: "Test fixture for stderr redaction coverage.", + recommended_interval_seconds: 60, + recommended_mode: "automatic", + }, + }, + connector_id: "stderr-server-test", + connector_key: "stderr-server-test", + display_name: "Stderr server test", + manifest_uri: "https://registry.pdpp.dev/connectors/stderr-server-test", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "0.1.0", }, - ], - version: "0.1.0", - }); + { sourceId: "https://registry.pdpp.dev/connectors/stderr-server-test" } + ) + ); await registerFixtureConnectorInstance({ connectorId: "stderr-server-test", connectorInstanceId: "cin_stderr_server", @@ -1271,7 +1281,13 @@ test("a store row suppresses credentials_required on the scheduled path (and its connectorInstanceId: "cin_github_test", connectorPath, intervalMs: 60_000, - manifest: BACKGROUND_SAFE_MANIFEST, + manifest: { + ...BACKGROUND_SAFE_MANIFEST, + capabilities: { + ...BACKGROUND_SAFE_MANIFEST.capabilities, + human_interaction: ["credentials"], + }, + }, ownerSubjectId: "scheduler_creds_required_user", ownerToken, }, diff --git a/reference-implementation/test/scheduler.test.ts b/reference-implementation/test/scheduler.test.ts index fae254c4d..9fa4ebdbd 100644 --- a/reference-implementation/test/scheduler.test.ts +++ b/reference-implementation/test/scheduler.test.ts @@ -42,6 +42,7 @@ import { import type { ActiveRunRecord, SchedulerRunHistoryRecord } from "../server/stores/scheduler-store.ts"; import { getDefaultSchedulerStore } from "../server/stores/scheduler-store.ts"; import { resolveCredentialFreeFixtureRunEnv } from "./helpers/credential-free-run-fixture.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; const __dirname = dirname(fileURLToPath(import.meta.url)); const REFERENCE_IMPL_DIR = join(__dirname, ".."); @@ -55,27 +56,30 @@ function readServerSchedulerFixtureManifest(): Record & { const base = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ) as Record & { capabilities?: Record }; - return { - ...base, - capabilities: { - ...base.capabilities, - refresh_policy: { - background_safe: true, - bot_detection_sensitivity: "low", - interaction_posture: "none", - maximum_staleness_seconds: 3600, - minimum_interval_seconds: 1, - rate_limit_sensitivity: "low", - rationale: "Deterministic scheduler integration fixture.", - recommended_interval_seconds: 60, - recommended_mode: "automatic", + return withFixtureSourceDeclaration( + { + ...base, + capabilities: { + ...base.capabilities, + refresh_policy: { + background_safe: true, + bot_detection_sensitivity: "low", + interaction_posture: "none", + maximum_staleness_seconds: 3600, + minimum_interval_seconds: 1, + rate_limit_sensitivity: "low", + rationale: "Deterministic scheduler integration fixture.", + recommended_interval_seconds: 60, + recommended_mode: "automatic", + }, }, + connector_id: "scheduler-fixture", + connector_key: "scheduler-fixture", + display_name: "Scheduler fixture", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-fixture", }, - connector_id: "scheduler-fixture", - connector_key: "scheduler-fixture", - display_name: "Scheduler fixture", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-fixture", - }; + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-fixture" } + ); } interface ClosableHttpServer { @@ -1262,29 +1266,32 @@ test("scheduler progress watchdog allows long direct runs that keep reporting pr }); test("scheduler preserves failure reasons and checkpoint summaries from failed runConnector results", async () => { - const manifest = { - connector_id: "scheduler-failure-test", - display_name: "Scheduler Failure Test Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-failure-test", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, - value: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-failure-test", + display_name: "Scheduler Failure Test Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-failure-test", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + value: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-failure-test" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-failure-")); const connectorPath = join(tmpDir, "connector.mjs"); writeFileSync( @@ -1395,43 +1402,46 @@ rl.on('line', (line) => { }); test("scheduler preserves partial checkpoint commit summaries from state persistence failures after DONE(succeeded)", async () => { - const manifest = { - connector_id: "scheduler-partial-checkpoint-test", - display_name: "Scheduler Partial Checkpoint Test Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-partial-checkpoint-test", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, - value: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-partial-checkpoint-test", + display_name: "Scheduler Partial Checkpoint Test Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-partial-checkpoint-test", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + value: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - { - name: "other_items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, - value: { type: "string" }, + { + name: "other_items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + value: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-partial-checkpoint-test" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-partial-checkpoint-")); const connectorPath = join(tmpDir, "connector.mjs"); writeFileSync( @@ -1601,29 +1611,32 @@ rl.on('line', (line) => { }); test("scheduler preserves terminal counter mismatch failures from runConnector results", async () => { - const manifest = { - connector_id: "scheduler-terminal-counter-mismatch-test", - display_name: "Scheduler Terminal Counter Mismatch Test Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-terminal-counter-mismatch-test", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, - value: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-terminal-counter-mismatch-test", + display_name: "Scheduler Terminal Counter Mismatch Test Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-terminal-counter-mismatch-test", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + value: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-terminal-counter-mismatch-test" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-terminal-counter-mismatch-")); const connectorPath = join(tmpDir, "connector.mjs"); writeFileSync( @@ -1749,29 +1762,32 @@ rl.on('line', (line) => { }); test("scheduler preserves connector-declared terminal error details from failed runs", async () => { - const manifest = { - connector_id: "scheduler-terminal-error-test", - display_name: "Scheduler Terminal Error Test Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-terminal-error-test", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, - value: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-terminal-error-test", + display_name: "Scheduler Terminal Error Test Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-terminal-error-test", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + value: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-terminal-error-test" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-terminal-error-")); const connectorPath = join(tmpDir, "connector.mjs"); writeFileSync( @@ -1885,28 +1901,31 @@ rl.on('line', (line) => { }); test("scheduler preserves known gaps from partial connector runs", async () => { - const manifest = { - connector_id: "scheduler-known-gap-test", - display_name: "Scheduler Known Gap Test Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-known-gap-test", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-known-gap-test", + display_name: "Scheduler Known Gap Test Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-known-gap-test", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-known-gap-test" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-known-gap-")); const connectorPath = join(tmpDir, "connector.mjs"); writeFileSync( @@ -1995,29 +2014,32 @@ rl.on('line', (line) => { }); test("scheduler preserves connector-declared terminal error details from cancelled runs", async () => { - const manifest = { - connector_id: "scheduler-cancelled-terminal-error-test", - display_name: "Scheduler Cancelled Terminal Error Test Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-cancelled-terminal-error-test", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, - value: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-cancelled-terminal-error-test", + display_name: "Scheduler Cancelled Terminal Error Test Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-cancelled-terminal-error-test", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + value: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-cancelled-terminal-error-test" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-cancelled-terminal-error-")); const connectorPath = join(tmpDir, "connector.mjs"); writeFileSync( @@ -2131,28 +2153,31 @@ rl.on('line', (line) => { }); test("scheduler does not retry deterministic connector protocol violations", async () => { - const manifest = { - connector_id: "scheduler-no-retry-protocol-violation", - display_name: "Scheduler No Retry Protocol Violation Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-no-retry-protocol-violation", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-no-retry-protocol-violation", + display_name: "Scheduler No Retry Protocol Violation Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-no-retry-protocol-violation", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-no-retry-protocol-violation" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-no-retry-protocol-")); const connectorPath = join(tmpDir, "connector.mjs"); const attemptsPath = join(tmpDir, "attempts.log"); @@ -2237,29 +2262,32 @@ rl.on('line', (line) => { }); test("scheduler retries connector-declared retryable failures and records the succeeding attempt", async () => { - const manifest = { - connector_id: "scheduler-retryable-terminal-error", - display_name: "Scheduler Retryable Terminal Error Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-retryable-terminal-error", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, - value: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-retryable-terminal-error", + display_name: "Scheduler Retryable Terminal Error Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-retryable-terminal-error", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + value: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-retryable-terminal-error" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-retryable-terminal-error-")); const connectorPath = join(tmpDir, "connector.mjs"); const attemptsPath = join(tmpDir, "attempts.log"); @@ -2369,29 +2397,32 @@ rl.on('line', (line) => { }); test("scheduler does not retry connector-declared non-retryable failures", async () => { - const manifest = { - connector_id: "scheduler-nonretryable-terminal-error", - display_name: "Scheduler Nonretryable Terminal Error Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-nonretryable-terminal-error", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { - properties: { - id: { type: "string" }, - value: { type: "string" }, + const manifest = withFixtureSourceDeclaration( + { + connector_id: "scheduler-nonretryable-terminal-error", + display_name: "Scheduler Nonretryable Terminal Error Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-nonretryable-terminal-error", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { + properties: { + id: { type: "string" }, + value: { type: "string" }, + }, + required: ["id"], + type: "object", }, - required: ["id"], - type: "object", + selection: { fields: true, resources: true }, + semantics: "append_only", }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-nonretryable-terminal-error" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-nonretryable-terminal-error-")); const connectorPath = join(tmpDir, "connector.mjs"); const attemptsPath = join(tmpDir, "attempts.log"); @@ -3945,11 +3976,11 @@ test("scheduler records one not-ready skip for automatic runs when runtime prere const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-not-ready-test", connector_key: "scheduler-not-ready-test", - }; + }); const server = await startServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; @@ -4014,11 +4045,11 @@ test("scheduler emits a fresh not-ready skip when readiness reason changes", asy const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-not-ready-changing-test", connector_key: "scheduler-not-ready-changing-test", - }; + }); const server = await startServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; @@ -4081,7 +4112,7 @@ test("scheduler default readiness checker skips missing manifest-declared extern const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-missing-tool-test", connector_key: "scheduler-missing-tool-test", @@ -4097,7 +4128,7 @@ test("scheduler default readiness checker skips missing manifest-declared extern }, ], }, - }; + }); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-missing-tool-")); const { attemptsPath, connectorPath } = writeLoggingConnector(tmpDir); const server = await startServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); @@ -4156,7 +4187,7 @@ test("scheduler default readiness checker probes a manifest-declared executable_ const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-slackdump-bin-test", connector_key: "scheduler-slackdump-bin-test", @@ -4177,7 +4208,7 @@ test("scheduler default readiness checker probes a manifest-declared executable_ }, ], }, - }; + }); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-slackdump-bin-")); const fakeSlackdumpPath = join(tmpDir, "mounted-slackdump"); writeFileSync(fakeSlackdumpPath, '#!/bin/sh\n[ "$1" = "version" ] || exit 2\nexit 0\n', "utf8"); @@ -4318,14 +4349,14 @@ test("scheduler default readiness checker does not treat browser bindings as rea const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-browser-not-ready-test", connector_key: "scheduler-browser-not-ready-test", runtime_requirements: { bindings: { browser: { required: true }, network: { required: true } }, }, - }; + }); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-browser-not-ready-")); const { attemptsPath, connectorPath } = writeLoggingConnector(tmpDir); const previousRemoteCdp = process.env.PDPP_BROWSER_SURFACE_REMOTE_CDP_URL; @@ -4428,14 +4459,14 @@ test("scheduler default readiness checker treats PDPP_NEKO_CDP_HTTP_URL as manag const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-browser-neko-cdp-ready-test", connector_key: "scheduler-browser-neko-cdp-ready-test", runtime_requirements: { bindings: { browser: { required: true }, network: { required: true } }, }, - }; + }); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-browser-neko-cdp-")); const { attemptsPath, connectorPath } = writeLoggingConnector(tmpDir); const previousRemoteCdp = process.env.PDPP_BROWSER_SURFACE_REMOTE_CDP_URL; @@ -4526,14 +4557,14 @@ test("scheduler default readiness checker treats PDPP_NEKO_MANAGED_CONNECTORS as const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-browser-neko-managed-ready-test", connector_key: "scheduler-browser-neko-managed-ready-test", runtime_requirements: { bindings: { browser: { required: true }, network: { required: true } }, }, - }; + }); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-browser-neko-managed-")); const { attemptsPath, connectorPath } = writeLoggingConnector(tmpDir); const previousRemoteCdp = process.env.PDPP_BROWSER_SURFACE_REMOTE_CDP_URL; @@ -4620,22 +4651,26 @@ test("scheduler default readiness checker treats PDPP_NEKO_MANAGED_CONNECTORS as }); test("scheduler marks connector as needs-human when automatic run triggers interaction", async () => { - const manifest = { - connector_id: "scheduler-interaction-test", - display_name: "Interaction Test Connector", - manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-interaction-test", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "append_only", - }, - ], - version: "1.0.0", - }; + const manifest = withFixtureSourceDeclaration( + { + capabilities: { human_interaction: ["otp"] }, + connector_id: "scheduler-interaction-test", + display_name: "Interaction Test Connector", + manifest_uri: "https://registry.pdpp.dev/connectors/scheduler-interaction-test", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "append_only", + }, + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/scheduler-interaction-test" } + ); const tmpDir = mkdtempSync(join(tmpdir(), "pdpp-scheduler-interaction-")); const connectorPath = join(tmpDir, "connector.mjs"); @@ -4745,11 +4780,11 @@ test("scheduler backoff skip derives next_attempt_at from history when last_run_ const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-backoff-1970-regression", connector_key: "scheduler-backoff-1970-regression", - }; + }); const server = await startServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; @@ -4868,11 +4903,11 @@ test("scheduler backoff skip uses gave_up phrasing once health-state crosses blo const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-backoff-blocked-msg", connector_key: "scheduler-backoff-blocked-msg", - }; + }); const server = await startServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; @@ -4970,11 +5005,11 @@ test("scheduler does not re-emit persisted backoff transition markers on restart const spotifyManifest = JSON.parse( readFileSync(join(REFERENCE_IMPL_DIR, "fixtures/seed-manifests/spotify.json"), "utf8") ); - const manifest = { + const manifest = withFixtureSourceDeclaration({ ...spotifyManifest, connector_id: "scheduler-backoff-restart-noise", connector_key: "scheduler-backoff-restart-noise", - }; + }); const server = await startServer({ asPort: 0, dbPath: ":memory:", quiet: true, rsPort: 0 }); const asUrl = `http://localhost:${server.asPort}`; const rsUrl = `http://localhost:${server.rsPort}`; diff --git a/reference-implementation/test/search-index-repair-correctness.postgres.test.ts b/reference-implementation/test/search-index-repair-correctness.postgres.test.ts index 1947e131f..077571d03 100644 --- a/reference-implementation/test/search-index-repair-correctness.postgres.test.ts +++ b/reference-implementation/test/search-index-repair-correctness.postgres.test.ts @@ -22,10 +22,12 @@ import { __setLexicalBackfillPhaseHookForTest } from "../server/search.ts"; import { reconcileSearchIndexDirtyScope } from "../server/search-index-reconcile.ts"; import { configureSemanticBackend } from "../server/search-semantic.ts"; import { recordSearchIndexDirtyFailure } from "../server/stores/search-index-dirty-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; const POSTGRES_URL = process.env.PDPP_TEST_POSTGRES_URL; -const manifestFor = (connectorId: string) => ({ +const manifestFor = (connectorId: string) => { + const manifest = { capabilities: { human_interaction: [] }, connector_id: connectorId, display_name: connectorId, @@ -50,7 +52,9 @@ const manifestFor = (connectorId: string) => ({ }, ], version: "0.1.0", -}); + }; + return withFixtureSourceDeclaration(manifest, { sourceId: manifest.manifest_uri }); +}; const target = (connectorId: string, connectorInstanceId: string) => ({ connector_id: connectorId, diff --git a/reference-implementation/test/semantic-startup-backfill-catches-crash-abandoned-record.test.ts b/reference-implementation/test/semantic-startup-backfill-catches-crash-abandoned-record.test.ts index 9467cde37..915e41e3e 100644 --- a/reference-implementation/test/semantic-startup-backfill-catches-crash-abandoned-record.test.ts +++ b/reference-implementation/test/semantic-startup-backfill-catches-crash-abandoned-record.test.ts @@ -40,6 +40,7 @@ import { semanticIndexUpsert, } from "../server/search-semantic.ts"; import { listDirtySearchIndexScopes } from "../server/stores/search-index-dirty-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; function target(connectorInstanceId: string) { return { @@ -57,36 +58,42 @@ function record(stream: string, key: string, subject: string) { }; } -const baseManifest = { - capabilities: { human_interaction: [] }, - connector_id: "crash-gap", - display_name: "Crash gap probe", - manifest_uri: "https://registry.pdpp.dev/connectors/crash-gap", - protocol_version: "0.1.0", - streams: [ - { - name: "items", - primary_key: ["id"], - query: { search: {} }, - schema: { - properties: { id: { type: "string" }, subject: { type: "string" } }, - required: ["id", "subject"], - type: "object", +const baseManifest = withFixtureSourceDeclaration( + { + capabilities: { human_interaction: [] }, + connector_id: "crash-gap", + display_name: "Crash gap probe", + manifest_uri: "https://registry.pdpp.dev/connectors/crash-gap", + protocol_version: "0.1.0", + streams: [ + { + name: "items", + primary_key: ["id"], + query: { search: {} }, + schema: { + properties: { id: { type: "string" }, subject: { type: "string" } }, + required: ["id", "subject"], + type: "object", + }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "1.0.0", -}; + ], + version: "1.0.0", + }, + { sourceId: "https://registry.pdpp.dev/connectors/crash-gap" } +); -const semanticManifest = { - ...baseManifest, - streams: baseManifest.streams.map((stream) => ({ - ...stream, - query: { search: { semantic_fields: ["subject"] } }, - })), -}; +const semanticManifest = withFixtureSourceDeclaration( + { + ...baseManifest, + streams: baseManifest.streams.map((stream) => ({ + ...stream, + query: { search: { semantic_fields: ["subject"] } }, + })), + }, + { sourceId: baseManifest.manifest_uri } +); /** * A real publisher in a separate Node process, therefore a separate @@ -279,17 +286,20 @@ test("a 501-field manifest publishes bounded batches and restart embeds only the initDb(":memory:"); const fields = Array.from({ length: 501 }, (_, index) => `field_${index}`); const properties = Object.fromEntries(fields.map((field) => [field, { type: "string" }])); - const manifest = { - ...semanticManifest, - connector_id: "semantic-field-batches", - streams: [ - { - ...semanticManifest.streams[0], - query: { search: { semantic_fields: fields } }, - schema: { properties: { id: { type: "string" }, ...properties }, type: "object" }, - }, - ], - }; + const manifest = withFixtureSourceDeclaration( + { + ...semanticManifest, + connector_id: "semantic-field-batches", + streams: [ + { + ...semanticManifest.streams[0], + query: { search: { semantic_fields: fields } }, + schema: { properties: { id: { type: "string" }, ...properties }, type: "object" }, + }, + ], + }, + { sourceId: baseManifest.manifest_uri } + ); let calls = 0; const stub = makeStubBackend({ dimensions: 8 }); configureSemanticBackend({ @@ -389,15 +399,18 @@ for (const kind of ["blob-flat", "sqlite-vec", "postgres"] as const) { }); const instance = `cin_identity_${kind.replaceAll("-", "_")}_${runId}_${++sequence}`; const connector = `identity-${kind}-${runId}-${sequence}`; - const manifest = { - ...semanticManifest, - connector_id: connector, - display_name: connector, - streams: ["items", "item_", "item%", "itemX"].map((name) => ({ - ...required(semanticManifest.streams[0]), - name, - })), - }; + const manifest = withFixtureSourceDeclaration( + { + ...semanticManifest, + connector_id: connector, + display_name: connector, + streams: ["items", "item_", "item%", "itemX"].map((name) => ({ + ...required(semanticManifest.streams[0]), + name, + })), + }, + { sourceId: baseManifest.manifest_uri } + ); const inputs: string[] = []; const stub = makeStubBackend({ dimensions: 8 }); const embedding = { diff --git a/reference-implementation/test/stream-evidence.test.ts b/reference-implementation/test/stream-evidence.test.ts index aeca2cecd..236e5482d 100644 --- a/reference-implementation/test/stream-evidence.test.ts +++ b/reference-implementation/test/stream-evidence.test.ts @@ -39,6 +39,7 @@ import { } from "../server/ref-control.ts"; import { createRequestConnectorInstanceStore } from "../server/request-store-factories.ts"; import { admitOwnerRunConnection } from "../server/stores/connector-instance-store.ts"; +import { withFixtureSourceDeclaration } from "./helpers/fixture-source-declaration.ts"; const NOT_STATE_STREAM_DECLARED_PATTERN = /which the manifest does not declare with a\s+static state_stream parent/; const OUTCOMES_SUM_MISMATCH_PATTERN = /outcomes.*do not sum to considered/i; @@ -163,41 +164,44 @@ function fakeAdmitRunConnection( * stream used only for the manifest-shape rejection test. */ function streamEvidenceManifest(connectorId: string) { - return { - capabilities: { human_interaction: [] }, - connector_id: connectorId, - display_name: connectorId, - manifest_uri: `https://sources.example/${connectorId}`, - protocol_version: "0.1.0", - streams: [ - { - name: "messages", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - { - coverage_strategy: "checkpoint_window", - name: "message_bodies", - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "append_only", - state_stream: "messages", - }, - { - coverage_strategy: "parent_detail_accounting", - name: "message_attachments", - parent_streams: ["messages"], - primary_key: ["id"], - schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, - selection: { fields: true, resources: true }, - semantics: "mutable_state", - }, - ], - version: "0.1.0", - }; + return withFixtureSourceDeclaration( + { + capabilities: { human_interaction: [] }, + connector_id: connectorId, + display_name: connectorId, + manifest_uri: `https://sources.example/${connectorId}`, + protocol_version: "0.1.0", + streams: [ + { + name: "messages", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + { + coverage_strategy: "checkpoint_window", + name: "message_bodies", + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "append_only", + state_stream: "messages", + }, + { + coverage_strategy: "parent_detail_accounting", + name: "message_attachments", + parent_streams: ["messages"], + primary_key: ["id"], + schema: { properties: { id: { type: "string" } }, required: ["id"], type: "object" }, + selection: { fields: true, resources: true }, + semantics: "mutable_state", + }, + ], + version: "0.1.0", + }, + { sourceId: `https://sources.example/${connectorId}` } + ); } /**