From f75875792c33ab9e6a6180aa118c58739c600a21 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Thu, 5 Nov 2026 13:22:00 -0600 Subject: [PATCH 1/8] build: pin one installer core and move call sites to its OCI-only API The root, reference-implementation and local-collector packages pinned three different data-connectors commits. All three now pin 708a6cc17d, so npm installs a single copy of the installer core. That core installs only signed OCI Collection Profiles: `source` is null, `loadConnectorIndex` and `generateLock` are gone, and the certificate identity resolver is no longer a per-call option. Call sites in the reference server, the local collector and the pin script follow, and the guards for APIs that older cores lacked are removed. The core imports `ajv-formats`, so the collector declares it. The core cannot read the 12 retired tarball entries in connectors/lock.json or github-pdpp 0.5.0, which has no signed source declaration. A second pinned dependency, data-connectors-legacy-installer (e1e497f3bc, an ancestor of data-connectors main), serves only those entries in scripts/resolve-connectors.js until they move to OCI profiles. Assisted-by: AI Signed-off-by: Tim Nunamaker --- package-lock.json | 81 ++++++++----- package.json | 3 +- packages/local-collector/package.json | 3 +- .../scripts/pin-collection-profiles.ts | 40 ++++++- .../src/managed/collection-profiles.ts | 6 +- packages/local-collector/src/managed/index.ts | 37 +++--- .../src/managed/installer-core.d.ts | 4 +- .../src/managed/registry-client.ts | 6 +- .../test/collection-profiles.test.ts | 4 +- reference-implementation/package.json | 2 +- .../server/connector-install/index.ts | 24 ++-- scripts/resolve-connectors.js | 106 ++++++++++++++++-- scripts/resolve-connectors.test.mjs | 65 ++++++++++- 13 files changed, 281 insertions(+), 100 deletions(-) diff --git a/package-lock.json b/package-lock.json index e109d1536..f995ed823 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,7 +42,7 @@ }, "devDependencies": { "@eslint/js": "^10.0.1", - "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#6c71697a1523926fe8d475cd243a8219ea21aa5b", + "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#708a6cc17d6821fad222a652084f2bc270c40052", "@semantic-release/exec": "^7.1.0", "@stryker-mutator/core": "10.0.0", "@stryker-mutator/vitest-runner": "10.0.0", @@ -56,6 +56,7 @@ "@vitejs/plugin-react": "^6.1.2", "autoprefixer": "^10.6.1", "conventional-changelog-conventionalcommits": "^10.4.1", + "data-connectors-legacy-installer": "github:PDP-Connect/data-connectors#e1e497f3bca842c0489c472c1ca286630eef5078", "eslint": "^10.12.0", "eslint-config-prettier": "^10.1.8", "eslint-plugin-react-hooks": "^7.1.1", @@ -3531,9 +3532,8 @@ }, "node_modules/@opendatalabs/data-connectors-tools": { "version": "0.1.0", - "resolved": "git+ssh://git@github.com/PDP-Connect/data-connectors.git#6c71697a1523926fe8d475cd243a8219ea21aa5b", - "integrity": "sha512-JXPJDjfKhFeAJEn8TZLVzhNrangqTYWOFIEff6VmNj5KuKP2md76sM71YVnCx4u8uRFWcHmTWvgJEilkikBLvA==", - "dev": true, + "resolved": "git+ssh://git@github.com/PDP-Connect/data-connectors.git#708a6cc17d6821fad222a652084f2bc270c40052", + "integrity": "sha512-0J0LBwzLI3FcT/7bP1zwgzRnz5c41aVTEQEE9dhf5/EjP+DMx+gPb3GGDT9faO4MvjWQe0xc8rLSWEzeOi2gCQ==", "license": "Apache-2.0", "workspaces": [ "packages/connector-installer-core", @@ -3542,6 +3542,7 @@ "dependencies": { "@sigstore/bundle": "^5.0.0", "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "sigstore": "^5.0.0", "tar": "^7.5.22" }, @@ -3553,7 +3554,6 @@ "version": "8.20.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", - "dev": true, "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.3", @@ -3570,7 +3570,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true, "license": "MIT" }, "node_modules/@opendatalabs/remote-surface": { @@ -10450,6 +10449,51 @@ "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", "license": "MIT" }, + "node_modules/data-connectors-legacy-installer": { + "name": "@opendatalabs/data-connectors-tools", + "version": "0.1.0", + "resolved": "git+ssh://git@github.com/PDP-Connect/data-connectors.git#e1e497f3bca842c0489c472c1ca286630eef5078", + "integrity": "sha512-FOFkJkRFU173i9t1QFW4F5zcU2MwwM+4A9a6bnGnsuftCqF7p4xfmKxNI2Y0vRMB00V4hTItVjpWd/k8p5QtSQ==", + "dev": true, + "license": "Apache-2.0", + "workspaces": [ + "packages/connector-installer-core", + "packages/polyfill-connectors" + ], + "dependencies": { + "@sigstore/bundle": "^5.0.0", + "ajv": "^8.20.0", + "sigstore": "^5.0.0", + "tar": "^7.5.22" + }, + "bin": { + "connector-installer": "packages/connector-installer-cli/index.mjs" + } + }, + "node_modules/data-connectors-legacy-installer/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/data-connectors-legacy-installer/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, "node_modules/data-urls": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-8.0.0.tgz", @@ -23716,6 +23760,7 @@ "dependencies": { "@sigstore/bundle": "^5.0.0", "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "sigstore": "^5.0.0", "tar": "^7.5.22", "zod": "^4.6.7" @@ -23724,7 +23769,7 @@ "pdpp-local-collector": "dist/local-collector/bin/pdpp-local-collector.js" }, "devDependencies": { - "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#6c71697a1523926fe8d475cd243a8219ea21aa5b", + "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#708a6cc17d6821fad222a652084f2bc270c40052", "@pdpp/collector-runtime": "0.0.1", "@pdpp/connector-protocol": "0.0.1", "playwright": "^1.64.1", @@ -23769,7 +23814,7 @@ "dependencies": { "@fastify/formbody": "^9.0.1", "@huggingface/transformers": "^4.3.1", - "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#06ed284e1b7ee92b2ebab6bc8ef25f0ae392f84a", + "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#708a6cc17d6821fad222a652084f2bc270c40052", "@pdpp/collector-runtime": "*", "@pdpp/connector-protocol": "*", "@pdpp/display": "*", @@ -23869,26 +23914,6 @@ "node": "^22.18.0 || >=24.11.0" } }, - "reference-implementation/node_modules/@opendatalabs/data-connectors-tools": { - "version": "0.1.0", - "resolved": "git+ssh://git@github.com/PDP-Connect/data-connectors.git#06ed284e1b7ee92b2ebab6bc8ef25f0ae392f84a", - "integrity": "sha512-lTcdI60BuD+pEAXcvmEb9iDATPrXWxGve/Ah08MmOXZ4emp8KaJgmKWZdtg5qcJmOrA8ZGX5kVNfcFeO2B62tg==", - "license": "Apache-2.0", - "workspaces": [ - "packages/connector-installer-core", - "packages/polyfill-connectors" - ], - "dependencies": { - "@sigstore/bundle": "^5.0.0", - "ajv": "^8.20.0", - "ajv-formats": "^3.0.1", - "sigstore": "^5.0.0", - "tar": "^7.5.22" - }, - "bin": { - "connector-installer": "packages/connector-installer-cli/index.mjs" - } - }, "reference-implementation/node_modules/ajv": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", diff --git a/package.json b/package.json index 0d4373aaa..b57e4a5ec 100644 --- a/package.json +++ b/package.json @@ -97,7 +97,7 @@ }, "devDependencies": { "@eslint/js": "^10.0.1", - "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#6c71697a1523926fe8d475cd243a8219ea21aa5b", + "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#708a6cc17d6821fad222a652084f2bc270c40052", "@semantic-release/exec": "^7.1.0", "@stryker-mutator/core": "10.0.0", "@stryker-mutator/vitest-runner": "10.0.0", @@ -111,6 +111,7 @@ "@vitejs/plugin-react": "^6.1.2", "autoprefixer": "^10.6.1", "conventional-changelog-conventionalcommits": "^10.4.1", + "data-connectors-legacy-installer": "github:PDP-Connect/data-connectors#e1e497f3bca842c0489c472c1ca286630eef5078", "eslint": "^10.12.0", "eslint-config-prettier": "^10.1.8", "eslint-plugin-react-hooks": "^7.1.1", diff --git a/packages/local-collector/package.json b/packages/local-collector/package.json index 32de4f9d4..f01d7c3f1 100644 --- a/packages/local-collector/package.json +++ b/packages/local-collector/package.json @@ -55,12 +55,13 @@ "dependencies": { "@sigstore/bundle": "^5.0.0", "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "sigstore": "^5.0.0", "tar": "^7.5.22", "zod": "^4.6.7" }, "devDependencies": { - "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#6c71697a1523926fe8d475cd243a8219ea21aa5b", + "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#708a6cc17d6821fad222a652084f2bc270c40052", "@pdpp/collector-runtime": "0.0.1", "@pdpp/connector-protocol": "0.0.1", "playwright": "^1.64.1", diff --git a/packages/local-collector/scripts/pin-collection-profiles.ts b/packages/local-collector/scripts/pin-collection-profiles.ts index 8bf5cdd05..5a70450e4 100644 --- a/packages/local-collector/scripts/pin-collection-profiles.ts +++ b/packages/local-collector/scripts/pin-collection-profiles.ts @@ -25,9 +25,12 @@ * Usage (network required): * * npm run pin:collection-profiles --workspace @pdpp/local-collector + * npm run pin:collection-profiles --workspace @pdpp/local-collector -- --version claude-code=0.3.6 * * Re-running it moves every pin to the catalog's latest release. Review the - * diff as a connector upgrade. + * diff as a connector upgrade. `--version =` (repeatable) + * pins that connector to a named catalog release instead of the latest, for a + * connector whose latest release drops a stream this collector still needs. */ import { mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs"; @@ -59,9 +62,15 @@ const REFERENCE_PROFILE_DIR = path.resolve( "local-collector-profiles" ); +interface CatalogRelease { + readonly digest: string; + readonly version: string; +} + interface CatalogConnector { readonly connector_key: string; - readonly latest?: { readonly digest: string; readonly version: string }; + readonly latest?: CatalogRelease; + readonly versions?: readonly CatalogRelease[]; } interface ResolvedArtifact { @@ -75,7 +84,24 @@ function connectorKeyFor(connectorId: string): string { return connectorId.replaceAll("_", "-"); } +/** `--version =` arguments, as a key → version map. */ +function requestedVersions(args: readonly string[]): Map { + const requested = new Map(); + args.forEach((arg, index) => { + if (arg !== "--version") { + return; + } + const [key, version] = (args[index + 1] ?? "").split("="); + if (!key || !version) { + throw new Error("--version takes ="); + } + requested.set(key, version); + }); + return requested; +} + async function main(): Promise { + const versions = requestedVersions(process.argv.slice(2)); const catalogResult = (await fetchCatalog({})) as { catalog: { connectors: readonly CatalogConnector[] } }; const catalog = new Map(catalogResult.catalog.connectors.map((entry) => [entry.connector_key, entry])); @@ -83,14 +109,18 @@ async function main(): Promise { const profiles = new Map(); for (const definition of LOCAL_COLLECTOR_DEFINITIONS) { const connectorKey = connectorKeyFor(definition.connector_id); - const latest = catalog.get(connectorKey)?.latest; + const listed = catalog.get(connectorKey); + const wanted = versions.get(connectorKey); + const latest = wanted ? listed?.versions?.find((release) => release.version === wanted) : listed?.latest; if (!latest) { - process.stdout.write(`unpinned ${definition.connector_id}: the signed catalog lists no ${connectorKey}\n`); + process.stdout.write( + `unpinned ${definition.connector_id}: the signed catalog lists no ${connectorKey}${wanted ? ` ${wanted}` : ""}\n` + ); continue; } const repository = collectionProfileRepository({ connectorKey }); const resolved = (await fetchResolvedArtifact( - { doc: {}, mode: "locked" }, + null, { artifactKind: "pdpp-collection-profile", connectorId: definition.connector_id, diff --git a/packages/local-collector/src/managed/collection-profiles.ts b/packages/local-collector/src/managed/collection-profiles.ts index 2b4b35109..45400f05d 100644 --- a/packages/local-collector/src/managed/collection-profiles.ts +++ b/packages/local-collector/src/managed/collection-profiles.ts @@ -69,7 +69,7 @@ export interface CollectionProfilePin { /** The part of the installer core this module calls. */ export interface CollectionProfileInstallerCore { readonly DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY: string; - installFromLock(options: Record): Promise; + installFromLock(options: Record & { source: null }): Promise; } export interface EnsureCollectionProfileOptions { @@ -163,8 +163,6 @@ export async function ensureCollectionProfileInstalled( const identityResolver = ({ registry, repository: requested }: { registry: string; repository: string }) => registry === REGISTRY && requested === repository ? core.DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY : null; await core.installFromLock({ - // Tarball entries are not accepted from this lock. - artifactCertificateIdentityResolver: () => null, installRoot: staging, layout: "source", lock: { @@ -183,7 +181,7 @@ export async function ensureCollectionProfileInstalled( ], }, ociCertificateIdentityResolver: identityResolver, - source: { doc: {}, mode: "locked" }, + source: null, }); const staged = join(staging, "collection-profiles", pin.connectorId); diff --git a/packages/local-collector/src/managed/index.ts b/packages/local-collector/src/managed/index.ts index 120fa79b3..21dcfd0b0 100644 --- a/packages/local-collector/src/managed/index.ts +++ b/packages/local-collector/src/managed/index.ts @@ -6,35 +6,28 @@ * * **Status.** The published collector does not use {@link obtainManagedConnectors}. * It installs through `collection-profiles.ts`, which delegates fetch and - * signature verification to data-connectors' installer core. That core now - * has the OCI and Sigstore path the paragraphs below describe as missing. - * The verifier in this directory is kept, with its tests, until an owner - * decides whether to remove it. + * signature verification to data-connectors' installer core, which has the + * OCI and Sigstore path. The verifier in this directory is kept, with its + * tests, until an owner decides whether to remove it. * * One call, {@link obtainManagedConnectors}, walks a pinned lock to a set of * definitions the existing runner seam accepts: resolve a pinned digest, * verify its signature against host policy, install to a content-addressed * cache, and hand the installed artifact to the runner. * - * **Relationship to `@pdpp/connector-manager`.** That package exists on - * data-connectors#97 (`packages/connector-installer-core`, published name - * `@pdpp/connector-manager`) and is where this logic belongs. It does not - * export it yet: its surface today is the incumbent tarball installer — - * `loadConnectorIndex`, `generateLock`, `installFromLock`, `verifyInstalled` - * and friends — and it contains no OCI, ORAS or Cosign code at all. #97's OCI - * work is publisher-side: `scripts/build-connector-oci-artifact.mjs`, - * `scripts/verify-connector-oci-artifact.mjs` and the publish workflow. The - * manager's *grow* half is unwritten. - * - * So this module is written against the artifact contract rather than against - * an API that does not exist, and it is shaped to be deleted: every step is a - * named export with an injected dependency, so when the manager grows - * `resolve`/`verify`/`install`, this becomes a thin adapter over those calls - * and the runner seam does not move again. What must not happen is the runner + * **Relationship to `@pdpp/connector-manager`.** That package is + * data-connectors' `packages/connector-installer-core` (published here as + * `@opendatalabs/data-connectors-tools/installer-core`) and is where this logic + * belongs. It no longer has the tarball installer: it is OCI-only, and its + * `fetchCatalog`, `fetchResolvedArtifact`, `installFromLock` and + * `verifyInstalled` resolve a pinned digest, verify its Sigstore signature, + * check every blob and install a signed Collection Profile. It still does not + * export the {@link ConnectorManager} shape below, so this module stays + * written against the artifact contract and is shaped to be deleted: every step + * is a named export with an injected dependency, so a swap to the core is a + * substitution rather than a rewrite. What must not happen is the runner * growing a dependency on a shape only this repo knows — hence - * {@link ConnectorManager}, which is the interface the manager is expected to - * satisfy, stated here so the eventual swap is a substitution rather than a - * rewrite. + * {@link ConnectorManager}, the interface the core is expected to satisfy. */ import type { ConnectorLock, PinnedConnector } from "./artifact-contract.ts"; diff --git a/packages/local-collector/src/managed/installer-core.d.ts b/packages/local-collector/src/managed/installer-core.d.ts index 92bb8462f..b17b10e78 100644 --- a/packages/local-collector/src/managed/installer-core.d.ts +++ b/packages/local-collector/src/managed/installer-core.d.ts @@ -8,9 +8,9 @@ declare module "@opendatalabs/data-connectors-tools/installer-core" { export const DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY: string; export function fetchCatalog(options: Record): Promise; export function fetchResolvedArtifact( - source: Record, + source: null, entry: Record, options: Record ): Promise; - export function installFromLock(options: Record): Promise; + export function installFromLock(options: Record & { source: null }): Promise; } diff --git a/packages/local-collector/src/managed/registry-client.ts b/packages/local-collector/src/managed/registry-client.ts index 23fbf2529..cbb070ace 100644 --- a/packages/local-collector/src/managed/registry-client.ts +++ b/packages/local-collector/src/managed/registry-client.ts @@ -11,10 +11,10 @@ * * **Why these are injected rather than implemented here.** The decision's * phrase is "registry-neutral", and the shape neutrality actually takes in - * this codebase is already established: the incumbent installer takes - * `artifactCertificateIdentityResolver` as a caller callback rather than + * this codebase is already established: the installer core takes + * `ociCertificateIdentityResolver` as a caller callback rather than * hardcoding a signer, and data-connect supplies it from - * `scripts/resolve-connectors.js`. Signer policy is the consumer's to decide. + * `collection-profiles.ts`. Signer policy is the consumer's to decide. * This module keeps that arrangement and extends it to blob transport, so a * different publisher serving a different registry is a different injected * client, not a patch to this file. diff --git a/packages/local-collector/test/collection-profiles.test.ts b/packages/local-collector/test/collection-profiles.test.ts index 424840d4c..adf659d0e 100644 --- a/packages/local-collector/test/collection-profiles.test.ts +++ b/packages/local-collector/test/collection-profiles.test.ts @@ -175,7 +175,7 @@ test("a fresh install lands the pinned files under the digest directory and pass const call = core.calls[0] as { lock: { connectors: { oci: unknown; version: string; artifactKind: string }[] }; ociCertificateIdentityResolver: (input: { registry: string; repository: string }) => string | null; - artifactCertificateIdentityResolver: () => unknown; + source: unknown; }; assert.deepEqual(call.lock.connectors[0]?.oci, { digest: fakePin().digest, @@ -190,7 +190,7 @@ test("a fresh install lands the pinned files under the digest directory and pass ); assert.equal(call.ociCertificateIdentityResolver({ registry: "ghcr.io", repository: "pdp-connect/connector/other" }), null); assert.equal(call.ociCertificateIdentityResolver({ registry: "docker.io", repository: "pdp-connect/connector/fake-connector" }), null); - assert.equal(call.artifactCertificateIdentityResolver(), null); + assert.equal(call.source, null); }); test("a cached release that matches its pin is used without the network", async () => { diff --git a/reference-implementation/package.json b/reference-implementation/package.json index 967752fe8..f27167821 100644 --- a/reference-implementation/package.json +++ b/reference-implementation/package.json @@ -71,7 +71,7 @@ "dependencies": { "@fastify/formbody": "^9.0.1", "@huggingface/transformers": "^4.3.1", - "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#06ed284e1b7ee92b2ebab6bc8ef25f0ae392f84a", + "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#708a6cc17d6821fad222a652084f2bc270c40052", "@pdpp/collector-runtime": "*", "@pdpp/connector-protocol": "*", "@pdpp/display": "*", diff --git a/reference-implementation/server/connector-install/index.ts b/reference-implementation/server/connector-install/index.ts index dcf2c284a..48bd559cc 100644 --- a/reference-implementation/server/connector-install/index.ts +++ b/reference-implementation/server/connector-install/index.ts @@ -94,13 +94,13 @@ export interface ConnectorCatalogSnapshot { interface InstallerCore { DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY: string; - fetchCatalog?: (options: Record) => Promise; - fetchResolvedArtifact?: ( - source: Record, + fetchCatalog: (options: Record) => Promise; + fetchResolvedArtifact: ( + source: null, entry: Record, options: Record ) => Promise; - installFromLock: (options: Record) => Promise; + installFromLock: (options: Record & { source: null }) => Promise; } export interface ConnectorInstallStore { @@ -1109,9 +1109,6 @@ export function createConnectorInstallService(options: { return null; } const core = await loadPinnedCore(); - if (!core.fetchResolvedArtifact) { - throw new Error("Pinned connector installer core does not expose fetchResolvedArtifact."); - } const identityResolver = ({ registry, repository }: { registry: string; repository: string }) => { if (registry === "ghcr.io" && repository === `pdp-connect/connector/${entry.connector_key}`) { return core.DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY; @@ -1120,7 +1117,7 @@ export function createConnectorInstallService(options: { }; const limitedTransport = createConfigLimitedFetch(fetch, entry.digest); const artifact = await core.fetchResolvedArtifact( - { doc: {}, mode: "locked" }, + null, { artifactKind: "pdpp-collection-profile", connectorId: entry.connector_id, @@ -1219,9 +1216,6 @@ async function loadPinnedCore(): Promise { async function loadCatalogFromPinnedCore(lastAcceptedGeneratedAt: string | null): Promise { const core = await loadPinnedCore(); - if (!core.fetchCatalog) { - throw new Error("Pinned connector installer core does not expose fetchCatalog."); - } const value = (await core.fetchCatalog({ identity: core.DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY, registry: "ghcr.io", @@ -1399,9 +1393,6 @@ async function installPinnedArtifact( const core = await loadPinnedCore(); const abortableFetch: FetchLike = (input, init) => fetch(input, { ...init, signal: init?.signal ? AbortSignal.any([init.signal, signal]) : signal }); - if (!core.fetchResolvedArtifact) { - throw new Error("Pinned connector installer core does not expose fetchResolvedArtifact."); - } const identityResolver = ({ registry, repository }: { registry: string; repository: string }) => { if (registry === "ghcr.io" && repository === `pdp-connect/connector/${entry.connector_key}`) { return core.DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY; @@ -1410,7 +1401,7 @@ async function installPinnedArtifact( }; const preflightTransport = createConfigLimitedFetch(abortableFetch, entry.digest); const preflight = await core.fetchResolvedArtifact( - { doc: {}, mode: "locked" }, + null, { artifactKind: "pdpp-collection-profile", connectorId: entry.connector_id, @@ -1447,7 +1438,6 @@ async function installPinnedArtifact( } const limitedTransport = createConfigLimitedFetch(abortableFetch, entry.digest); await core.installFromLock({ - artifactCertificateIdentityResolver: () => null, fetchImpl: limitedTransport.fetchImpl, installRoot: root, layout: "source", @@ -1476,7 +1466,7 @@ async function installPinnedArtifact( registry === "ghcr.io" && repository === `pdp-connect/connector/${entry.connector_key}` ? core.DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY : null, - source: { doc: {}, mode: "locked" }, + source: null, }); const installedConfigDigest = limitedTransport.configDigest(); if (!installedConfigDigest || installedConfigDigest !== configDigest) { diff --git a/scripts/resolve-connectors.js b/scripts/resolve-connectors.js index 9b4c263ac..375c6af72 100644 --- a/scripts/resolve-connectors.js +++ b/scripts/resolve-connectors.js @@ -17,17 +17,29 @@ import { import { basename, dirname, join, relative, resolve, sep } from "path" import { fileURLToPath } from "url" import { - DEFAULT_CONNECTOR_INDEX_URL, DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY, fetchResolvedArtifact, - generateLock, installFromLock, - loadConnectorIndex, readJson, - satisfies, sha256Buffer, verifyInstalled, } from "@opendatalabs/data-connectors-tools/installer-core" +// STOPGAP, removed when the legacy tarball connectors move to Collection +// Profiles. The current installer core installs only signed OCI Collection +// Profiles that carry a signed source declaration. The bundled tarball +// connectors, and the one OCI release published before source declarations +// existed (see PRE_DECLARATION_OCI_CONNECTORS), still need the earlier core, +// which is pinned under its own name so it cannot be mistaken for the current +// one. +import { + DEFAULT_CONNECTOR_INDEX_URL, + fetchResolvedArtifact as legacyFetchResolvedArtifact, + generateLock, + installFromLock as legacyInstallFromLock, + loadConnectorIndex, + satisfies, + verifyInstalled as legacyVerifyInstalled, +} from "data-connectors-legacy-installer/installer-core" import { isMainModule } from "./is-main-module.js" const __dirname = dirname(fileURLToPath(import.meta.url)) @@ -55,11 +67,76 @@ const NON_CONNECTOR_FILES = new Set([ "types", ]) +// What the earlier installer core reads in place of a connector index. export const LOCKED_ARTIFACT_SOURCE = Object.freeze({ mode: "locked", doc: Object.freeze({}), }) +// STOPGAP. OCI connectors whose pinned release has no signed source +// declaration, so the current installer core refuses it. They install and +// author through the earlier core, like the tarball connectors. Delete an id +// here once its connector is re-pinned to a release that carries a declaration. +export const PRE_DECLARATION_OCI_CONNECTORS = Object.freeze( + new Set(["github-pdpp"]) +) + +const usesEarlierInstallerCore = entry => + !entry.oci || PRE_DECLARATION_OCI_CONNECTORS.has(entry.connectorId) + +function splitLockByInstallerCore(lock) { + const earlier = lock.connectors.filter(usesEarlierInstallerCore) + const current = lock.connectors.filter( + entry => !usesEarlierInstallerCore(entry) + ) + return { + earlier: { ...lock, connectors: earlier }, + current: { ...lock, connectors: current }, + } +} + +// Install each entry with the core that accepts it. The two cores write +// disjoint paths (`/...` and `collection-profiles//...`). +export async function installLockedConnectors({ lock, source, ...options }) { + const { earlier, current } = splitLockByInstallerCore(lock) + const results = [] + if (earlier.connectors.length > 0) { + results.push( + await legacyInstallFromLock({ lock: earlier, source, ...options }) + ) + } + if (current.connectors.length > 0) { + // The current core takes no connector index; `source` stays null. + results.push( + await installFromLock({ lock: current, source: null, ...options }) + ) + } + return { + connectorCount: results.reduce((n, r) => n + r.connectorCount, 0), + filesWritten: results.reduce((n, r) => n + r.filesWritten, 0), + } +} + +async function verifyLockedConnectors({ lock, source, ...options }) { + const { earlier, current } = splitLockByInstallerCore(lock) + const results = [] + if (earlier.connectors.length > 0) { + results.push( + await legacyVerifyInstalled({ lock: earlier, source, ...options }) + ) + } + if (current.connectors.length > 0) { + results.push( + await verifyInstalled({ lock: current, source: null, ...options }) + ) + } + return { + ok: results.every(r => r.ok), + missing: results.flatMap(r => r.missing), + mismatched: results.flatMap(r => r.mismatched), + } +} + // Absent an explicit `--index-url`, stay pinned to whatever the existing lock // already resolved against — for BOTH `--check` and a normal (re)generation. // A bare `node scripts/resolve-connectors.js` must not silently float the 12 @@ -280,7 +357,7 @@ export async function installConnectorsAtomically({ lock, source = LOCKED_ARTIFACT_SOURCE, installRoot, - install = installFromLock, + install = installLockedConnectors, ...options }) { recoverInterruptedInstall(installRoot) @@ -432,11 +509,18 @@ export async function authorOciProfile( repository: `pdp-connect/connector/${connectorKey}`, }, } - const artifact = await fetchResolvedArtifact( - LOCKED_ARTIFACT_SOURCE, - candidate, - { allowTagResolution: true, ociCertificateIdentityResolver, ...options } - ) + const fetchOptions = { + allowTagResolution: true, + ociCertificateIdentityResolver, + ...options, + } + const artifact = PRE_DECLARATION_OCI_CONNECTORS.has(connectorId) + ? await legacyFetchResolvedArtifact( + LOCKED_ARTIFACT_SOURCE, + candidate, + fetchOptions + ) + : await fetchResolvedArtifact(null, candidate, fetchOptions) const { registry, repository, digest, configDigest } = artifact.oci return { ...candidate, @@ -630,7 +714,7 @@ async function main() { } if ( !reportInstalledCheck( - await verifyInstalled({ + await verifyLockedConnectors({ lock, source, installRoot: CONNECTORS_DIR, diff --git a/scripts/resolve-connectors.test.mjs b/scripts/resolve-connectors.test.mjs index a62fb5852..fb87fe917 100644 --- a/scripts/resolve-connectors.test.mjs +++ b/scripts/resolve-connectors.test.mjs @@ -17,13 +17,14 @@ import { tmpdir } from "node:os" import { dirname, join, resolve } from "node:path" import { pathToFileURL } from "node:url" import { describe, expect, it } from "vitest" -import { installFromLock } from "@opendatalabs/data-connectors-tools/installer-core" +import { installFromLock as legacyInstallFromLock } from "data-connectors-legacy-installer/installer-core" import { artifactCertificateIdentityResolver, authorOciLock, authorOciProfile, checkInstalledLock, installConnectorsAtomically, + installLockedConnectors, ociCertificateIdentityResolver, LOCKED_ARTIFACT_SOURCE, resolveIndexUrl, @@ -36,6 +37,9 @@ const installerCorePath = const { readTarGzEntries } = await import( pathToFileURL(join(dirname(installerCorePath), "tar-stream.mjs")).href ) +const { buildSourceDeclaration, serializeSourceDeclaration } = await import( + pathToFileURL(join(dirname(installerCorePath), "source-declaration.mjs")).href +) const lock = JSON.parse(readFileSync("connectors/lock.json", "utf8")) const npmPackage = JSON.parse(readFileSync("package.json", "utf8")) @@ -47,7 +51,11 @@ const legacyArtifacts = lock.connectors.filter(connector => // This is the reviewed data-connectors main commit containing the duplicate // normalized-member guard. Keep the ancestry check local and deterministic; // acceptance must not turn into a live git or network lookup. -const DATA_CONNECTORS_MAIN_ANCESTOR = "6c71697a1523926fe8d475cd243a8219ea21aa5b" +const DATA_CONNECTORS_MAIN_ANCESTOR = "708a6cc17d6821fad222a652084f2bc270c40052" +// Last commit whose installer core still reads the 12 retired tarball entries +// and the one OCI release that predates signed source declarations. +const EARLIER_INSTALLER_CORE_ANCESTOR = + "e1e497f3bca842c0489c472c1ca286630eef5078" describe("connector artifact signer identities", () => { it("trusts only the six exact legacy artifact URLs retained by the lock", () => { @@ -203,6 +211,32 @@ describe("resolved connector-installer archive acceptance", () => { expect(lockCommit).toBe(DATA_CONNECTORS_MAIN_ANCESTOR) }) + it("pins the earlier installer core to one commit in package.json and the lockfile", () => { + const dependency = "data-connectors-legacy-installer" + const packageCommit = + npmPackage.devDependencies[dependency]?.match(/#([0-9a-f]{40})$/)?.[1] + const lockCommit = npmLock.packages[`node_modules/${dependency}`]?.resolved?.match( + /#([0-9a-f]{40})$/ + )?.[1] + expect(packageCommit).toBe(EARLIER_INSTALLER_CORE_ANCESTOR) + expect(lockCommit).toBe(EARLIER_INSTALLER_CORE_ANCESTOR) + }) + + it("keeps one current-core pin across the root, reference implementation and collector packages", () => { + const dependency = "@opendatalabs/data-connectors-tools" + const read = path => JSON.parse(readFileSync(path, "utf8")) + const specs = [ + npmPackage.devDependencies[dependency], + read("reference-implementation/package.json").dependencies?.[dependency] ?? + read("reference-implementation/package.json").devDependencies?.[dependency], + read("packages/local-collector/package.json").dependencies?.[dependency] ?? + read("packages/local-collector/package.json").devDependencies?.[dependency], + ] + for (const spec of specs) { + expect(spec).toMatch(new RegExp(`#${DATA_CONNECTORS_MAIN_ANCESTOR}$`)) + } + }) + it("refuses two differing collection-profile.mjs archive members through the resolved reader", async () => { await temporary(async root => { const archive = archiveMembers(root, "duplicate-entrypoint", [ @@ -262,7 +296,7 @@ describe("resolved connector-installer archive acceptance", () => { } const installRoot = join(root, "installed") - await installFromLock({ + await legacyInstallFromLock({ lock: { connectors: [entry] }, source: { mode: "local", rootDir: root, doc: {} }, installRoot, @@ -304,7 +338,27 @@ function fixture(root) { version, protocol_version: "1.0", display_name: "YNAB", + source: { + id: "https://registry.pdpp.dev/sources/ynab", + display: { name: "YNAB" }, + }, + streams: [ + { + name: "budgets", + primary_key: ["id"], + semantics: "mutable_state", + selection: { fields: true, resources: true }, + schema: { + type: "object", + properties: { id: { type: "string" } }, + required: ["id"], + }, + }, + ], }) + const declaration = serializeSourceDeclaration( + buildSourceDeclaration([JSON.parse(profile)]) + ) const code = Buffer.from("export const collect = () => {};\n") const provenance = json({ connector_key: key, version }) const files = { @@ -337,6 +391,7 @@ function fixture(root) { ...JSON.parse(profile), config_version: "1.0", profile_digest: digest(profile), + source_declaration_digest: digest(declaration), entrypoint: "code/collection-profile.mjs", }) const manifest = json({ @@ -354,6 +409,10 @@ function fixture(root) { "application/vnd.pdpp.connector.licenses.v1.tar+gzip" ), layer(provenance, "application/vnd.pdpp.connector.provenance.v1+json"), + layer( + declaration, + "application/vnd.pdpp.connector.source-declaration.v1+json" + ), ], }) const manifestDigest = digest(manifest) From 205210ab2d678aa8808777ed502dc7c7f342013b Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Thu, 5 Nov 2026 15:58:16 -0600 Subject: [PATCH 2/8] build: re-pin Collection Profiles to releases with a signed source declaration The installer core refuses an artifact that has no signed source declaration. Eight pins predate it and fail that check. Each moves to a release the core accepts. Collector pins (reference server copies change with them): claude-code 0.3.0 to 0.3.4, codex 0.3.0 to 0.3.3, google-takeout 0.1.0 to 0.1.3, apple-photos 0.1.0 to 0.1.3, imessage 0.1.0 to 0.1.25. Lock entries: apple-health 0.1.0 to 0.1.17, chatgpt 0.1.0 to 0.2.5, ynab 0.3.0 to 0.3.26. github-pdpp 0.5.0 and the 12 tarball entries are unchanged. Rule: the oldest release that carries a declaration and still emits every stream of the release it replaces, unless a newer one fixes a defect that can be named. apple-health, imessage and ynab take 0.1.17, 0.1.25 and 0.3.26 for two fixes in the shared runtime (time-bounded runs, and failing an unproven connector read). chatgpt stays below 0.3.0, which changes every message record id without re-keying stored records, and below 0.2.56, which needs a host that supports BLOB; the local collector does not advertise it (collector-runtime runtime-capabilities.ts), so it would refuse to place such a connector. The new releases change the contract in ways the tests now pin: - claude-code adds the required `usage` stream to profile `full`, and makes `sessions.kind` and `messages.has_pasted_content` required. The generated collector definition now requests `usage`; without it the collector would never emit a stream the server calls required, and the coverage rollup would stay `unknown`. - chatgpt 0.2.5 drops the `setup` block, so a credential sealed under 0.1.0 is no longer read or injected, and stays stored. - A server that still holds the previous profile refuses `usage` at ingest; after the new profile is registered the same connection keeps its id, checkpoint and records, and its coverage proof is re-earned by the next run (`generation_mismatch` until then). Pass rule (from the repository root): (cd reference-implementation && node --import tsx --test \ test/static-secret-run-credentials.test.ts \ test/collection-report-projection.test.ts \ test/collector-profile-upgrade.test.ts \ test/local-collector-profiles.test.ts) (cd packages/local-collector && npx tsx --test test/collection-profiles.test.ts) All pass: 134 tests, and 15 tests with 1 skipped (the live GHCR test). Assisted-by: AI Signed-off-by: Tim Nunamaker --- connectors/connector-dependencies.json | 6 +- connectors/lock.json | 50 +- .../collection-profile-pins.generated.ts | 40 +- .../collector-definitions.generated.ts | 2 +- .../test/collection-profiles.test.ts | 18 + .../apple-photos.json | 10 +- .../apple-photos.pin.json | 6 +- .../local-collector-profiles/claude-code.json | 207 +++- .../claude-code.pin.json | 6 +- .../local-collector-profiles/codex.json | 10 +- .../local-collector-profiles/codex.pin.json | 6 +- .../google-takeout.json | 12 +- .../google-takeout.pin.json | 6 +- .../local-collector-profiles/imessage.json | 10 +- .../imessage.pin.json | 6 +- .../test/collection-report-projection.test.ts | 28 + .../test/collector-profile-upgrade.test.ts | 284 ++++++ .../claude-code-0.3.0.json | 880 ++++++++++++++++++ .../static-secret-run-credentials.test.ts | 53 ++ 19 files changed, 1560 insertions(+), 80 deletions(-) create mode 100644 reference-implementation/test/collector-profile-upgrade.test.ts create mode 100644 reference-implementation/test/fixtures/collector-profile-upgrade/claude-code-0.3.0.json diff --git a/connectors/connector-dependencies.json b/connectors/connector-dependencies.json index fe988352c..b26f1fd3c 100644 --- a/connectors/connector-dependencies.json +++ b/connectors/connector-dependencies.json @@ -15,9 +15,9 @@ "wholefoods-playwright": "1.0.0", "youtube-playwright": "1.0.0", "claude-export-playwright": "2.0.1", - "apple-health-pdpp": "0.1.0", - "chatgpt-pdpp": "0.1.0", + "apple-health-pdpp": "0.1.17", + "chatgpt-pdpp": "0.2.5", "github-pdpp": "0.5.0", - "ynab-pdpp": "0.3.0" + "ynab-pdpp": "0.3.26" } } diff --git a/connectors/lock.json b/connectors/lock.json index c3d9584bf..becd52bc5 100644 --- a/connectors/lock.json +++ b/connectors/lock.json @@ -1,7 +1,7 @@ { "lockVersion": "2.0", "dependencyFile": "connectors/connector-dependencies.json", - "generatedAt": "2026-09-15T19:43:18.976Z", + "generatedAt": "2026-11-05T20:25:11.519Z", "sourceRepo": "https://github.com/PDP-Connect/data-connectors", "sourceTag": "mixed", "sourceCommit": "mixed", @@ -25,10 +25,10 @@ "wholefoods-playwright": "1.0.0", "youtube-playwright": "1.0.0", "claude-export-playwright": "2.0.1", - "apple-health-pdpp": "0.1.0", - "chatgpt-pdpp": "0.1.0", + "apple-health-pdpp": "0.1.17", + "chatgpt-pdpp": "0.2.5", "github-pdpp": "0.5.0", - "ynab-pdpp": "0.3.0" + "ynab-pdpp": "0.3.26" }, "connectors": [ { @@ -38,20 +38,20 @@ "manifestPath": "profile/collection-profile.json", "entrypointPath": "dist/collection-profile.mjs", "provenancePath": "provenance.json", - "version": "0.1.0", - "resolvedFrom": "0.1.0", + "version": "0.1.17", + "resolvedFrom": "0.1.17", "oci": { "registry": "ghcr.io", "repository": "pdp-connect/connector/apple-health", - "digest": "sha256:1e497ac3d5cbc868f27ddac42e792e9c2da929438ed7e38d24aa8ac84777ad8e", - "configDigest": "sha256:e1270b82e9ea0a24fc9d66c4f9cbfb775c74355c88928a0d3e51c95482b179fd" + "digest": "sha256:1555695184a65da4c8630ce47f18f7747cbdbfd6db14b09b48fe3ab1672299ec", + "configDigest": "sha256:e4c88e4a2688c4e9f86575f3f9567745a36e3839f1e60ca9402c8fa192a2cf6f" }, "company": "apple", "name": "Apple Health", "description": "Collects Apple Health export data through the PDPP Collection Profile protocol.", - "manifestSha256": "sha256:91d7e8af7ece79089c2e1b857bb15770fdccdcc5e80e767b3f6c7d072a5bc600", - "entrypointSha256": "sha256:7513de5db4e94deb75d82c7fdaae08add1689732d66952b156d4ecf21419cea7", - "provenanceSha256": "sha256:682be3f0a126e8ab148f43dd5ff8fa13cdc490ee78c0cd519d33a32c8b34de3b" + "manifestSha256": "sha256:83e71487ef1905966f00bd286f8139d5232d98830f722cfda4fe9770814dee9f", + "entrypointSha256": "sha256:6227d0fda6985887c2472d62d2bfd4eb7478e5257986d9287a3a2bee9208de8d", + "provenanceSha256": "sha256:d3e097df9d79c2aa522012b2c55460a98918ab2b78b73d582b6fb681dfe1cb46" }, { "connectorId": "chatgpt-pdpp", @@ -60,20 +60,20 @@ "manifestPath": "profile/collection-profile.json", "entrypointPath": "dist/collection-profile.mjs", "provenancePath": "provenance.json", - "version": "0.1.0", - "resolvedFrom": "0.1.0", + "version": "0.2.5", + "resolvedFrom": "0.2.5", "oci": { "registry": "ghcr.io", "repository": "pdp-connect/connector/chatgpt", - "digest": "sha256:811daaf8f2a9a346f1ff5109c118ea404212c8e00790cc5d448e89ad9fa9c4a4", - "configDigest": "sha256:8f1bc246633ae908636d841f8cf737d17daf0144207a375b694d8eea9ee1e30d" + "digest": "sha256:727b4469ad71ba3cd9f71d0e72191214c5439c189acf02c1f6949a41378c1ea5", + "configDigest": "sha256:aa81039f0ef1bd8997bebbdd49e3fd675e015a5d22aa346beef9d2436a05aeec" }, "company": "openai", "name": "ChatGPT (PDPP Collection Profile)", "description": "Collects ChatGPT conversations, messages, memories, custom GPTs, custom instructions, and shared conversations through the PDPP Collection Profile protocol.", - "manifestSha256": "sha256:eb491a782ba61925340c5334d3b35a6d0111757f63ccda72bc466f4f84b8f417", - "entrypointSha256": "sha256:e50f93dee4d48925f623f3be04acaed87f77783f1255579c0db7618fabe1a963", - "provenanceSha256": "sha256:d9695809c157e431a725aea21d1f65b48b652c5f9cc3b2b18ac925d122e2dbc5" + "manifestSha256": "sha256:4901f84e8b07874d47405fcb5618ed2867ebcbb0050193d18625045889f3a92c", + "entrypointSha256": "sha256:0b3dcbeb58422874da31a621ed36a78e0f690431d8a63970b1082089bba1b8ac", + "provenanceSha256": "sha256:2a643baea69b48aa7c40ea665250df4d0841c392703343c051b96a37f42430ec" }, { "connectorId": "chatgpt-playwright", @@ -390,20 +390,20 @@ "manifestPath": "profile/collection-profile.json", "entrypointPath": "dist/collection-profile.mjs", "provenancePath": "provenance.json", - "version": "0.3.0", - "resolvedFrom": "0.3.0", + "version": "0.3.26", + "resolvedFrom": "0.3.26", "oci": { "registry": "ghcr.io", "repository": "pdp-connect/connector/ynab", - "digest": "sha256:9da6a382500368ef28a8c9f59351d1f3ab6871297bf27418c7c63e8d9e241fad", - "configDigest": "sha256:20cc66f94d44748af7cc0c7eac1a20fa10e795b4a89f1b376e1827be947b8388" + "digest": "sha256:863fe543e7774252af4bd64a815fabada12a9a45dffaaebfca175411ffa4478d", + "configDigest": "sha256:32df5499e9e23a5639767fe7f04931808e42d78238829bac7af0df37cafa0738" }, "company": "ynab", "name": "YNAB", "description": "Collects YNAB budgets, accounts, categories, transactions, and other financial data through the PDPP Collection Profile protocol.", - "manifestSha256": "sha256:ce2e9b6badc9ff6b0573d36a1dd3875d24eefd69da4184179354613f71ffde9f", - "entrypointSha256": "sha256:9f90880f8dca821b8929305ddcf1a7948a8fbea1df66910e08298774cf3cb4b8", - "provenanceSha256": "sha256:0cab8f965bfaa5e9e0df643e424b2cd29e77b44324c2d738f2d8590d8583dc1d" + "manifestSha256": "sha256:c23d2b3a6c68beac5a8d082ed2843833f755794faaf4e629229230d447bec328", + "entrypointSha256": "sha256:58ba4e7c97899fb25918e8e90e4ca49fb59c9c0aad4a199b081b8b3cf862ce8d", + "provenanceSha256": "sha256:d0d7dd10baa9854d57f3cf076aed2d035d20e438f11158557f554888293f78be" }, { "connectorId": "youtube-playwright", diff --git a/packages/local-collector/src/generated/collection-profile-pins.generated.ts b/packages/local-collector/src/generated/collection-profile-pins.generated.ts index 925445c30..2e5527db2 100644 --- a/packages/local-collector/src/generated/collection-profile-pins.generated.ts +++ b/packages/local-collector/src/generated/collection-profile-pins.generated.ts @@ -14,41 +14,41 @@ export const COLLECTION_PROFILE_PINS: readonly CollectionProfilePin[] = Object.f { "connectorId": "claude_code", "connectorKey": "claude-code", - "version": "0.3.0", - "digest": "sha256:5b8866392bb11f24cbc3827e37e96faae9b30c8863d7d82bf9cd21aa2e636c82", - "profileSha256": "sha256:b6398b12d41043706ac4701db92f577ed9964d86b6e4776d471eca65587ae79f", - "entrypointSha256": "sha256:2a76a5b518c474a2a342bd2511ad825aec289c1310b2575801a419813bb9bea3" + "version": "0.3.4", + "digest": "sha256:08a82d8da8ed4eb70551007141383d3afaa9dd9878afb4fa76543ecfd53ee447", + "profileSha256": "sha256:1d9f5132894c8c14e5aa296b55f90c60414ef772b0fa6f6b3c0d9b55525b1381", + "entrypointSha256": "sha256:bbc13637cd863ec774d92c4f43dd926ff5cec39d1abe25d2b881023353f05072" }, { "connectorId": "codex", "connectorKey": "codex", - "version": "0.3.0", - "digest": "sha256:bb75129071d1afcf60bf5e20097b84e1262d3cb2ce3ca5b3c228d23f00495d28", - "profileSha256": "sha256:3287a16b01517439443e8641e97c7f1bfcb358d01641d3b3490331b2ddc107ab", - "entrypointSha256": "sha256:0ae6336d89b88d50ae6a13d11e62d4b844a623d76cfeb7f83dcd29c889764b3b" + "version": "0.3.3", + "digest": "sha256:8cbea0c5b56578ab94cc830bd5c69dce77aa5927106f8401d44a84fd3e39fdab", + "profileSha256": "sha256:f8d1773bb2b856c278748f512aba16fbc993b9a833fbfa9d50c357b7a232776f", + "entrypointSha256": "sha256:20e26674be1ec1ea64ec13bfa6b62717c85a6e474502e70a9f0e8e225000b6e3" }, { "connectorId": "google_takeout", "connectorKey": "google-takeout", - "version": "0.1.0", - "digest": "sha256:38d4bd1312073b167b668cad1701ebc78fea8cf9c19a0edb37a98d11fd660562", - "profileSha256": "sha256:708f149f0efedae0b41cc3f67a065c541ec6ab5305edb322b9f983a9d0913353", - "entrypointSha256": "sha256:710d01455fa0cc146665de92612682b749810b54f8f72d5e235a5538c4f43fda" + "version": "0.1.3", + "digest": "sha256:9d1f4934c42cddd73a00b2792ee7edcc389058c566e462a00cd8210fcb014653", + "profileSha256": "sha256:d645fffba61b0a47a077dc4fe4d04a803481844b3281d333818ab4532d9f01cd", + "entrypointSha256": "sha256:8eafae144013735b8dc44b6ff40d873feed95cbc934d50c08ad68937306f7d99" }, { "connectorId": "imessage", "connectorKey": "imessage", - "version": "0.1.0", - "digest": "sha256:ace3798a8e222e73fa1555a39ba52b245e2566d31d84bb60fdec61c9ae1a9ca6", - "profileSha256": "sha256:654eb8abb9625554c19edcf7e1ef6416fca556929456454ba29277751cde8efb", - "entrypointSha256": "sha256:aee91cec10278cca82acc000d504420f2135eac2e44838d3b4693d08db5f3d84" + "version": "0.1.25", + "digest": "sha256:920246f08102aea9c7958d6b3a0bf89eb07e3b776a921333f6d7ae93f99394b0", + "profileSha256": "sha256:399958b7d89e1263b5f61e6b09de4c7f5d6a81c1084ff887e75a504278820ef4", + "entrypointSha256": "sha256:2875a87d04b77e52b9b880a651814112db1b38cd97bd94dab7948a72666670a1" }, { "connectorId": "apple_photos", "connectorKey": "apple-photos", - "version": "0.1.0", - "digest": "sha256:0952cef101b05b5416d0ea0620ee12c857f493bae547ce71119a243131741415", - "profileSha256": "sha256:1d941bd3fc02831618fcd29b899b091ed2ce51f25535f35f58c5a2aaa37ce596", - "entrypointSha256": "sha256:eccd685d0d5d9b9c675365b0b3073c46028067aac5bceb2719770e5069bf8ca2" + "version": "0.1.3", + "digest": "sha256:3e58cf196d7fc119fa7f462432d069a257f5e62036a7c70c6b542bb7b779f07f", + "profileSha256": "sha256:d290a1d201b9b38955225e4b5cae6054a84097723dac8d600e06c5db4676036a", + "entrypointSha256": "sha256:8a2f1b66106b3574d1b675db9ee754904ac52f00921dfb174ab4ca7ededbaff2" }, ]); diff --git a/packages/local-collector/src/generated/collector-definitions.generated.ts b/packages/local-collector/src/generated/collector-definitions.generated.ts index 0ffc988ce..f49e38cb3 100644 --- a/packages/local-collector/src/generated/collector-definitions.generated.ts +++ b/packages/local-collector/src/generated/collector-definitions.generated.ts @@ -37,7 +37,7 @@ export const LOCAL_COLLECTOR_DEFINITIONS: readonly LocalCollectorDefinition[] = "filesystem": { required: true }, }, protocol_capabilities: [], - streams: ["sessions", "messages", "attachments", "memory_notes", "skills", "slash_commands", "file_history", "cache_inventory", "coverage_diagnostics", "backup_inventory", "config_inventory"], + streams: ["sessions", "messages", "attachments", "memory_notes", "skills", "slash_commands", "usage", "file_history", "cache_inventory", "coverage_diagnostics", "backup_inventory", "config_inventory"], time_scopable_streams: ["sessions", "messages", "attachments"], source_root_scopable_streams: ["sessions", "messages", "attachments"], enforces_source_roots: true, diff --git a/packages/local-collector/test/collection-profiles.test.ts b/packages/local-collector/test/collection-profiles.test.ts index adf659d0e..c1a7372c6 100644 --- a/packages/local-collector/test/collection-profiles.test.ts +++ b/packages/local-collector/test/collection-profiles.test.ts @@ -101,6 +101,24 @@ test("each pinned profile is the release it pins and declares its definition's b } }); +test("a collector's default streams include every required stream of its pinned profile", () => { + // A connector emits a stream only when it is requested, and the server counts + // a required stream that is never reported as unknown coverage. A pin that + // adds a required stream must therefore add it to the default stream set. + for (const pin of COLLECTION_PROFILE_PINS) { + const definition = LOCAL_COLLECTOR_DEFINITIONS.find((candidate) => candidate.connector_id === pin.connectorId); + assert.ok(definition, pin.connectorId); + const profile = JSON.parse( + readFileSync(new URL(`${pin.connectorKey}.json`, REFERENCE_PROFILE_DIR), "utf8") + ) as { streams: { name: string; required?: boolean }[] }; + const requested = new Set(definition.streams); + const missing = profile.streams + .filter((stream) => stream.required !== false && !requested.has(stream.name)) + .map((stream) => stream.name); + assert.deepEqual(missing, [], `${pin.connectorId} never requests its required streams`); + } +}); + test("the collector build compiles no connector code", () => { const tsconfig = readFileSync(new URL("../tsconfig.build.json", import.meta.url), "utf8"); assert.doesNotMatch(tsconfig, /polyfill-connectors\/(connectors|src)/); diff --git a/reference-implementation/server/local-collector-profiles/apple-photos.json b/reference-implementation/server/local-collector-profiles/apple-photos.json index 42e5d2c8e..c9b26f3be 100644 --- a/reference-implementation/server/local-collector-profiles/apple-photos.json +++ b/reference-implementation/server/local-collector-profiles/apple-photos.json @@ -1,12 +1,18 @@ { "protocol_version": "0.1.0", "connector_id": "https://registry.pdpp.dev/connectors/apple-photos", + "source": { + "id": "https://registry.pdpp.dev/sources/apple_photos", + "display": { + "name": "Apple Photos" + } + }, "connector_key": "apple-photos", "manifest_uri": "https://registry.pdpp.dev/connectors/apple-photos", - "version": "0.1.0", + "version": "0.1.3", "display_name": "Apple Photos", "brand": { - "icon": "icons/apple_photos.svg", + "icon": "icon.svg", "background_color": "#FA633E" }, "runtime_requirements": { diff --git a/reference-implementation/server/local-collector-profiles/apple-photos.pin.json b/reference-implementation/server/local-collector-profiles/apple-photos.pin.json index c625dfd1f..afa614a7e 100644 --- a/reference-implementation/server/local-collector-profiles/apple-photos.pin.json +++ b/reference-implementation/server/local-collector-profiles/apple-photos.pin.json @@ -1,6 +1,6 @@ { "connector_key": "apple-photos", - "version": "0.1.0", - "digest": "sha256:0952cef101b05b5416d0ea0620ee12c857f493bae547ce71119a243131741415", - "profile_sha256": "sha256:1d941bd3fc02831618fcd29b899b091ed2ce51f25535f35f58c5a2aaa37ce596" + "version": "0.1.3", + "digest": "sha256:3e58cf196d7fc119fa7f462432d069a257f5e62036a7c70c6b542bb7b779f07f", + "profile_sha256": "sha256:d290a1d201b9b38955225e4b5cae6054a84097723dac8d600e06c5db4676036a" } diff --git a/reference-implementation/server/local-collector-profiles/claude-code.json b/reference-implementation/server/local-collector-profiles/claude-code.json index 11b6cf048..954725a6f 100644 --- a/reference-implementation/server/local-collector-profiles/claude-code.json +++ b/reference-implementation/server/local-collector-profiles/claude-code.json @@ -1,12 +1,18 @@ { "protocol_version": "0.1.0", "connector_id": "https://registry.pdpp.dev/connectors/claude-code", + "source": { + "id": "https://registry.pdpp.dev/sources/claude_code", + "display": { + "name": "Claude Code" + } + }, "connector_key": "claude-code", "manifest_uri": "https://registry.pdpp.dev/connectors/claude-code", - "version": "0.3.0", + "version": "0.3.4", "display_name": "Claude Code", "brand": { - "icon": "icons/claude_code.svg", + "icon": "icon.svg", "background_color": "#D97757" }, "runtime_requirements": { @@ -97,14 +103,25 @@ "message_count": { "type": ["integer", "null"] }, + "title": { + "type": ["string", "null"], + "x_pdpp_role": "secondary" + }, "user_type": { "type": ["string", "null"] }, "entrypoint": { "type": ["string", "null"] + }, + "kind": { + "type": "string", + "enum": ["session", "subagent"] + }, + "parent_session_id": { + "type": ["string", "null"] } }, - "required": ["id"] + "required": ["id", "kind"] }, "primary_key": ["id"], "cursor_field": "last_event_at", @@ -114,6 +131,14 @@ "resources": true }, "incremental": true, + "relationships": [ + { + "name": "parent_session", + "stream": "sessions", + "foreign_key": "parent_session_id", + "cardinality": "has_one" + } + ], "query": { "search": { "lexical_fields": [ @@ -121,9 +146,11 @@ "cwd", "git_branch", "version", + "title", "user_type", "entrypoint" - ] + ], + "semantic_fields": ["title"] }, "range_filters": { "started_at": ["gte", "gt", "lte", "lt"], @@ -186,9 +213,15 @@ }, "agent_id": { "type": ["string", "null"] + }, + "subagent_session_id": { + "type": ["string", "null"] + }, + "has_pasted_content": { + "type": "boolean" } }, - "required": ["id", "session_id"] + "required": ["id", "session_id", "has_pasted_content"] }, "primary_key": ["id"], "cursor_field": "timestamp", @@ -204,6 +237,12 @@ "stream": "sessions", "foreign_key": "session_id", "cardinality": "has_one" + }, + { + "name": "subagent_session", + "stream": "sessions", + "foreign_key": "subagent_session_id", + "cardinality": "has_one" } ], "query": { @@ -272,6 +311,27 @@ "type": ["string", "null"], "format": "date-time", "x_pdpp_role": "event-time" + }, + "content_binary_reason": { + "type": ["string", "null"] + }, + "blob_ref": { + "type": ["object", "null"], + "properties": { + "blob_id": { "type": "string" }, + "mime_type": { "type": "string" }, + "size_bytes": { "type": "integer" }, + "sha256": { "type": "string" } + }, + "required": ["blob_id", "mime_type", "size_bytes", "sha256"], + "additionalProperties": true + }, + "artifact_capture": { + "type": ["string", "null"], + "enum": ["captured", "failed", "unavailable", null] + }, + "artifact_sha256": { + "type": ["string", "null"] } }, "required": ["id", "session_id"] @@ -507,6 +567,140 @@ "coverage_strategy": "snapshot_import_receipt", "freshness_strategy": "device_heartbeat" }, + { + "name": "usage", + "required": true, + "description": "Aggregate Claude Code usage from ~/.claude/stats-cache.json: total sessions and messages, per-model token/cost totals, day-by-day activity, hour-of-day distribution, and the longest session.", + "compaction_class": "recurring_snapshot", + "compaction_class_note": "single aggregate record re-derived in full from stats-cache.json every run; never compactable, never re-alarms as growth accumulates.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Claude Code usage", + "detail": "Total sessions and messages, per-model token and cost totals, daily activity, hour-of-day distribution, and your longest session." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "total_sessions": { + "type": ["integer", "null"] + }, + "total_messages": { + "type": ["integer", "null"] + }, + "first_session_date": { + "type": ["string", "null"], + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "last_computed_date": { + "type": ["string", "null"], + "format": "date" + }, + "total_speculation_time_saved_ms": { + "type": ["integer", "null"] + }, + "total_cost_usd_cents": { + "type": "integer" + }, + "currency": { + "type": "string" + }, + "models": { + "type": "array", + "items": { + "type": "object", + "properties": { + "model": { "type": "string" }, + "input_tokens": { "type": "integer" }, + "output_tokens": { "type": "integer" }, + "cache_read_input_tokens": { "type": "integer" }, + "cache_creation_input_tokens": { "type": "integer" }, + "web_search_requests": { "type": "integer" }, + "cost_usd_cents": { "type": "integer" }, + "context_window": { "type": ["integer", "null"] }, + "max_output_tokens": { "type": ["integer", "null"] } + } + } + }, + "daily_activity": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { "type": "string", "format": "date" }, + "message_count": { "type": "integer" }, + "session_count": { "type": "integer" }, + "tool_call_count": { "type": "integer" } + } + } + }, + "daily_model_tokens": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { "type": "string", "format": "date" }, + "tokens_by_model": { "type": "object" } + } + } + }, + "hour_counts": { + "type": "object" + }, + "longest_session": { + "type": ["object", "null"], + "properties": { + "session_id": { "type": ["string", "null"] }, + "duration_seconds": { "type": ["integer", "null"] }, + "message_count": { "type": ["integer", "null"] }, + "timestamp": { "type": ["string", "null"], "format": "date-time" } + } + }, + "source": { + "type": "string" + } + }, + "required": [ + "id", + "total_cost_usd_cents", + "currency", + "models", + "daily_activity", + "daily_model_tokens", + "hour_counts", + "source" + ] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "search": { + "lexical_fields": ["source"] + }, + "range_filters": { + "first_session_date": ["gte", "gt", "lte", "lt"], + "last_computed_date": ["gte", "gt", "lte", "lt"], + "total_sessions": ["gte", "gt", "lte", "lt"], + "total_messages": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["first_session_date"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, { "name": "file_history", "description": "Metadata-only inventory for Claude Code file-history stores. Does not emit file-history payload content.", @@ -815,6 +1009,9 @@ }, { "name": "slash_commands" + }, + { + "name": "usage" } ] }, diff --git a/reference-implementation/server/local-collector-profiles/claude-code.pin.json b/reference-implementation/server/local-collector-profiles/claude-code.pin.json index 430406a60..3af964134 100644 --- a/reference-implementation/server/local-collector-profiles/claude-code.pin.json +++ b/reference-implementation/server/local-collector-profiles/claude-code.pin.json @@ -1,6 +1,6 @@ { "connector_key": "claude-code", - "version": "0.3.0", - "digest": "sha256:5b8866392bb11f24cbc3827e37e96faae9b30c8863d7d82bf9cd21aa2e636c82", - "profile_sha256": "sha256:b6398b12d41043706ac4701db92f577ed9964d86b6e4776d471eca65587ae79f" + "version": "0.3.4", + "digest": "sha256:08a82d8da8ed4eb70551007141383d3afaa9dd9878afb4fa76543ecfd53ee447", + "profile_sha256": "sha256:1d9f5132894c8c14e5aa296b55f90c60414ef772b0fa6f6b3c0d9b55525b1381" } diff --git a/reference-implementation/server/local-collector-profiles/codex.json b/reference-implementation/server/local-collector-profiles/codex.json index 62d967652..41c04e98f 100644 --- a/reference-implementation/server/local-collector-profiles/codex.json +++ b/reference-implementation/server/local-collector-profiles/codex.json @@ -1,12 +1,18 @@ { "protocol_version": "0.1.0", "connector_id": "https://registry.pdpp.dev/connectors/codex", + "source": { + "id": "https://registry.pdpp.dev/sources/codex", + "display": { + "name": "OpenAI Codex CLI" + } + }, "connector_key": "codex", "manifest_uri": "https://registry.pdpp.dev/connectors/codex", - "version": "0.3.0", + "version": "0.3.3", "display_name": "OpenAI Codex CLI", "brand": { - "icon": "icons/codex.svg", + "icon": "icon.svg", "background_color": "#10A37F" }, "runtime_requirements": { diff --git a/reference-implementation/server/local-collector-profiles/codex.pin.json b/reference-implementation/server/local-collector-profiles/codex.pin.json index b168c290a..4d7d81d05 100644 --- a/reference-implementation/server/local-collector-profiles/codex.pin.json +++ b/reference-implementation/server/local-collector-profiles/codex.pin.json @@ -1,6 +1,6 @@ { "connector_key": "codex", - "version": "0.3.0", - "digest": "sha256:bb75129071d1afcf60bf5e20097b84e1262d3cb2ce3ca5b3c228d23f00495d28", - "profile_sha256": "sha256:3287a16b01517439443e8641e97c7f1bfcb358d01641d3b3490331b2ddc107ab" + "version": "0.3.3", + "digest": "sha256:8cbea0c5b56578ab94cc830bd5c69dce77aa5927106f8401d44a84fd3e39fdab", + "profile_sha256": "sha256:f8d1773bb2b856c278748f512aba16fbc993b9a833fbfa9d50c357b7a232776f" } diff --git a/reference-implementation/server/local-collector-profiles/google-takeout.json b/reference-implementation/server/local-collector-profiles/google-takeout.json index b942a406d..392ff2a84 100644 --- a/reference-implementation/server/local-collector-profiles/google-takeout.json +++ b/reference-implementation/server/local-collector-profiles/google-takeout.json @@ -1,14 +1,16 @@ { "protocol_version": "0.1.0", "connector_id": "https://registry.pdpp.dev/connectors/google-takeout", + "source": { + "id": "https://registry.pdpp.dev/sources/google_takeout", + "display": { + "name": "Google Takeout" + } + }, "connector_key": "google-takeout", "manifest_uri": "https://registry.pdpp.dev/connectors/google-takeout", - "version": "0.1.0", + "version": "0.1.3", "display_name": "Google Takeout", - "brand": { - "icon": "icons/google_takeout.svg", - "background_color": "#FBBC05" - }, "runtime_requirements": { "bindings": { "network": { diff --git a/reference-implementation/server/local-collector-profiles/google-takeout.pin.json b/reference-implementation/server/local-collector-profiles/google-takeout.pin.json index c9a40966c..b648bb441 100644 --- a/reference-implementation/server/local-collector-profiles/google-takeout.pin.json +++ b/reference-implementation/server/local-collector-profiles/google-takeout.pin.json @@ -1,6 +1,6 @@ { "connector_key": "google-takeout", - "version": "0.1.0", - "digest": "sha256:38d4bd1312073b167b668cad1701ebc78fea8cf9c19a0edb37a98d11fd660562", - "profile_sha256": "sha256:708f149f0efedae0b41cc3f67a065c541ec6ab5305edb322b9f983a9d0913353" + "version": "0.1.3", + "digest": "sha256:9d1f4934c42cddd73a00b2792ee7edcc389058c566e462a00cd8210fcb014653", + "profile_sha256": "sha256:d645fffba61b0a47a077dc4fe4d04a803481844b3281d333818ab4532d9f01cd" } diff --git a/reference-implementation/server/local-collector-profiles/imessage.json b/reference-implementation/server/local-collector-profiles/imessage.json index ac221b9ac..960489d23 100644 --- a/reference-implementation/server/local-collector-profiles/imessage.json +++ b/reference-implementation/server/local-collector-profiles/imessage.json @@ -1,12 +1,18 @@ { "protocol_version": "0.1.0", "connector_id": "https://registry.pdpp.dev/connectors/imessage", + "source": { + "id": "https://registry.pdpp.dev/sources/imessage", + "display": { + "name": "iMessage" + } + }, "connector_key": "imessage", "manifest_uri": "https://registry.pdpp.dev/connectors/imessage", - "version": "0.1.0", + "version": "0.1.25", "display_name": "iMessage (macOS)", "brand": { - "icon": "icons/imessage.svg", + "icon": "icon.svg", "background_color": "#30D158" }, "runtime_requirements": { diff --git a/reference-implementation/server/local-collector-profiles/imessage.pin.json b/reference-implementation/server/local-collector-profiles/imessage.pin.json index c978468a2..5215000f6 100644 --- a/reference-implementation/server/local-collector-profiles/imessage.pin.json +++ b/reference-implementation/server/local-collector-profiles/imessage.pin.json @@ -1,6 +1,6 @@ { "connector_key": "imessage", - "version": "0.1.0", - "digest": "sha256:ace3798a8e222e73fa1555a39ba52b245e2566d31d84bb60fdec61c9ae1a9ca6", - "profile_sha256": "sha256:654eb8abb9625554c19edcf7e1ef6416fca556929456454ba29277751cde8efb" + "version": "0.1.25", + "digest": "sha256:920246f08102aea9c7958d6b3a0bf89eb07e3b776a921333f6d7ae93f99394b0", + "profile_sha256": "sha256:399958b7d89e1263b5f61e6b09de4c7f5d6a81c1084ff887e75a504278820ef4" } diff --git a/reference-implementation/test/collection-report-projection.test.ts b/reference-implementation/test/collection-report-projection.test.ts index 4cd803802..4e2ea219b 100644 --- a/reference-implementation/test/collection-report-projection.test.ts +++ b/reference-implementation/test/collection-report-projection.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; import test from "node:test"; import { @@ -295,6 +296,33 @@ test("a required stream reading unknown drags the connection coverage axis off c assert.notEqual(override, "partial"); }); +test("the pinned claude-code profile: a collector that never reports its required usage stream leaves coverage unknown", () => { + // The pinned profile requires `usage`. A connector emits a stream only when + // it is requested, so a collector that does not list `usage` among its + // default streams reports every other stream complete and still cannot reach + // `complete`. The collector's definition lists `usage`; this keeps the server + // side of that contract visible. + const profile = JSON.parse( + readFileSync(new URL("../server/local-collector-profiles/claude-code.json", import.meta.url), "utf8") + ) as { streams: ManifestStreamFixture[] }; + const requiredNames = profile.streams.filter((stream) => stream.required !== false).map((stream) => stream.name); + assert.ok(requiredNames.includes("usage")); + const reportFor = (reported: string[]) => + buildCollectionReport({ + attentionOpen: false, + collectionFacts: { + streams: reported.map((stream) => fact({ considered: 1, covered: 1, stream })), + }, + freshness: "fresh", + manifestStreams: profile.streams, + refresh: null, + }); + const withoutUsage = reportFor(requiredNames.filter((name) => name !== "usage")); + assert.equal(rollupCollectionReportCoverageOverride("complete", withoutUsage, profile.streams), "unknown"); + const withUsage = reportFor(requiredNames); + assert.notEqual(rollupCollectionReportCoverageOverride("complete", withUsage, profile.streams), "unknown"); +}); + test("unreliable projection withholds required complete coverage but preserves optional policy outcomes", () => { const entries = buildCollectionReport({ attentionOpen: false, diff --git a/reference-implementation/test/collector-profile-upgrade.test.ts b/reference-implementation/test/collector-profile-upgrade.test.ts new file mode 100644 index 000000000..b5b74a2eb --- /dev/null +++ b/reference-implementation/test/collector-profile-upgrade.test.ts @@ -0,0 +1,284 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Mixed versions of the claude-code Collection Profile. + * + * The collector pin and the profile this server registers move together + * (claude-code 0.3.0 to 0.3.4), but a device and a server do not upgrade at the + * same moment, and the server registers a pinned profile only when a device + * enrolls (`ensureReferenceConnectorCatalogEntry`). This test holds one enrolled + * device across both directions: + * + * - a new collector pushes `usage` to a server that still holds the previous + * profile: the push is refused at ingest, nothing is stored; + * - the server then registers the new profile: the same connection keeps its + * id, its checkpoint and its records, `usage` is accepted, the coverage proof + * stamped under the previous profile reads `generation_mismatch` (`unknown`), + * and the next run's coverage write brings the axis back. + * + * `fixtures/collector-profile-upgrade/claude-code-0.3.0.json` is the verbatim + * profile the previous pin shipped. Only the device-side process is simulated; + * enrollment, registration, ingest, state and the health projection are the + * production paths. + */ + +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { expectedLocalCoverageStoreDescriptors } from "@pdpp/polyfill-connectors/local-source-inventory"; +import { registerConnector } from "../server/auth.ts"; +import { COLLECTOR_PROTOCOL_VERSION } from "../server/collector-protocol.ts"; +import { reconcileDirtyConnectorSummaryEvidence } from "../server/connector-summary-read-model.ts"; +import { getDb } from "../server/db.ts"; +import { startServer } from "../server/index.ts"; +import { readLocalCollectorProfile } from "../server/local-collector-profiles.ts"; +import { putSyncState, readCommittedLocalCoverageDiagnostics } from "../server/records.ts"; +import { deriveLocalCoverageAxis, listConnectorSummaries } from "../server/ref-control.ts"; +import { closeTestServer } from "./helpers/close-test-server.ts"; +import { fetchJson, type TestServer } from "./helpers/legacy-upgrade.ts"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const CONNECTOR_ID = "claude-code"; +const PROTOCOL_HEADERS = { "X-PDPP-Collector-Protocol": COLLECTOR_PROTOCOL_VERSION }; + +function postJson(url: string, body: unknown, headers: Record = {}) { + return fetchJson(url, { + body: JSON.stringify(body), + headers: { Accept: "application/json", "Content-Type": "application/json", ...headers }, + method: "POST", + }); +} + +function canonicalValue(value: unknown): unknown { + if (value === null || typeof value !== "object") { + return value; + } + if (Array.isArray(value)) { + return value.map(canonicalValue); + } + const record = value as Record; + return Object.fromEntries( + Object.keys(record) + .sort() + .filter((key) => record[key] !== undefined) + .map((key) => [key, canonicalValue(record[key])]) + ); +} + +function bodyHash(records: unknown): string { + return createHash("sha256") + .update(JSON.stringify(canonicalValue(records))) + .digest("hex"); +} + +/** The manifest the server registers for a local-collector connector (see `readReferenceLocalConnectorCatalogManifest`). */ +function enrollmentManifest(profile: Record) { + return { ...profile, connector_id: CONNECTOR_ID, display_name: profile.display_name || CONNECTOR_ID }; +} + +function register(profile: Record) { + return registerConnector(enrollmentManifest(profile), { + backfillRetrievalIndexes: false, + replaceAcceptedDigest: true, + }); +} + +interface Device { + connector_instance_id: string; + device_id: string; + device_token: string; + source_instance_id: string; +} + +let batchSeq = 0; +function push(asUrl: string, device: Device, records: readonly Record[]) { + batchSeq += 1; + return postJson( + `${asUrl}/_ref/device-exporters/${encodeURIComponent(device.device_id)}/ingest-batches`, + { + batch_id: `profile-upgrade-${batchSeq}`, + batch_seq: batchSeq, + body_hash: bodyHash(records), + connector_id: CONNECTOR_ID, + device_id: device.device_id, + records, + source_instance_id: device.source_instance_id, + }, + { Authorization: `Bearer ${device.device_token}`, ...PROTOCOL_HEADERS } + ); +} + +function emitted(stream: string, key: string, data: Record) { + return { data, emitted_at: "2026-11-05T10:00:00.000Z", record_key: key, stream }; +} + +/** One `coverage_diagnostics` state write, as the connector commits it at the end of a successful run. */ +const COVERAGE_STORES = [ + { status: "collected", store: "projects", stream: "sessions" }, + { status: "collected", store: "derived_messages", stream: "messages" }, + { status: "collected", store: "derived_attachments", stream: "attachments" }, + { status: "collected", store: "derived_memory_notes", stream: "memory_notes" }, + { status: "inventory_only", store: "cache", stream: null }, + { status: "excluded", store: "auth", stream: null }, +]; + +function coverageRows() { + const expected = expectedLocalCoverageStoreDescriptors(CONNECTOR_ID); + assert.ok(expected, "the production claude-code connector declares a fixed inventory"); + const supplied = new Map(COVERAGE_STORES.map((row) => [row.store, row])); + return expected.map(({ store, stream }) => ({ + status: supplied.get(store)?.status ?? "inventory_only", + store, + stream, + })); +} + +async function commitCoverage(connectorInstanceId: string, rows: ReturnType) { + for (const row of rows) { + getDb() + .prepare( + `INSERT INTO records(connector_id, connector_instance_id, stream, record_key, record_json, emitted_at, version, deleted) + VALUES (?, ?, 'coverage_diagnostics', ?, ?, ?, 1, 0) + ON CONFLICT DO NOTHING` + ) + .run( + CONNECTOR_ID, + connectorInstanceId, + `coverage:${row.store}`, + JSON.stringify({ id: `coverage:${row.store}`, ...row }), + "2026-11-05T10:00:00.000Z" + ); + } + await putSyncState( + { connector_id: CONNECTOR_ID, connector_instance_id: connectorInstanceId }, + { coverage_diagnostics: { fetched_at: new Date(Date.now() - 30_000).toISOString(), stores: rows } } + ); +} + +/** The local-coverage axis the server derives from the committed proof (`deriveLocalCoverageAxis`). */ +async function localCoverage(connectorInstanceId: string) { + const proof = (await readCommittedLocalCoverageDiagnostics({ + connector_id: CONNECTOR_ID, + connector_instance_id: connectorInstanceId, + })) as Parameters[0]; + const { axis, reliable, unreliableReason } = deriveLocalCoverageAxis(proof); + return { axis, reliable, unreliableReason }; +} + +async function projectedConnectionIds(): Promise { + await reconcileDirtyConnectorSummaryEvidence(null); + return (await listConnectorSummaries()).map((summary) => summary.connector_instance_id); +} + +function stateRow(connectorInstanceId: string, stream: string) { + return getDb() + .prepare("SELECT state_json FROM connector_state WHERE connector_instance_id = ? AND stream = ?") + .get<{ state_json: string }>(connectorInstanceId, stream)?.state_json; +} + +function recordCount(connectorInstanceId: string, stream: string): number { + return ( + getDb() + .prepare("SELECT COUNT(*) AS n FROM records WHERE connector_instance_id = ? AND stream = ?") + .get<{ n: number }>(connectorInstanceId, stream)?.n ?? 0 + ); +} + +test("a device enrolled on the previous claude-code profile survives the server registering the new profile, in both push directions", async () => { + const previous = JSON.parse( + readFileSync(join(HERE, "fixtures", "collector-profile-upgrade", "claude-code-0.3.0.json"), "utf8") + ) as Record; + const current = readLocalCollectorProfile(CONNECTOR_ID); + assert.ok(current, "claude-code must have a pinned Collection Profile"); + const names = (profile: Record) => + (profile.streams as { name: string }[]).map((stream) => stream.name); + assert.equal(previous.version, "0.3.0"); + assert.equal(names(previous).includes("usage"), false); + assert.equal(names(current).includes("usage"), true); + + const server = (await startServer({ + asPort: 0, + dbPath: ":memory:", + quiet: true, + reconcilePolyfillManifests: false, + rsPort: 0, + })) as TestServer; + try { + const asUrl = `http://localhost:${server.asPort}`; + const code = await postJson(`${asUrl}/_ref/device-exporters/enrollment-codes`, { + connector_id: CONNECTOR_ID, + local_binding_name: "laptop", + }); + assert.equal(code.status, 201, JSON.stringify(code.body)); + const enrolled = await postJson( + `${asUrl}/_ref/device-exporters/enroll`, + { enrollment_code: code.body.enrollment_code }, + PROTOCOL_HEADERS + ); + assert.equal(enrolled.status, 201, JSON.stringify(enrolled.body)); + const device = enrolled.body as Device; + const connectionId = device.connector_instance_id; + + // The server a previous release left behind: it holds the previous profile. + await register(previous); + + const first = await push(asUrl, device, [ + emitted("sessions", "s1", { id: "s1" }), + emitted("messages", "m1", { id: "m1", session_id: "s1" }), + ]); + assert.equal(first.status, 201, JSON.stringify(first.body)); + assert.equal(first.body.connector_instance_id, connectionId); + const checkpoint = { cursor: "sessions-checkpoint-1" }; + await putSyncState({ connector_id: CONNECTOR_ID, connector_instance_id: connectionId }, { sessions: checkpoint }); + await commitCoverage(connectionId, coverageRows()); + assert.deepEqual(await localCoverage(connectionId), { axis: "complete", reliable: true, unreliableReason: undefined }); + assert.deepEqual(await projectedConnectionIds(), [connectionId]); + + // New collector, previous server: `usage` is not declared, so the push is refused and stores nothing. + const usage = emitted("usage", "usage", { + currency: "USD", + daily_activity: [], + daily_model_tokens: [], + hour_counts: {}, + id: "usage", + models: [], + source: "stats-cache", + total_cost_usd_cents: 0, + }); + const refused = await push(asUrl, device, [usage]); + assert.equal(refused.status, 400, JSON.stringify(refused.body)); + assert.equal(refused.body.error.message, "records[0].stream is not declared by the connector manifest"); + assert.equal(recordCount(connectionId, "usage"), 0); + + // The server registers the new profile. + await register(current); + + const accepted = await push(asUrl, device, [usage]); + assert.equal(accepted.status, 201, JSON.stringify(accepted.body)); + assert.equal(accepted.body.connector_instance_id, connectionId); + assert.equal(recordCount(connectionId, "usage"), 1); + // Identity, checkpoint and stored records are untouched by the profile change. + assert.equal(recordCount(connectionId, "sessions"), 1); + assert.equal(recordCount(connectionId, "messages"), 1); + assert.deepEqual(JSON.parse(stateRow(connectionId, "sessions") ?? "null"), checkpoint); + assert.deepEqual(await projectedConnectionIds(), [connectionId]); + // The coverage proof was stamped under the previous profile, so it is not + // trusted until the next run writes it again (`generation_mismatch`). + assert.deepEqual(await localCoverage(connectionId), { + axis: "unknown", + reliable: false, + unreliableReason: "generation_mismatch", + }); + + await commitCoverage(connectionId, coverageRows()); + assert.deepEqual(await localCoverage(connectionId), { axis: "complete", reliable: true, unreliableReason: undefined }); + assert.deepEqual(await projectedConnectionIds(), [connectionId]); + } finally { + await closeTestServer(server); + } +}); diff --git a/reference-implementation/test/fixtures/collector-profile-upgrade/claude-code-0.3.0.json b/reference-implementation/test/fixtures/collector-profile-upgrade/claude-code-0.3.0.json new file mode 100644 index 000000000..11b6cf048 --- /dev/null +++ b/reference-implementation/test/fixtures/collector-profile-upgrade/claude-code-0.3.0.json @@ -0,0 +1,880 @@ +{ + "protocol_version": "0.1.0", + "connector_id": "https://registry.pdpp.dev/connectors/claude-code", + "connector_key": "claude-code", + "manifest_uri": "https://registry.pdpp.dev/connectors/claude-code", + "version": "0.3.0", + "display_name": "Claude Code", + "brand": { + "icon": "icons/claude_code.svg", + "background_color": "#D97757" + }, + "runtime_requirements": { + "bindings": { + "filesystem": { + "required": true + } + }, + "environment_variables": [ + "CLAUDE_CODE_PROJECT_INCLUDE", + "CLAUDE_CODE_PROJECT_EXCLUDE" + ], + "local_paths": { + "home_env_override": "CLAUDE_CODE_HOME", + "home_default_relative_to_user_home": ".claude", + "paths": [ + { + "default_relative_to_home": "projects", + "env_override": "CLAUDE_CODE_PROJECTS_DIR", + "label": "projects directory", + "required_for_readiness": true + } + ] + } + }, + "capabilities": { + "human_interaction": [], + "proven": { + "local_collector": true + }, + "refresh_policy": { + "recommended_mode": "automatic", + "recommended_interval_seconds": 3600, + "minimum_interval_seconds": 300, + "maximum_staleness_seconds": 21600, + "interaction_posture": "none", + "rate_limit_sensitivity": "low", + "bot_detection_sensitivity": "low", + "background_safe": true, + "rationale": "Claude Code session logs live on local disk; re-reading them is cheap and has no platform cost." + }, + "public_listing": { + "tier": "supported" + } + }, + "streams": [ + { + "name": "sessions", + "required": true, + "description": "Claude Code sessions. One session = one CLI invocation in a specific project directory.", + "compaction_class": "recurring_snapshot", + "compaction_class_note": "aggregated per-session record; message_count/last_event_at widen on growth; never compactable, never re-alarms as growth accumulates.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Claude Code sessions", + "detail": "Session start time, duration, project directory, git branch, Claude Code version, total message count." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "project_path": { + "type": ["string", "null"] + }, + "cwd": { + "type": ["string", "null"] + }, + "git_branch": { + "type": ["string", "null"] + }, + "version": { + "type": ["string", "null"] + }, + "started_at": { + "type": ["string", "null"], + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "last_event_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "message_count": { + "type": ["integer", "null"] + }, + "user_type": { + "type": ["string", "null"] + }, + "entrypoint": { + "type": ["string", "null"] + } + }, + "required": ["id"] + }, + "primary_key": ["id"], + "cursor_field": "last_event_at", + "consent_time_field": "started_at", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "query": { + "search": { + "lexical_fields": [ + "project_path", + "cwd", + "git_branch", + "version", + "user_type", + "entrypoint" + ] + }, + "range_filters": { + "started_at": ["gte", "gt", "lte", "lt"], + "last_event_at": ["gte", "gt", "lte", "lt"], + "message_count": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["last_event_at", "started_at"] + } + }, + "coverage_strategy": "checkpoint_window", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "messages", + "required": true, + "description": "Messages within Claude Code sessions: user prompts and assistant responses. Includes both top-level session transcripts and nested subagent (sidechain) transcripts.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Claude Code messages", + "detail": "Role (user/assistant), text content, timestamp, parent message (for threading), session reference, subagent id when the message came from a sidechain run." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "session_id": { + "type": "string" + }, + "parent_uuid": { + "type": ["string", "null"] + }, + "role": { + "type": ["string", "null"], + "x_pdpp_role": "actor" + }, + "type": { + "type": ["string", "null"] + }, + "content": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "is_sidechain": { + "type": ["boolean", "null"] + }, + "user_type": { + "type": ["string", "null"] + }, + "agent_id": { + "type": ["string", "null"] + } + }, + "required": ["id", "session_id"] + }, + "primary_key": ["id"], + "cursor_field": "timestamp", + "consent_time_field": "timestamp", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "relationships": [ + { + "name": "session", + "stream": "sessions", + "foreign_key": "session_id", + "cardinality": "has_one" + } + ], + "query": { + "search": { + "lexical_fields": [ + "role", + "type", + "content", + "user_type", + "agent_id" + ], + "semantic_fields": ["content"] + }, + "range_filters": { + "timestamp": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["timestamp"] + } + }, + "coverage_strategy": "checkpoint_window", + "state_stream": "sessions", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "attachments", + "required": true, + "description": "Non-message events: hook outputs, tool uses, file snapshots, plus tool-results blob files written to disk.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Claude Code events", + "detail": "System events attached to sessions: tool calls, hook outputs, file-history snapshots, permission-mode changes, and persisted tool-result files." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "session_id": { + "type": "string" + }, + "parent_uuid": { + "type": ["string", "null"] + }, + "event_type": { + "type": ["string", "null"] + }, + "hook_name": { + "type": ["string", "null"] + }, + "tool_use_id": { + "type": ["string", "null"] + }, + "content_preview": { + "type": ["string", "null"] + }, + "content_bytes": { + "type": ["integer", "null"] + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time", + "x_pdpp_role": "event-time" + } + }, + "required": ["id", "session_id"] + }, + "primary_key": ["id"], + "cursor_field": "timestamp", + "consent_time_field": "timestamp", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "relationships": [ + { + "name": "session", + "stream": "sessions", + "foreign_key": "session_id", + "cardinality": "has_one" + } + ], + "query": { + "search": { + "lexical_fields": ["event_type", "hook_name", "content_preview"], + "semantic_fields": ["content_preview"] + }, + "range_filters": { + "timestamp": ["gte", "gt", "lte", "lt"], + "content_bytes": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["timestamp"] + } + }, + "coverage_strategy": "checkpoint_window", + "state_stream": "sessions", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "skills", + "required": true, + "description": "User-authored Claude Code skills installed under ~/.claude/skills//SKILL.md. Each record captures frontmatter metadata and the full body (capped at 20k chars).", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Claude Code skills", + "detail": "Skill name, description, full SKILL.md body, frontmatter, and on-disk path." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "description": { + "type": ["string", "null"], + "x_pdpp_role": "secondary" + }, + "source": { + "type": ["string", "null"] + }, + "path": { + "type": ["string", "null"] + }, + "content": { + "type": ["string", "null"], + "x_pdpp_role": "secondary" + }, + "frontmatter": { + "type": ["object", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"] + } + }, + "required": ["id"] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "search": { + "lexical_fields": ["name", "description", "source", "content"], + "semantic_fields": ["description", "content"] + }, + "range_filters": { + "mtime_epoch": ["gte", "gt", "lte", "lt"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "memory_notes", + "required": true, + "description": "Project-scoped Claude Code memory notes under ~/.claude/projects//memory/*.md. Each record captures frontmatter metadata and the note body (capped at 20k chars).", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Claude Code memory notes", + "detail": "Project path, note path, name, description, body, frontmatter, and on-disk path." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "project_path": { + "type": "string" + }, + "note_path": { + "type": "string" + }, + "name": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "description": { + "type": ["string", "null"], + "x_pdpp_role": "secondary" + }, + "path": { + "type": ["string", "null"] + }, + "content": { + "type": ["string", "null"] + }, + "frontmatter": { + "type": ["object", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"] + } + }, + "required": ["id", "project_path", "note_path"] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "search": { + "lexical_fields": [ + "project_path", + "note_path", + "name", + "description", + "content" + ], + "semantic_fields": ["description", "content"] + }, + "range_filters": { + "mtime_epoch": ["gte", "gt", "lte", "lt"] + } + }, + "coverage_strategy": "checkpoint_window", + "state_stream": "sessions", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "slash_commands", + "required": true, + "description": "User-authored Claude Code slash commands under ~/.claude/commands/*.md. Each record captures frontmatter, the command body (capped at 20k chars), and the file's absolute path.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your custom slash commands", + "detail": "Command name, description, full command body, frontmatter, and on-disk path." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "description": { + "type": ["string", "null"], + "x_pdpp_role": "secondary" + }, + "path": { + "type": ["string", "null"] + }, + "content": { + "type": ["string", "null"] + }, + "frontmatter": { + "type": ["object", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"] + } + }, + "required": ["id"] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "search": { + "lexical_fields": ["name", "description", "content"], + "semantic_fields": ["description", "content"] + }, + "range_filters": { + "mtime_epoch": ["gte", "gt", "lte", "lt"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "file_history", + "description": "Metadata-only inventory for Claude Code file-history stores. Does not emit file-history payload content.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "cache_inventory", + "description": "Metadata-only inventory for Claude Code cache stores; raw cache payloads are not collected.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "coverage_diagnostics", + "description": "Safe coverage status for known Claude Code local stores, including excluded auth-adjacent stores without payloads or raw absolute paths.", + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "stream": { + "type": ["string", "null"] + }, + "status": { + "type": "string" + }, + "reason": { + "type": "string", + "x_pdpp_role": "primary-title" + } + }, + "required": ["id", "store", "status", "reason"] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "backup_inventory", + "description": "Metadata-only inventory for Claude Code backup stores; raw backup payloads are not collected.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "config_inventory", + "description": "Metadata-only inventory for non-secret Claude Code configuration; raw config payloads are not collected.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + } + ], + "profiles": [ + { + "id": "full", + "label": "Your full Claude Code history", + "streams": [ + { + "name": "sessions" + }, + { + "name": "messages" + }, + { + "name": "attachments" + }, + { + "name": "skills" + }, + { + "name": "memory_notes" + }, + { + "name": "slash_commands" + } + ] + }, + { + "id": "messages_only", + "label": "Just prompts and responses", + "streams": [ + { + "name": "sessions" + }, + { + "name": "messages" + } + ] + }, + { + "id": "customizations", + "label": "Your skills, memory notes, and slash commands", + "streams": [ + { + "name": "skills" + }, + { + "name": "memory_notes" + }, + { + "name": "slash_commands" + } + ] + } + ], + "reason_display_messages": { + "claude_dir_not_found": "We couldn't find your Claude Code data folder", + "malformed_jsonl_line": "A line in this Claude Code session log contains invalid JSON and was skipped", + "non_object_jsonl_record": "A line in this Claude Code session log has an unexpected format and was skipped", + "scope_matched_no_sources": "None of your configured project folders matched any Claude Code projects", + "source_directory_read_error": "We couldn't read this Claude Code folder. It will be checked again on the next run", + "source_read_error": "This Claude Code session log could not be collected and will be retried on the next run", + "source_unreadable": "We couldn't read this folder, so we don't know what's in it", + "symlink_skipped": "A symbolic link was skipped without reading its target. The skipped path is listed in the source diagnostics", + "truncated_jsonl_tail": "The last line of this Claude Code session log is incomplete and will be checked again on the next run" + }, + "options_schema": { + "type": "object", + "description": "Owner tuning knobs for the Claude Code connector, resolved from the connector_instance_config_revisions spine. declared_option_kind is this connector's CLAIM only; the platform registry in packages/polyfill-connectors/src/connector-config-option-kind-registry.ts decides what actually gates self-activation, and an option the registry does not classify is enforced as collection_scope.", + "properties": { + "CLAUDE_CODE_PROJECT_INCLUDE": { + "type": "array", + "items": { "type": "string" }, + "description": "Substrings of project folder paths to include. A folder is collected if it contains any of these. Empty list means every project folder is eligible.", + "default": [], + "declared_option_kind": "collection_scope" + }, + "CLAUDE_CODE_PROJECT_EXCLUDE": { + "type": "array", + "items": { "type": "string" }, + "description": "Substrings of project folder paths to exclude. A folder is dropped if it contains any of these, even when the include list matched it.", + "default": [], + "declared_option_kind": "collection_scope" + } + } + } +} diff --git a/reference-implementation/test/static-secret-run-credentials.test.ts b/reference-implementation/test/static-secret-run-credentials.test.ts index c989a1c23..b4053dc77 100644 --- a/reference-implementation/test/static-secret-run-credentials.test.ts +++ b/reference-implementation/test/static-secret-run-credentials.test.ts @@ -6,6 +6,7 @@ import test from "node:test"; import { readPolyfillManifests } from "@pdpp/polyfill-connectors/manifests"; import { closeDb, getDb, initDb } from "../server/db.ts"; +import { classifyConnectorSetupModality, type ConnectorManifestLike } from "../server/connection-setup-plan.ts"; const MANIFESTS: Record = { chatgpt: { @@ -669,3 +670,55 @@ test("optional capture may omit credentials, while wrong-kind and invalid bundle (error) => error instanceof StaticSecretRunCredentialError && error.code === "recovered_secret_bundle_invalid", ); }); + +// chatgpt 0.1.0 declared a sealed email-and-password setup; chatgpt 0.2.5 +// declares none. A connection that sealed credentials under 0.1.0 keeps its +// stored credential after the upgrade, but no run reads or injects it. +const CHATGPT_0_1_0_MANIFEST = { + capabilities: { human_interaction: ["manual_action"] }, + runtime_requirements: { bindings: { browser: { required: true }, network: { required: true } } }, + setup: { + credential_capture: { + fields: [ + { autocomplete: "username", env: ["CHATGPT_USERNAME"], label: "ChatGPT email", name: "username", required: true, secret: true, type: "email" }, + { autocomplete: "current-password", env: ["CHATGPT_PASSWORD"], label: "ChatGPT password", name: "password", required: true, secret: true, type: "password" }, + ], + kind: "username_password", + label: "ChatGPT sign-in details", + }, + modality: "static_secret", + }, +}; +const { setup: _removedSetup, ...CHATGPT_0_2_5_MANIFEST } = CHATGPT_0_1_0_MANIFEST; + +test( + "chatgpt 0.2.5 stops injecting a credential sealed under 0.1.0, leaves it stored, and does not fail the run", + withStore(async (store) => { + seedConnectorInstance({ connectorId: "chatgpt-pdpp", connectorInstanceId: "cin_chat", ownerSubjectId: "owner_1" }); + await store.capture({ + connectorInstanceId: "cin_chat", + credentialKind: "username_password", + now: NOW, + ownerSubjectId: "owner_1", + secret: JSON.stringify({ password: "chat-pw", username: "owner@example.test" }), + }); + const args = { connectorId: "chatgpt-pdpp", connectorInstanceId: "cin_chat", ownerSubjectId: "owner_1" }; + + const before = await resolveStaticSecretRunEnv({ ...args, credentialStore: store, manifest: CHATGPT_0_1_0_MANIFEST }); + assert.deepEqual(before, { CHATGPT_PASSWORD: "chat-pw", CHATGPT_USERNAME: "owner@example.test" }); + assert.equal(classifyConnectorSetupModality("chatgpt", CHATGPT_0_1_0_MANIFEST as unknown as ConnectorManifestLike), "static_secret"); + + let recovered = 0; + const spy = { + recoverSecret: async (recoverArgs: { connectorInstanceId: string; ownerSubjectId: string }) => { + recovered += 1; + return store.recoverSecret(recoverArgs); + }, + }; + const after = await resolveStaticSecretRunEnv({ ...args, credentialStore: spy, manifest: CHATGPT_0_2_5_MANIFEST }); + assert.equal(after, null, "no credential env reaches the 0.2.5 run"); + assert.equal(recovered, 0, "the sealed credential is not read"); + assert.notEqual(classifyConnectorSetupModality("chatgpt", CHATGPT_0_2_5_MANIFEST as unknown as ConnectorManifestLike), "static_secret"); + assert.equal(await store.hasActiveCredential("cin_chat"), true, "the sealed credential stays stored"); + }) +); From fc9fc347449725c8e28221fe621502cc567b6d61 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Thu, 5 Nov 2026 15:58:22 -0600 Subject: [PATCH 3/8] build: keep Collection Profile pins in place unless the script is told to move them A bare run of pin-collection-profiles moved every pin to the catalog's latest release, which silently undid the holds that keep claude-code, codex, google-takeout and apple-photos below releases that drop the coverage_diagnostics stream this collector requests. `--version` ignored an unknown connector key, and `a=b=c` truncated to `a=b`. The usage example named a release (0.3.6) that is not a pin. - With no flag, each connector keeps its current pin; a connector with no pin yet takes the latest release. - `--latest` is explicit and still honors holds. - Holds live in packages/local-collector/collection-profile-holds.json with the reason, and the script reads them. A test fails when a hold names a version other than the one pinned. - Any other argument, an unknown connector key, a repeated key and a malformed `--version` are errors. A bare run against the catalog rewrites the same five pins and the reference server copies with no diff. Pass rule (from packages/local-collector): npx tsx --test test/pin-selection.test.ts test/collection-profiles.test.ts All 24 tests run; 23 pass and 1 is skipped (the live GHCR test). Assisted-by: AI Signed-off-by: Tim Nunamaker --- .../collection-profile-holds.json | 18 +++ .../scripts/pin-collection-profiles.ts | 58 +++++----- .../local-collector/scripts/pin-selection.ts | 105 ++++++++++++++++++ .../test/pin-selection.test.ts | 80 +++++++++++++ 4 files changed, 234 insertions(+), 27 deletions(-) create mode 100644 packages/local-collector/collection-profile-holds.json create mode 100644 packages/local-collector/scripts/pin-selection.ts create mode 100644 packages/local-collector/test/pin-selection.test.ts diff --git a/packages/local-collector/collection-profile-holds.json b/packages/local-collector/collection-profile-holds.json new file mode 100644 index 000000000..99c6a4fe2 --- /dev/null +++ b/packages/local-collector/collection-profile-holds.json @@ -0,0 +1,18 @@ +{ + "claude-code": { + "version": "0.3.4", + "reason": "The catalog's latest release (0.3.29 when checked) drops the coverage_diagnostics stream, which this collector requests. 0.3.4 is the oldest release with a signed source declaration." + }, + "codex": { + "version": "0.3.3", + "reason": "The catalog's latest release (0.3.15 when checked) drops the coverage_diagnostics stream, which this collector requests. 0.3.3 is the oldest release with a signed source declaration." + }, + "google-takeout": { + "version": "0.1.3", + "reason": "The catalog's latest release (0.1.27 when checked) drops the coverage_diagnostics stream, which this collector requests. 0.1.3 is the oldest release with a signed source declaration." + }, + "apple-photos": { + "version": "0.1.3", + "reason": "The catalog's latest release (0.1.26 when checked) drops the coverage_diagnostics stream, which this collector requests. 0.1.3 is the oldest release with a signed source declaration." + } +} diff --git a/packages/local-collector/scripts/pin-collection-profiles.ts b/packages/local-collector/scripts/pin-collection-profiles.ts index 5a70450e4..77accdf03 100644 --- a/packages/local-collector/scripts/pin-collection-profiles.ts +++ b/packages/local-collector/scripts/pin-collection-profiles.ts @@ -25,15 +25,23 @@ * Usage (network required): * * npm run pin:collection-profiles --workspace @pdpp/local-collector - * npm run pin:collection-profiles --workspace @pdpp/local-collector -- --version claude-code=0.3.6 + * npm run pin:collection-profiles --workspace @pdpp/local-collector -- --latest + * npm run pin:collection-profiles --workspace @pdpp/local-collector -- --version claude-code=0.3.4 * - * Re-running it moves every pin to the catalog's latest release. Review the - * diff as a connector upgrade. `--version =` (repeatable) - * pins that connector to a named catalog release instead of the latest, for a - * connector whose latest release drops a stream this collector still needs. + * With no flag the current pins stay where they are: the files are rewritten + * from the same releases, and a connector with no pin yet takes the catalog's + * latest. `--latest` moves every connector without a hold to the latest + * release; review the diff as a connector upgrade. `--version + * =` (repeatable) pins that connector to a named + * catalog release and beats a hold. Any other argument, an unknown connector + * key, or a malformed `--version` is an error. + * + * A hold keeps a pin below the latest release, with the reason, in + * `collection-profile-holds.json`. Holds apply under `--latest`; change a hold + * in that file in the same change that moves the pin. */ -import { mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -43,6 +51,7 @@ import { fetchResolvedArtifact, } from "@opendatalabs/data-connectors-tools/installer-core"; +import { COLLECTION_PROFILE_PINS } from "../src/generated/collection-profile-pins.generated.ts"; import { LOCAL_COLLECTOR_DEFINITIONS } from "../src/generated/collector-definitions.generated.ts"; import { COLLECTION_PROFILE_ENTRYPOINT, @@ -50,9 +59,11 @@ import { type CollectionProfilePin, collectionProfileRepository, } from "../src/managed/collection-profiles.ts"; +import { choosePin, parseHolds, parsePinArguments } from "./pin-selection.ts"; const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const pinsModule = path.join(packageRoot, "src", "generated", "collection-profile-pins.generated.ts"); +const holdsFile = path.join(packageRoot, "collection-profile-holds.json"); const REFERENCE_PROFILE_DIR = path.resolve( packageRoot, "..", @@ -84,24 +95,11 @@ function connectorKeyFor(connectorId: string): string { return connectorId.replaceAll("_", "-"); } -/** `--version =` arguments, as a key → version map. */ -function requestedVersions(args: readonly string[]): Map { - const requested = new Map(); - args.forEach((arg, index) => { - if (arg !== "--version") { - return; - } - const [key, version] = (args[index + 1] ?? "").split("="); - if (!key || !version) { - throw new Error("--version takes ="); - } - requested.set(key, version); - }); - return requested; -} - async function main(): Promise { - const versions = requestedVersions(process.argv.slice(2)); + const knownKeys = new Set(LOCAL_COLLECTOR_DEFINITIONS.map((definition) => connectorKeyFor(definition.connector_id))); + const args = parsePinArguments(process.argv.slice(2), knownKeys); + const holds = parseHolds(JSON.parse(readFileSync(holdsFile, "utf8"))); + const currentVersions = new Map(COLLECTION_PROFILE_PINS.map((pin) => [pin.connectorKey, pin.version])); const catalogResult = (await fetchCatalog({})) as { catalog: { connectors: readonly CatalogConnector[] } }; const catalog = new Map(catalogResult.catalog.connectors.map((entry) => [entry.connector_key, entry])); @@ -110,11 +108,17 @@ async function main(): Promise { for (const definition of LOCAL_COLLECTOR_DEFINITIONS) { const connectorKey = connectorKeyFor(definition.connector_id); const listed = catalog.get(connectorKey); - const wanted = versions.get(connectorKey); - const latest = wanted ? listed?.versions?.find((release) => release.version === wanted) : listed?.latest; + const choice = choosePin({ + connectorKey, + args, + hold: holds.get(connectorKey), + currentVersion: currentVersions.get(connectorKey), + }); + const latest = + choice.kind === "latest" ? listed?.latest : listed?.versions?.find((release) => release.version === choice.version); if (!latest) { process.stdout.write( - `unpinned ${definition.connector_id}: the signed catalog lists no ${connectorKey}${wanted ? ` ${wanted}` : ""}\n` + `unpinned ${definition.connector_id}: the signed catalog lists no ${connectorKey}${choice.kind === "version" ? ` ${choice.version}` : ""}\n` ); continue; } @@ -145,7 +149,7 @@ async function main(): Promise { entrypointSha256: resolved.checksums.entrypoint, }); profiles.set(connectorKey, resolved.manifestBuffer); - process.stdout.write(`pinned ${definition.connector_id} ${latest.version} ${resolved.oci.digest}\n`); + process.stdout.write(`pinned ${definition.connector_id} ${latest.version} ${resolved.oci.digest} (${choice.why})\n`); } writeFileSync(pinsModule, renderPinsModule(pins)); diff --git a/packages/local-collector/scripts/pin-selection.ts b/packages/local-collector/scripts/pin-selection.ts new file mode 100644 index 000000000..c8f4795d6 --- /dev/null +++ b/packages/local-collector/scripts/pin-selection.ts @@ -0,0 +1,105 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Which catalog release `pin-collection-profiles.ts` pins for each connector. + * + * Kept apart from the script so the rules run without the network: a re-run + * must never move a pin the owner did not ask it to move. + */ + +/** A pin that stays where it is, with the reason, read from `collection-profile-holds.json`. */ +export interface CollectionProfileHold { + readonly version: string; + readonly reason: string; +} + +export interface PinArguments { + /** `--latest`: move connectors that have no hold to the catalog's latest release. */ + readonly latest: boolean; + /** `--version =`: a named release, which beats a hold. */ + readonly versions: ReadonlyMap; +} + +export type PinChoice = + | { readonly kind: "version"; readonly version: string; readonly why: string } + | { readonly kind: "latest"; readonly why: string }; + +/** + * Parse `--latest` and `--version =` (repeatable). + * Anything else is an error, so a mistyped flag cannot pass for a no-op. + */ +export function parsePinArguments(args: readonly string[], knownKeys: ReadonlySet): PinArguments { + let latest = false; + const versions = new Map(); + for (let index = 0; index < args.length; index += 1) { + const arg = args[index] ?? ""; + if (arg === "--latest") { + latest = true; + } else if (arg === "--version") { + const spec = args[index + 1] ?? ""; + index += 1; + const parts = spec.split("="); + const [key, version] = parts; + if (parts.length !== 2 || !key || !version) { + throw new Error(`--version takes exactly =, got "${spec}"`); + } + if (!knownKeys.has(key)) { + throw new Error(`--version names unknown connector "${key}"; known: ${[...knownKeys].sort().join(", ")}`); + } + if (versions.has(key)) { + throw new Error(`--version names ${key} twice`); + } + versions.set(key, version); + } else { + throw new Error(`unknown argument "${arg}"; accepted: --latest, --version =`); + } + } + return { latest, versions }; +} + +/** + * Choose a release for one connector, in this order: + * 1. `--version` names it; + * 2. a hold keeps it, even under `--latest`; + * 3. `--latest` takes the catalog's latest; + * 4. the current pin stays; + * 5. a connector with no pin yet takes the latest. + */ +export function choosePin(input: { + readonly connectorKey: string; + readonly args: PinArguments; + readonly hold: CollectionProfileHold | undefined; + readonly currentVersion: string | undefined; +}): PinChoice { + const requested = input.args.versions.get(input.connectorKey); + if (requested) { + return { kind: "version", version: requested, why: "named by --version" }; + } + if (input.hold) { + return { kind: "version", version: input.hold.version, why: `held: ${input.hold.reason}` }; + } + if (input.args.latest) { + return { kind: "latest", why: "--latest" }; + } + if (input.currentVersion) { + return { kind: "version", version: input.currentVersion, why: "current pin kept" }; + } + return { kind: "latest", why: "no pin yet" }; +} + +/** Validate the holds file's shape; a malformed hold must not be read as no hold. */ +export function parseHolds(raw: unknown): ReadonlyMap { + if (typeof raw !== "object" || raw === null || Array.isArray(raw)) { + throw new Error("collection-profile-holds.json must be an object keyed by connector key"); + } + const holds = new Map(); + for (const [key, value] of Object.entries(raw)) { + const hold = value as Partial | null; + if (!hold || typeof hold.version !== "string" || !hold.version || typeof hold.reason !== "string" || !hold.reason) { + throw new Error(`hold for "${key}" needs a non-empty version and reason`); + } + holds.set(key, { version: hold.version, reason: hold.reason }); + } + return holds; +} diff --git a/packages/local-collector/test/pin-selection.test.ts b/packages/local-collector/test/pin-selection.test.ts new file mode 100644 index 000000000..bfd6d83b1 --- /dev/null +++ b/packages/local-collector/test/pin-selection.test.ts @@ -0,0 +1,80 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * `pin-collection-profiles.ts` decides which release each connector pins. A + * re-run must not move a pin nobody asked it to move, and a mistyped argument + * must fail instead of passing for a no-op. + */ + +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { test } from "node:test"; + +import { COLLECTION_PROFILE_PINS } from "../src/generated/collection-profile-pins.generated.ts"; +import { choosePin, parseHolds, parsePinArguments } from "../scripts/pin-selection.ts"; + +const KEYS = new Set(["claude-code", "codex", "imessage"]); +const hold = { version: "0.3.4", reason: "latest drops a stream" }; + +test("with no flag a connector keeps its current pin, even one below the latest release", () => { + const args = parsePinArguments([], KEYS); + assert.deepEqual(choosePin({ connectorKey: "codex", args, hold: undefined, currentVersion: "0.3.3" }), { + kind: "version", + version: "0.3.3", + why: "current pin kept", + }); +}); + +test("with no flag a connector that has no pin yet takes the latest release", () => { + const args = parsePinArguments([], KEYS); + assert.equal(choosePin({ connectorKey: "codex", args, hold: undefined, currentVersion: undefined }).kind, "latest"); +}); + +test("--latest moves an unheld connector and leaves a held one where its hold puts it", () => { + const args = parsePinArguments(["--latest"], KEYS); + assert.equal(choosePin({ connectorKey: "imessage", args, hold: undefined, currentVersion: "0.1.3" }).kind, "latest"); + const held = choosePin({ connectorKey: "claude-code", args, hold, currentVersion: "0.3.4" }); + assert.deepEqual(held, { kind: "version", version: "0.3.4", why: "held: latest drops a stream" }); +}); + +test("--version names a release and beats a hold", () => { + const args = parsePinArguments(["--version", "claude-code=0.3.9"], KEYS); + const choice = choosePin({ connectorKey: "claude-code", args, hold, currentVersion: "0.3.4" }); + assert.equal(choice.kind === "version" && choice.version, "0.3.9"); +}); + +test("an unknown connector key is an error", () => { + assert.throws(() => parsePinArguments(["--version", "claude_code=0.3.4"], KEYS), /unknown connector "claude_code"/); +}); + +test("a malformed --version is an error, including a spec with more than one =", () => { + for (const spec of ["claude-code", "claude-code=", "=0.3.4", "claude-code=0.3.4=0.3.5", ""]) { + assert.throws(() => parsePinArguments(["--version", spec], KEYS), /exactly =/, spec); + } + assert.throws(() => parsePinArguments(["--version"], KEYS), /exactly/); +}); + +test("an unknown argument and a repeated key are errors", () => { + assert.throws(() => parsePinArguments(["--lastest"], KEYS), /unknown argument "--lastest"/); + assert.throws( + () => parsePinArguments(["--version", "codex=0.3.3", "--version", "codex=0.3.5"], KEYS), + /names codex twice/ + ); +}); + +test("a hold needs a version and a reason", () => { + assert.throws(() => parseHolds({ codex: { version: "0.3.3" } }), /needs a non-empty version and reason/); + assert.throws(() => parseHolds([]), /must be an object/); +}); + +test("every hold in collection-profile-holds.json names the version this collector pins", () => { + const holds = parseHolds( + JSON.parse(readFileSync(new URL("../collection-profile-holds.json", import.meta.url), "utf8")) + ); + assert.ok(holds.size > 0); + const pinned = new Map(COLLECTION_PROFILE_PINS.map((pin) => [pin.connectorKey, pin.version])); + for (const [key, entry] of holds) { + assert.equal(pinned.get(key), entry.version, `${key} is held at ${entry.version} but pinned at ${pinned.get(key)}`); + } +}); From 61086059324feb886b730b5c96edac679898f056 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Thu, 5 Nov 2026 15:59:32 -0600 Subject: [PATCH 4/8] test: pin the connector installer's routing between its two cores scripts/resolve-connectors.js sends each lock entry to the core that accepts it: the 12 tarball entries and github-pdpp (which has no signed source declaration) to the earlier core, every other OCI entry to the current one. Nothing tested that split, and the test file imported installLockedConnectors without using it. B-T9 pushes one mixed lock through installConnectorsAtomically with its default install and checks the installed tree, the summed counts (3 connectors, 11 files) and which signature each entry was verified against: - a tarball entry; - ynab, a declaration-bearing OCI entry with no generated local-collector pin, so it arrives through the lock; - github-pdpp, a declaration-less OCI entry for a listed id. The same declaration-less release under an unlisted id is refused with "no signed source declaration layer or referrer", and the install root is unchanged. The two STOPGAP comments now name the issues that remove them (#633 for the 12 tarball entries, #634 for github-pdpp). Pass rule (from the repository root; these are exact counts at this commit): npx vitest run scripts/resolve-connectors.test.mjs 26 passed (cd packages/local-collector && npx tsx scripts/run-tests.ts) 301 tests, 296 pass, 5 skipped (cd reference-implementation && node --import tsx --test \ test/static-secret-run-credentials.test.ts \ test/collection-report-projection.test.ts \ test/collector-profile-upgrade.test.ts \ test/local-collector-profiles.test.ts) 134 pass (cd packages/local-collector && npx tsc --noEmit -p tsconfig.json) (cd reference-implementation && npx tsc --noEmit) both exit 0 Assisted-by: AI Signed-off-by: Tim Nunamaker --- scripts/resolve-connectors.js | 7 +- scripts/resolve-connectors.test.mjs | 185 ++++++++++++++++++++++++++-- 2 files changed, 176 insertions(+), 16 deletions(-) diff --git a/scripts/resolve-connectors.js b/scripts/resolve-connectors.js index 375c6af72..c6a074006 100644 --- a/scripts/resolve-connectors.js +++ b/scripts/resolve-connectors.js @@ -25,8 +25,8 @@ import { verifyInstalled, } from "@opendatalabs/data-connectors-tools/installer-core" // STOPGAP, removed when the legacy tarball connectors move to Collection -// Profiles. The current installer core installs only signed OCI Collection -// Profiles that carry a signed source declaration. The bundled tarball +// Profiles (tracked in #633). The current installer core installs only signed +// OCI Collection Profiles that carry a signed source declaration. The bundled tarball // connectors, and the one OCI release published before source declarations // existed (see PRE_DECLARATION_OCI_CONNECTORS), still need the earlier core, // which is pinned under its own name so it cannot be mistaken for the current @@ -76,7 +76,8 @@ export const LOCKED_ARTIFACT_SOURCE = Object.freeze({ // STOPGAP. OCI connectors whose pinned release has no signed source // declaration, so the current installer core refuses it. They install and // author through the earlier core, like the tarball connectors. Delete an id -// here once its connector is re-pinned to a release that carries a declaration. +// here once its connector is re-pinned to a release that carries a declaration +// (tracked in #634). export const PRE_DECLARATION_OCI_CONNECTORS = Object.freeze( new Set(["github-pdpp"]) ) diff --git a/scripts/resolve-connectors.test.mjs b/scripts/resolve-connectors.test.mjs index fb87fe917..704b255ca 100644 --- a/scripts/resolve-connectors.test.mjs +++ b/scripts/resolve-connectors.test.mjs @@ -24,9 +24,9 @@ import { authorOciProfile, checkInstalledLock, installConnectorsAtomically, - installLockedConnectors, ociCertificateIdentityResolver, LOCKED_ARTIFACT_SOURCE, + PRE_DECLARATION_OCI_CONNECTORS, resolveIndexUrl, recoverInterruptedInstall, } from "./resolve-connectors.js" @@ -328,19 +328,18 @@ describe("resolved connector-installer archive acceptance", () => { // Reproduce PR A's OCI wire fixture. The core still parses manifests, checks // digests, extracts archives and assembles the cosign bundle. Only registry I/O // and Fulcio/Rekor are replaced; the test verifier checks a real EC signature. -function fixture(root) { - const key = "ynab", - connectorId = `${key}-pdpp`, +function fixture(root, { key = "ynab", withDeclaration = true } = {}) { + const connectorId = `${key}-pdpp`, version = "0.3.0" const profile = json({ connector_key: key, connector_id: `https://github.com/PDP-Connect/data-connectors/connector/${key}`, version, protocol_version: "1.0", - display_name: "YNAB", + display_name: key, source: { - id: "https://registry.pdpp.dev/sources/ynab", - display: { name: "YNAB" }, + id: `https://registry.pdpp.dev/sources/${key}`, + display: { name: key }, }, streams: [ { @@ -370,7 +369,7 @@ function fixture(root) { const entry = { connectorId, connectorKey: key, - company: "YNAB", + company: key, version, resolvedFrom: version, artifactKind: "pdpp-collection-profile", @@ -391,7 +390,9 @@ function fixture(root) { ...JSON.parse(profile), config_version: "1.0", profile_digest: digest(profile), - source_declaration_digest: digest(declaration), + ...(withDeclaration + ? { source_declaration_digest: digest(declaration) } + : {}), entrypoint: "code/collection-profile.mjs", }) const manifest = json({ @@ -409,10 +410,14 @@ function fixture(root) { "application/vnd.pdpp.connector.licenses.v1.tar+gzip" ), layer(provenance, "application/vnd.pdpp.connector.provenance.v1+json"), - layer( - declaration, - "application/vnd.pdpp.connector.source-declaration.v1+json" - ), + ...(withDeclaration + ? [ + layer( + declaration, + "application/vnd.pdpp.connector.source-declaration.v1+json" + ), + ] + : []), ], }) const manifestDigest = digest(manifest) @@ -849,6 +854,160 @@ describe("OCI consumer acceptance", () => { } })) + it("B-T9 a mixed lock installs through the default install: tarball by the earlier core, declaration-bearing OCI by the current core, declaration-less listed OCI by the earlier core; an unlisted declaration-less OCI is refused", async () => + temporary(async root => { + // `ynab` stands for a lock-delivered connector that has no generated + // local-collector pin: it must install and resolve the same way. + const pins = readFileSync( + "packages/local-collector/src/generated/collection-profile-pins.generated.ts", + "utf8" + ) + expect(pins).not.toContain('"connectorKey": "ynab"') + const signed = fixture(join(root, "signed"), { key: "ynab" }) + const listed = fixture(join(root, "listed"), { + key: "github", + withDeclaration: false, + }) + expect(PRE_DECLARATION_OCI_CONNECTORS.has(listed.oci.connectorId)).toBe( + true + ) + + const tarballUrl = legacyArtifacts[0].artifactUrl + const bundleUrl = `${tarballUrl}.sigstore.json` + const manifest = json({ + connector_id: "acme-tarball", + version: "1.0.0", + name: "Acme", + }) + const script = Buffer.from("export const run = () => {};\n") + const tar = archive(join(root, "tarball"), "legacy", { + "manifest.json": manifest, + "script.js": script, + }) + const tarballEntry = { + connectorId: "acme-tarball", + company: "acme", + version: "1.0.0", + resolvedFrom: "1.0.0", + sourceFiles: { + metadata: "acme/acme-tarball.json", + script: "acme/acme-tarball.js", + }, + artifactUrl: tarballUrl, + artifactSha256: digest(tar), + artifactSignature: { + type: "sigstoreBundle", + bundlePath: "acme-tarball.tgz.sigstore.json", + bundleUrl, + }, + manifestSha256: digest(manifest), + scriptSha256: digest(script), + } + + const verified = [] + const options = { + fetchImpl: async url => { + if (url === tarballUrl) + return new Response(tar, { status: 200 }) + if (url === bundleUrl) + return new Response(JSON.stringify({ bundle: "legacy" }), { + status: 200, + }) + const repository = new URL(url).pathname + const owner = [signed, listed].find(f => + repository.startsWith(`/v2/${f.oci.oci.repository}/`) + ) + if (!owner) throw new Error(`Unexpected request: ${url}`) + return owner.options.fetchImpl(url) + }, + sigstoreVerifier: async (bundle, bytes, policy) => { + if (bundle.bundle === "legacy") { + expect(bytes).toEqual(tar) + verified.push("tarball") + return + } + let failure + for (const f of [signed, listed]) { + try { + await f.options.sigstoreVerifier(bundle, bytes, policy) + verified.push(f.oci.connectorId) + return + } catch (error) { + failure = error + } + } + throw failure + }, + } + const lock = { + lockVersion: "2.0", + dependencies: { + "acme-tarball": "1.0.0", + [signed.oci.connectorId]: "0.3.0", + [listed.oci.connectorId]: "0.3.0", + }, + connectors: [tarballEntry, signed.oci, listed.oci], + } + const installRoot = join(root, "installed") + + const result = await installConnectorsAtomically({ + lock, + installRoot, + ...options, + }) + + // Summed across both cores: 3 connectors, 2 tarball files + 4 files for + // the declaration-less release + 5 for the signed one. + expect(result.connectorCount).toBe(3) + const paths = tree(installRoot).map(([path]) => path) + expect(paths).toEqual([ + "acme/acme-tarball.js", + "acme/acme-tarball.json", + "collection-profiles/github-pdpp/dist/collection-profile.mjs", + "collection-profiles/github-pdpp/licenses/LICENSE", + "collection-profiles/github-pdpp/profile/collection-profile.json", + "collection-profiles/github-pdpp/provenance.json", + "collection-profiles/ynab-pdpp/dist/collection-profile.mjs", + "collection-profiles/ynab-pdpp/licenses/LICENSE", + "collection-profiles/ynab-pdpp/profile/collection-profile.json", + "collection-profiles/ynab-pdpp/provenance.json", + // Only the signed release carries a source declaration. + "collection-profiles/ynab-pdpp/source-declaration.json", + "lock.json", + ]) + expect(result.filesWritten).toBe(11) + expect(verified.sort()).toEqual( + ["github-pdpp", "tarball", "ynab-pdpp"].sort() + ) + expect( + checkInstalledLock({ + lock, + dependencies: { connectors: lock.dependencies }, + installRoot, + }).ok + ).toBe(true) + + // The same declaration-less release under an id the stopgap does not + // list is refused by the current core, and nothing is published. + const unlisted = fixture(join(root, "unlisted"), { + key: "gitlab", + withDeclaration: false, + }) + const before = tree(installRoot) + await expect( + installConnectorsAtomically({ + lock: { + lockVersion: "2.0", + dependencies: { [unlisted.oci.connectorId]: "0.3.0" }, + connectors: [unlisted.oci], + }, + installRoot, + ...unlisted.options, + }) + ).rejects.toThrow(/no signed source declaration layer or referrer/) + expect(tree(installRoot)).toEqual(before) + })) + it("B-T8 generate-platform-registry output is unchanged by a mixed v2 lock", async () => temporary(async root => { const f = fixture(root), From 9f9d95364e420a498c03925fc70809636f9b84cc Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Thu, 5 Nov 2026 23:14:24 -0600 Subject: [PATCH 5/8] fix: stop the Collection Profile pin script from dropping a connector whose release it cannot find When the signed catalog did not list a release the script was asked to keep or move to, it printed "unpinned" and went on. It then rewrote the pins module without that connector and deleted its profile and pin record from the reference server's profile directory, and exited 0. `--version codex=99.99.99`, a current pin whose release left the catalog, and `--latest` on a pinned connector the catalog dropped all did this. Only a connector with no pin and no request may now be left unpinned. A named, held or already pinned connector with no matching catalog release fails the run before anything is written, so every existing output stays as it was. The script's work moves into `runPinScript(deps)`, with the catalog fetch, the artifact fetch, the paths and the log injected, and the command-line entry builds the real ones. pin-collection-profiles.test.ts runs it offline against a temp tree: the three refusals (each asserts the pins module and profile directory are byte-for-byte unchanged), the unpinned never-pinned connector, and a bare run. With the fix removed, the three refusal tests fail; with it, all five pass. Pass rule (from packages/local-collector; exact counts at this commit): npx tsx --test test/pin-collection-profiles.test.ts 5 passed npx tsx scripts/run-tests.ts 306 tests, 301 pass, 0 fail, 5 skipped npx tsc --noEmit -p tsconfig.json exit 0 Assisted-by: AI Signed-off-by: Tim Nunamaker --- .../scripts/pin-collection-profiles.ts | 68 +++++-- .../test/pin-collection-profiles.test.ts | 168 ++++++++++++++++++ 2 files changed, 221 insertions(+), 15 deletions(-) create mode 100644 packages/local-collector/test/pin-collection-profiles.test.ts diff --git a/packages/local-collector/scripts/pin-collection-profiles.ts b/packages/local-collector/scripts/pin-collection-profiles.ts index 77accdf03..c001f09f2 100644 --- a/packages/local-collector/scripts/pin-collection-profiles.ts +++ b/packages/local-collector/scripts/pin-collection-profiles.ts @@ -34,7 +34,10 @@ * release; review the diff as a connector upgrade. `--version * =` (repeatable) pins that connector to a named * catalog release and beats a hold. Any other argument, an unknown connector - * key, or a malformed `--version` is an error. + * key, or a malformed `--version` is an error. A release the run keeps or + * moves to (named, held, or already pinned) that the catalog does not list + * fails the run before anything is written; only a connector with no pin and + * no request may be left unpinned. * * A hold keeps a pin below the latest release, with the reason, in * `collection-profile-holds.json`. Holds apply under `--latest`; change a hold @@ -95,17 +98,35 @@ function connectorKeyFor(connectorId: string): string { return connectorId.replaceAll("_", "-"); } -async function main(): Promise { - const knownKeys = new Set(LOCAL_COLLECTOR_DEFINITIONS.map((definition) => connectorKeyFor(definition.connector_id))); - const args = parsePinArguments(process.argv.slice(2), knownKeys); - const holds = parseHolds(JSON.parse(readFileSync(holdsFile, "utf8"))); - const currentVersions = new Map(COLLECTION_PROFILE_PINS.map((pin) => [pin.connectorKey, pin.version])); - const catalogResult = (await fetchCatalog({})) as { catalog: { connectors: readonly CatalogConnector[] } }; +/** Everything the script touches outside its own logic, so a test can run it offline against a temp tree. */ +export interface PinScriptDeps { + readonly argv: readonly string[]; + readonly definitions: readonly { readonly connector_id: string }[]; + readonly currentPins: readonly { readonly connectorKey: string; readonly version: string }[]; + readonly holdsFile: string; + readonly pinsModule: string; + readonly referenceProfileDir: string; + readonly fetchCatalog: () => Promise<{ catalog: { connectors: readonly CatalogConnector[] } }>; + readonly fetchResolvedArtifact: ( + context: null, + artifact: Record, + options: Record + ) => Promise; + readonly log: (line: string) => void; +} + +export async function runPinScript(deps: PinScriptDeps): Promise { + const { pinsModule, referenceProfileDir: REFERENCE_PROFILE_DIR } = deps; + const knownKeys = new Set(deps.definitions.map((definition) => connectorKeyFor(definition.connector_id))); + const args = parsePinArguments(deps.argv, knownKeys); + const holds = parseHolds(JSON.parse(readFileSync(deps.holdsFile, "utf8"))); + const currentVersions = new Map(deps.currentPins.map((pin) => [pin.connectorKey, pin.version])); + const catalogResult = await deps.fetchCatalog(); const catalog = new Map(catalogResult.catalog.connectors.map((entry) => [entry.connector_key, entry])); const pins: CollectionProfilePin[] = []; const profiles = new Map(); - for (const definition of LOCAL_COLLECTOR_DEFINITIONS) { + for (const definition of deps.definitions) { const connectorKey = connectorKeyFor(definition.connector_id); const listed = catalog.get(connectorKey); const choice = choosePin({ @@ -117,13 +138,19 @@ async function main(): Promise { const latest = choice.kind === "latest" ? listed?.latest : listed?.versions?.find((release) => release.version === choice.version); if (!latest) { - process.stdout.write( - `unpinned ${definition.connector_id}: the signed catalog lists no ${connectorKey}${choice.kind === "version" ? ` ${choice.version}` : ""}\n` - ); + // A release this run was asked to keep or move to must exist. Only a + // connector with no pin and no request may be left out, and then the + // collector refuses it by name. + if (choice.kind === "version" || currentVersions.has(connectorKey)) { + throw new Error( + `cannot pin ${connectorKey}: the signed catalog lists no ${choice.kind === "version" ? choice.version : "latest release"} (${choice.why}). Nothing was written.` + ); + } + deps.log(`unpinned ${definition.connector_id}: the signed catalog lists no ${connectorKey}`); continue; } const repository = collectionProfileRepository({ connectorKey }); - const resolved = (await fetchResolvedArtifact( + const resolved = await deps.fetchResolvedArtifact( null, { artifactKind: "pdpp-collection-profile", @@ -139,7 +166,7 @@ async function main(): Promise { ociCertificateIdentityResolver: ({ registry, repository: requested }: { registry: string; repository: string }) => registry === "ghcr.io" && requested === repository ? DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY : null, } - )) as ResolvedArtifact; + ); pins.push({ connectorId: definition.connector_id, connectorKey, @@ -149,7 +176,7 @@ async function main(): Promise { entrypointSha256: resolved.checksums.entrypoint, }); profiles.set(connectorKey, resolved.manifestBuffer); - process.stdout.write(`pinned ${definition.connector_id} ${latest.version} ${resolved.oci.digest} (${choice.why})\n`); + deps.log(`pinned ${definition.connector_id} ${latest.version} ${resolved.oci.digest} (${choice.why})`); } writeFileSync(pinsModule, renderPinsModule(pins)); @@ -192,5 +219,16 @@ ${body} } if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { - await main(); + await runPinScript({ + argv: process.argv.slice(2), + currentPins: COLLECTION_PROFILE_PINS, + definitions: LOCAL_COLLECTOR_DEFINITIONS, + fetchCatalog: async () => (await fetchCatalog({})) as { catalog: { connectors: readonly CatalogConnector[] } }, + fetchResolvedArtifact: async (context, artifact, options) => + (await fetchResolvedArtifact(context, artifact, options)) as ResolvedArtifact, + holdsFile, + log: (line) => process.stdout.write(`${line}\n`), + pinsModule, + referenceProfileDir: REFERENCE_PROFILE_DIR, + }); } diff --git a/packages/local-collector/test/pin-collection-profiles.test.ts b/packages/local-collector/test/pin-collection-profiles.test.ts new file mode 100644 index 000000000..8e6727131 --- /dev/null +++ b/packages/local-collector/test/pin-collection-profiles.test.ts @@ -0,0 +1,168 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Runs `pin-collection-profiles.ts` end to end against a temp tree and a fake + * catalog. A release the run was asked to keep or move to, and the catalog no + * longer lists, must fail the run with every existing output untouched; it + * must not drop the connector from the pins. + */ + +import assert from "node:assert/strict"; +import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; + +import { type PinScriptDeps, runPinScript } from "../scripts/pin-collection-profiles.ts"; + +const DEFINITIONS = [{ connector_id: "codex" }, { connector_id: "imessage" }]; + +interface Tree { + readonly deps: PinScriptDeps; + readonly logs: string[]; + readonly snapshot: () => Record; + readonly cleanup: () => void; +} + +function release(version: string) { + return { digest: `sha256:${version}`, version }; +} + +function tree(options: { + readonly argv: readonly string[]; + readonly catalog: Record; + readonly currentPins: readonly { connectorKey: string; version: string }[]; +}): Tree { + const root = mkdtempSync(path.join(tmpdir(), "pin-script-")); + const pinsModule = path.join(root, "pins.generated.ts"); + const referenceProfileDir = path.join(root, "profiles"); + const holdsFile = path.join(root, "holds.json"); + mkdirSync(referenceProfileDir); + writeFileSync(holdsFile, "{}\n"); + writeFileSync(pinsModule, "OLD PINS MODULE\n"); + for (const { connectorKey, version } of options.currentPins) { + writeFileSync(path.join(referenceProfileDir, `${connectorKey}.json`), `{"old":"${version}"}\n`); + writeFileSync(path.join(referenceProfileDir, `${connectorKey}.pin.json`), `{"old_pin":"${version}"}\n`); + } + const logs: string[] = []; + const deps: PinScriptDeps = { + argv: options.argv, + currentPins: options.currentPins, + definitions: DEFINITIONS, + fetchCatalog: async () => ({ + catalog: { + connectors: Object.entries(options.catalog).map(([connector_key, versions]) => ({ + connector_key, + latest: release(versions.at(-1) ?? ""), + versions: versions.map(release), + })), + }, + }), + fetchResolvedArtifact: async (_context, artifact) => ({ + checksums: { entrypoint: "entry", manifest: "manifest" }, + manifestBuffer: Buffer.from(`{"profile":"${String(artifact.connectorKey)} ${String(artifact.version)}"}\n`), + oci: { digest: `sha256:${String(artifact.version)}` }, + }), + holdsFile, + log: (line) => logs.push(line), + pinsModule, + referenceProfileDir, + }; + const snapshot = () => { + const files: Record = { "pins.generated.ts": readFileSync(pinsModule, "utf8") }; + for (const name of readdirSync(referenceProfileDir).sort()) { + files[`profiles/${name}`] = readFileSync(path.join(referenceProfileDir, name), "utf8"); + } + return files; + }; + return { cleanup: () => rmSync(root, { force: true, recursive: true }), deps, logs, snapshot }; +} + +const BOTH_PINNED = [ + { connectorKey: "codex", version: "0.3.3" }, + { connectorKey: "imessage", version: "0.1.25" }, +]; + +async function assertRefusedAndUntouched(options: Parameters[0], message: RegExp) { + const t = tree(options); + try { + const before = t.snapshot(); + await assert.rejects(runPinScript(t.deps), message); + assert.deepEqual(t.snapshot(), before, "a refused run leaves every pin output as it was"); + } finally { + t.cleanup(); + } +} + +test("--version naming a release the catalog does not list fails and keeps every output", async () => { + await assertRefusedAndUntouched( + { + argv: ["--version", "codex=99.99.99"], + catalog: { codex: ["0.3.3", "0.3.5"], imessage: ["0.1.25"] }, + currentPins: BOTH_PINNED, + }, + /cannot pin codex.*99\.99\.99/ + ); +}); + +test("a current pin whose release left the catalog fails and keeps every output", async () => { + await assertRefusedAndUntouched( + { + argv: [], + catalog: { codex: ["0.3.5"], imessage: ["0.1.25"] }, + currentPins: BOTH_PINNED, + }, + /cannot pin codex.*0\.3\.3/ + ); +}); + +test("--latest on a pinned connector the catalog no longer lists fails and keeps every output", async () => { + await assertRefusedAndUntouched( + { + argv: ["--latest"], + catalog: { imessage: ["0.1.25", "0.1.26"] }, + currentPins: BOTH_PINNED, + }, + /cannot pin codex/ + ); +}); + +test("a connector with no pin and no request that the catalog does not list is left out, and the rest is written", async () => { + const t = tree({ + argv: [], + catalog: { imessage: ["0.1.25"] }, + currentPins: [{ connectorKey: "imessage", version: "0.1.25" }], + }); + try { + await runPinScript(t.deps); + assert.ok(t.logs.some((line) => line.startsWith("unpinned codex"))); + const files = t.snapshot(); + assert.deepEqual(Object.keys(files), [ + "pins.generated.ts", + "profiles/imessage.json", + "profiles/imessage.pin.json", + ]); + assert.match(files["pins.generated.ts"] ?? "", /"connectorKey": "imessage"/); + assert.doesNotMatch(files["pins.generated.ts"] ?? "", /"connectorKey": "codex"/); + } finally { + t.cleanup(); + } +}); + +test("a bare run rewrites both connectors from the pinned releases", async () => { + const t = tree({ + argv: [], + catalog: { codex: ["0.3.3", "0.3.5"], imessage: ["0.1.25", "0.1.26"] }, + currentPins: BOTH_PINNED, + }); + try { + await runPinScript(t.deps); + const files = t.snapshot(); + assert.match(files["pins.generated.ts"] ?? "", /"version": "0\.3\.3"/); + assert.match(files["pins.generated.ts"] ?? "", /"version": "0\.1\.25"/); + assert.equal(files["profiles/codex.json"], '{"profile":"codex 0.3.3"}\n'); + } finally { + t.cleanup(); + } +}); From b3b21d41bd1ff9e027e4bf2f7530f887ee23ec2a Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 6 Nov 2026 07:57:22 -0600 Subject: [PATCH 6/8] docs: name the 13 connectors the earlier installer core installs, and check the list Nothing stated which lock entries go to the earlier installer core. They came from `!entry.oci` plus the id set PRE_DECLARATION_OCI_CONNECTORS, and collection-profile-holds.json, a different axis with a similar name, sat beside them. The comment above the set now lists each of the 13, which axis holds it back (tarball, or no signed source declaration) and the issue that releases it (#633, #634), says why github-pdpp is named rather than derived (a lock entry and a catalog release carry digests and a version, not the declaration, and the catalog has no github entry; reading it needs a registry fetch per entry on every install and `--check`, or a new lock field), and says the profile hold is a separate thing. B-T10 checks that connectors/lock.json routes exactly those 13 entries to the earlier core and that each id appears in that comment, so the list cannot drift. With the set emptied it fails; with it restored it passes. usesEarlierInstallerCore is exported for the test. Pass rule (from the repository root; exact count at this commit): npx vitest run scripts/resolve-connectors.test.mjs 27 passed Assisted-by: AI Signed-off-by: Tim Nunamaker --- scripts/resolve-connectors.js | 40 ++++++++++++++++++++++++----- scripts/resolve-connectors.test.mjs | 27 +++++++++++++++++++ 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/scripts/resolve-connectors.js b/scripts/resolve-connectors.js index c6a074006..f2e6cd168 100644 --- a/scripts/resolve-connectors.js +++ b/scripts/resolve-connectors.js @@ -73,16 +73,44 @@ export const LOCKED_ARTIFACT_SOURCE = Object.freeze({ doc: Object.freeze({}), }) -// STOPGAP. OCI connectors whose pinned release has no signed source -// declaration, so the current installer core refuses it. They install and -// author through the earlier core, like the tarball connectors. Delete an id -// here once its connector is re-pinned to a release that carries a declaration -// (tracked in #634). +// STOPGAP. The earlier installer core installs exactly 13 lock entries, and +// this is the one place that names them. A guard test in +// resolve-connectors.test.mjs fails if connectors/lock.json and this list +// disagree. +// +// held back by connectors released by +// not an OCI artifact (tarball) chatgpt-playwright, #633 +// claude-export-playwright, +// github-playwright, +// heb-playwright, +// instagram-ads-playwright, +// instagram-playwright, +// linkedin-playwright, +// oura-playwright, +// shop-playwright, +// spotify-playwright, +// wholefoods-playwright, +// youtube-playwright +// no signed source declaration github-pdpp #634 +// +// The tarballs are picked out by `!entry.oci`. github-pdpp has to be named: +// nothing a lock entry or the catalog carries says its release lacks a signed +// source declaration. A lock entry holds the OCI digests and checksums, and a +// catalog release holds a digest and a version; the declaration is a layer of +// the artifact itself, and the catalog has no github entry at all. Reading it +// would need a registry fetch per entry on every install and `--check`, or a +// new lock field. Delete the id here when #634 re-pins github-pdpp to a release +// that carries a declaration. +// +// This is not the profile hold in +// packages/local-collector/collection-profile-holds.json. A hold keeps a +// collector pin below the catalog's latest release for a stated reason, and +// that pin installs through the current core. export const PRE_DECLARATION_OCI_CONNECTORS = Object.freeze( new Set(["github-pdpp"]) ) -const usesEarlierInstallerCore = entry => +export const usesEarlierInstallerCore = entry => !entry.oci || PRE_DECLARATION_OCI_CONNECTORS.has(entry.connectorId) function splitLockByInstallerCore(lock) { diff --git a/scripts/resolve-connectors.test.mjs b/scripts/resolve-connectors.test.mjs index 704b255ca..0bd568778 100644 --- a/scripts/resolve-connectors.test.mjs +++ b/scripts/resolve-connectors.test.mjs @@ -29,6 +29,7 @@ import { PRE_DECLARATION_OCI_CONNECTORS, resolveIndexUrl, recoverInterruptedInstall, + usesEarlierInstallerCore, } from "./resolve-connectors.js" const require = createRequire(import.meta.url) @@ -854,6 +855,32 @@ describe("OCI consumer acceptance", () => { } })) + it("B-T10 the earlier core installs exactly the 13 connectors resolve-connectors.js names", () => { + const named = [ + "chatgpt-playwright", + "claude-export-playwright", + "github-playwright", + "heb-playwright", + "instagram-ads-playwright", + "instagram-playwright", + "linkedin-playwright", + "oura-playwright", + "shop-playwright", + "spotify-playwright", + "wholefoods-playwright", + "youtube-playwright", + "github-pdpp", + ].sort() + const routed = lock.connectors + .filter(usesEarlierInstallerCore) + .map(connector => connector.connectorId) + .sort() + expect(routed).toEqual(named) + // The comment above PRE_DECLARATION_OCI_CONNECTORS is the one list of them. + const source = readFileSync("scripts/resolve-connectors.js", "utf8") + for (const id of named) expect(source).toContain(id) + }) + it("B-T9 a mixed lock installs through the default install: tarball by the earlier core, declaration-bearing OCI by the current core, declaration-less listed OCI by the earlier core; an unlisted declaration-less OCI is refused", async () => temporary(async root => { // `ynab` stands for a lock-delivered connector that has no generated From 0143fae0850ea7b7ccacb60b5ab042e741c688a0 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 6 Nov 2026 07:57:22 -0600 Subject: [PATCH 7/8] test: cover every re-pinned collector profile in the mixed-version registration test collector-profile-upgrade.test.ts proved a device survives the server registering the new profile for claude-code only. The other four re-pinned collector profiles (codex, google-takeout, apple-photos, imessage) now run the same registration against the verbatim profile the previous pin shipped, kept as a fixture: - enroll, register the previous profile, push, write a checkpoint; - register the new profile, push again; - the connection id, the checkpoint and the stored records survive, and the connection is still the only one projected; - for the connectors that keep a local-coverage proof (codex), the proof written under the previous profile reads generation_mismatch and is trusted again after the next write. google-takeout, apple-photos and imessage keep none. Their streams, keys and cursors do not change in the re-pin, so the claude-code test keeps the stream-level cases (usage refused by the old profile, accepted after registration). The helpers now take the connector id. Pass rule (from reference-implementation; exact counts at this commit): node --import tsx --test test/collector-profile-upgrade.test.ts 5 passed node --import tsx --test test/static-secret-run-credentials.test.ts \ test/collection-report-projection.test.ts \ test/collector-profile-upgrade.test.ts \ test/local-collector-profiles.test.ts 138 passed npx tsc --noEmit exit 0 Assisted-by: AI Signed-off-by: Tim Nunamaker --- .../test/collector-profile-upgrade.test.ts | 213 +++- .../apple-photos-0.1.0.json | 185 ++++ .../codex-0.3.0.json | 908 ++++++++++++++++++ .../google-takeout-0.1.0.json | 354 +++++++ .../imessage-0.1.0.json | 226 +++++ 5 files changed, 1848 insertions(+), 38 deletions(-) create mode 100644 reference-implementation/test/fixtures/collector-profile-upgrade/apple-photos-0.1.0.json create mode 100644 reference-implementation/test/fixtures/collector-profile-upgrade/codex-0.3.0.json create mode 100644 reference-implementation/test/fixtures/collector-profile-upgrade/google-takeout-0.1.0.json create mode 100644 reference-implementation/test/fixtures/collector-profile-upgrade/imessage-0.1.0.json diff --git a/reference-implementation/test/collector-profile-upgrade.test.ts b/reference-implementation/test/collector-profile-upgrade.test.ts index b5b74a2eb..4136972ed 100644 --- a/reference-implementation/test/collector-profile-upgrade.test.ts +++ b/reference-implementation/test/collector-profile-upgrade.test.ts @@ -17,7 +17,13 @@ * stamped under the previous profile reads `generation_mismatch` (`unknown`), * and the next run's coverage write brings the axis back. * - * `fixtures/collector-profile-upgrade/claude-code-0.3.0.json` is the verbatim + * The same registration is then repeated for the other four re-pinned profiles + * (codex, google-takeout, apple-photos, imessage), whose streams and keys do not + * change: the connection id, the checkpoint and the records survive, and a + * local-coverage proof written under the previous profile is untrusted until + * the next run (the connectors that keep one). + * + * `fixtures/collector-profile-upgrade/-.json` is the verbatim * profile the previous pin shipped. Only the device-side process is simulated; * enrollment, registration, ingest, state and the health projection are the * production paths. @@ -44,6 +50,7 @@ import { fetchJson, type TestServer } from "./helpers/legacy-upgrade.ts"; const HERE = dirname(fileURLToPath(import.meta.url)); const CONNECTOR_ID = "claude-code"; +const FIXTURES = join(HERE, "fixtures", "collector-profile-upgrade"); const PROTOCOL_HEADERS = { "X-PDPP-Collector-Protocol": COLLECTOR_PROTOCOL_VERSION }; function postJson(url: string, body: unknown, headers: Record = {}) { @@ -77,12 +84,12 @@ function bodyHash(records: unknown): string { } /** The manifest the server registers for a local-collector connector (see `readReferenceLocalConnectorCatalogManifest`). */ -function enrollmentManifest(profile: Record) { - return { ...profile, connector_id: CONNECTOR_ID, display_name: profile.display_name || CONNECTOR_ID }; +function enrollmentManifest(connectorId: string, profile: Record) { + return { ...profile, connector_id: connectorId, display_name: profile.display_name || connectorId }; } -function register(profile: Record) { - return registerConnector(enrollmentManifest(profile), { +function register(connectorId: string, profile: Record) { + return registerConnector(enrollmentManifest(connectorId, profile), { backfillRetrievalIndexes: false, replaceAcceptedDigest: true, }); @@ -96,7 +103,7 @@ interface Device { } let batchSeq = 0; -function push(asUrl: string, device: Device, records: readonly Record[]) { +function push(asUrl: string, device: Device, connectorId: string, records: readonly Record[]) { batchSeq += 1; return postJson( `${asUrl}/_ref/device-exporters/${encodeURIComponent(device.device_id)}/ingest-batches`, @@ -104,7 +111,7 @@ function push(asUrl: string, device: Device, records: readonly Record) { } /** One `coverage_diagnostics` state write, as the connector commits it at the end of a successful run. */ -const COVERAGE_STORES = [ - { status: "collected", store: "projects", stream: "sessions" }, - { status: "collected", store: "derived_messages", stream: "messages" }, - { status: "collected", store: "derived_attachments", stream: "attachments" }, - { status: "collected", store: "derived_memory_notes", stream: "memory_notes" }, - { status: "inventory_only", store: "cache", stream: null }, - { status: "excluded", store: "auth", stream: null }, -]; +const CLAUDE_CODE_STORE_STATUSES = new Map([ + ["projects", "collected"], + ["derived_messages", "collected"], + ["derived_attachments", "collected"], + ["derived_memory_notes", "collected"], + ["cache", "inventory_only"], + ["auth", "excluded"], +]); -function coverageRows() { - const expected = expectedLocalCoverageStoreDescriptors(CONNECTOR_ID); - assert.ok(expected, "the production claude-code connector declares a fixed inventory"); - const supplied = new Map(COVERAGE_STORES.map((row) => [row.store, row])); +/** + * The stores the connector's fixed inventory declares. `statuses` says which are + * collected; the rest are inventory only. + */ +function coverageRows(connectorId: string, statuses: ReadonlyMap = new Map()) { + const expected = expectedLocalCoverageStoreDescriptors(connectorId); + assert.ok(expected, `the production ${connectorId} connector declares a fixed inventory`); return expected.map(({ store, stream }) => ({ - status: supplied.get(store)?.status ?? "inventory_only", + status: statuses.get(store) ?? "inventory_only", store, stream, })); } -async function commitCoverage(connectorInstanceId: string, rows: ReturnType) { +async function commitCoverage( + connectorId: string, + connectorInstanceId: string, + rows: ReturnType +) { for (const row of rows) { getDb() .prepare( @@ -147,7 +161,7 @@ async function commitCoverage(connectorInstanceId: string, rows: ReturnType[0]; const { axis, reliable, unreliableReason } = deriveLocalCoverageAxis(proof); @@ -175,6 +189,10 @@ async function projectedConnectionIds(): Promise { return (await listConnectorSummaries()).map((summary) => summary.connector_instance_id); } +function readProfile(file: string): Record { + return JSON.parse(readFileSync(join(FIXTURES, file), "utf8")) as Record; +} + function stateRow(connectorInstanceId: string, stream: string) { return getDb() .prepare("SELECT state_json FROM connector_state WHERE connector_instance_id = ? AND stream = ?") @@ -190,9 +208,7 @@ function recordCount(connectorInstanceId: string, stream: string): number { } test("a device enrolled on the previous claude-code profile survives the server registering the new profile, in both push directions", async () => { - const previous = JSON.parse( - readFileSync(join(HERE, "fixtures", "collector-profile-upgrade", "claude-code-0.3.0.json"), "utf8") - ) as Record; + const previous = readProfile("claude-code-0.3.0.json"); const current = readLocalCollectorProfile(CONNECTOR_ID); assert.ok(current, "claude-code must have a pinned Collection Profile"); const names = (profile: Record) => @@ -225,9 +241,9 @@ test("a device enrolled on the previous claude-code profile survives the server const connectionId = device.connector_instance_id; // The server a previous release left behind: it holds the previous profile. - await register(previous); + await register(CONNECTOR_ID, previous); - const first = await push(asUrl, device, [ + const first = await push(asUrl, device, CONNECTOR_ID, [ emitted("sessions", "s1", { id: "s1" }), emitted("messages", "m1", { id: "m1", session_id: "s1" }), ]); @@ -235,8 +251,8 @@ test("a device enrolled on the previous claude-code profile survives the server assert.equal(first.body.connector_instance_id, connectionId); const checkpoint = { cursor: "sessions-checkpoint-1" }; await putSyncState({ connector_id: CONNECTOR_ID, connector_instance_id: connectionId }, { sessions: checkpoint }); - await commitCoverage(connectionId, coverageRows()); - assert.deepEqual(await localCoverage(connectionId), { axis: "complete", reliable: true, unreliableReason: undefined }); + await commitCoverage(CONNECTOR_ID, connectionId, coverageRows(CONNECTOR_ID, CLAUDE_CODE_STORE_STATUSES)); + assert.deepEqual(await localCoverage(CONNECTOR_ID, connectionId), { axis: "complete", reliable: true, unreliableReason: undefined }); assert.deepEqual(await projectedConnectionIds(), [connectionId]); // New collector, previous server: `usage` is not declared, so the push is refused and stores nothing. @@ -250,15 +266,15 @@ test("a device enrolled on the previous claude-code profile survives the server source: "stats-cache", total_cost_usd_cents: 0, }); - const refused = await push(asUrl, device, [usage]); + const refused = await push(asUrl, device, CONNECTOR_ID, [usage]); assert.equal(refused.status, 400, JSON.stringify(refused.body)); assert.equal(refused.body.error.message, "records[0].stream is not declared by the connector manifest"); assert.equal(recordCount(connectionId, "usage"), 0); // The server registers the new profile. - await register(current); + await register(CONNECTOR_ID, current); - const accepted = await push(asUrl, device, [usage]); + const accepted = await push(asUrl, device, CONNECTOR_ID, [usage]); assert.equal(accepted.status, 201, JSON.stringify(accepted.body)); assert.equal(accepted.body.connector_instance_id, connectionId); assert.equal(recordCount(connectionId, "usage"), 1); @@ -269,16 +285,137 @@ test("a device enrolled on the previous claude-code profile survives the server assert.deepEqual(await projectedConnectionIds(), [connectionId]); // The coverage proof was stamped under the previous profile, so it is not // trusted until the next run writes it again (`generation_mismatch`). - assert.deepEqual(await localCoverage(connectionId), { + assert.deepEqual(await localCoverage(CONNECTOR_ID, connectionId), { axis: "unknown", reliable: false, unreliableReason: "generation_mismatch", }); - await commitCoverage(connectionId, coverageRows()); - assert.deepEqual(await localCoverage(connectionId), { axis: "complete", reliable: true, unreliableReason: undefined }); + await commitCoverage(CONNECTOR_ID, connectionId, coverageRows(CONNECTOR_ID, CLAUDE_CODE_STORE_STATUSES)); + assert.deepEqual(await localCoverage(CONNECTOR_ID, connectionId), { axis: "complete", reliable: true, unreliableReason: undefined }); assert.deepEqual(await projectedConnectionIds(), [connectionId]); } finally { await closeTestServer(server); } }); + +interface StreamSchema { + properties?: Record; + required?: string[]; +} + +/** The smallest record the profile's own schema accepts for `stream`. */ +function minimalRecord(profile: Record, stream: string, key: string) { + const declared = (profile.streams as { name: string; schema: StreamSchema }[]).find((s) => s.name === stream); + assert.ok(declared, `${stream} is declared`); + const data: Record = {}; + for (const name of declared.schema.required ?? []) { + const property = declared.schema.properties?.[name] ?? {}; + const type = [property.type].flat().find((t) => t !== "null"); + if (name === "id") { + data[name] = key; + } else if (property.enum) { + data[name] = property.enum[0]; + } else if (property.format === "date-time") { + data[name] = "2026-11-05T10:00:00.000Z"; + } else if (type === "integer" || type === "number") { + data[name] = 1; + } else if (type === "boolean") { + data[name] = true; + } else { + data[name] = "x"; + } + } + return emitted(stream, key, data); +} + +/** + * The four other re-pinned profiles. Their streams, keys and cursors do not + * change, so the claude-code test above carries the stream-level behaviour; this + * one proves the registration itself is safe for each of them, against the + * verbatim profile the previous pin shipped. + */ +const OTHER_REPINNED = [ + { id: "codex", previous: "codex-0.3.0.json" }, + { id: "google-takeout", previous: "google-takeout-0.1.0.json" }, + { id: "apple-photos", previous: "apple-photos-0.1.0.json" }, + { id: "imessage", previous: "imessage-0.1.0.json" }, +]; + +for (const { id, previous: previousFile } of OTHER_REPINNED) { + test(`${id}: a device enrolled on the previous profile keeps its connection, checkpoint and records when the server registers the new profile`, async () => { + const previous = readProfile(previousFile); + const current = readLocalCollectorProfile(id); + assert.ok(current, `${id} must have a pinned Collection Profile`); + assert.notEqual(previous.version, current.version); + const streamNames = (profile: Record) => + (profile.streams as { name: string }[]).map((stream) => stream.name).sort(); + assert.deepEqual(streamNames(previous), streamNames(current), "the re-pin leaves the declared streams alone"); + const stream = (current.streams as { name: string }[]).find((s) => s.name !== "coverage_diagnostics")?.name; + assert.ok(stream); + const hasCoverageProof = expectedLocalCoverageStoreDescriptors(id) !== null; + + const server = (await startServer({ + asPort: 0, + dbPath: ":memory:", + quiet: true, + reconcilePolyfillManifests: false, + rsPort: 0, + })) as TestServer; + try { + const asUrl = `http://localhost:${server.asPort}`; + const code = await postJson(`${asUrl}/_ref/device-exporters/enrollment-codes`, { + connector_id: id, + local_binding_name: "laptop", + }); + assert.equal(code.status, 201, JSON.stringify(code.body)); + const enrolled = await postJson( + `${asUrl}/_ref/device-exporters/enroll`, + { enrollment_code: code.body.enrollment_code }, + PROTOCOL_HEADERS + ); + assert.equal(enrolled.status, 201, JSON.stringify(enrolled.body)); + const device = enrolled.body as Device; + const connectionId = device.connector_instance_id; + + await register(id, previous); + const first = await push(asUrl, device, id, [minimalRecord(previous, stream, "r1")]); + assert.equal(first.status, 201, JSON.stringify(first.body)); + assert.equal(first.body.connector_instance_id, connectionId); + const checkpoint = { cursor: `${id}-checkpoint-1` }; + await putSyncState({ connector_id: id, connector_instance_id: connectionId }, { [stream]: checkpoint }); + const proof = () => + coverageRows( + id, + new Map( + (expectedLocalCoverageStoreDescriptors(id) ?? []).filter((d) => d.stream).map((d) => [d.store, "collected"]) + ) + ); + if (hasCoverageProof) { + await commitCoverage(id, connectionId, proof()); + assert.equal((await localCoverage(id, connectionId)).reliable, true); + } + assert.deepEqual(await projectedConnectionIds(), [connectionId]); + + await register(id, current); + + const second = await push(asUrl, device, id, [minimalRecord(current, stream, "r2")]); + assert.equal(second.status, 201, JSON.stringify(second.body)); + assert.equal(second.body.connector_instance_id, connectionId); + assert.equal(recordCount(connectionId, stream), 2); + assert.deepEqual(JSON.parse(stateRow(connectionId, stream) ?? "null"), checkpoint); + assert.deepEqual(await projectedConnectionIds(), [connectionId]); + if (hasCoverageProof) { + assert.deepEqual(await localCoverage(id, connectionId), { + axis: "unknown", + reliable: false, + unreliableReason: "generation_mismatch", + }); + await commitCoverage(id, connectionId, proof()); + assert.equal((await localCoverage(id, connectionId)).reliable, true); + } + } finally { + await closeTestServer(server); + } + }); +} diff --git a/reference-implementation/test/fixtures/collector-profile-upgrade/apple-photos-0.1.0.json b/reference-implementation/test/fixtures/collector-profile-upgrade/apple-photos-0.1.0.json new file mode 100644 index 000000000..42e5d2c8e --- /dev/null +++ b/reference-implementation/test/fixtures/collector-profile-upgrade/apple-photos-0.1.0.json @@ -0,0 +1,185 @@ +{ + "protocol_version": "0.1.0", + "connector_id": "https://registry.pdpp.dev/connectors/apple-photos", + "connector_key": "apple-photos", + "manifest_uri": "https://registry.pdpp.dev/connectors/apple-photos", + "version": "0.1.0", + "display_name": "Apple Photos", + "brand": { + "icon": "icons/apple_photos.svg", + "background_color": "#FA633E" + }, + "runtime_requirements": { + "bindings": { + "filesystem": { + "required": true + } + } + }, + "capabilities": { + "human_interaction": [], + "proven": { + "local_collector": true + }, + "refresh_policy": { + "recommended_mode": "manual", + "interaction_posture": "manual_action_likely", + "rate_limit_sensitivity": "low", + "bot_detection_sensitivity": "low", + "background_safe": false, + "rationale": "Apple Photos imports require the user to manually export files from Photos.app (File → Export → Export Unmodified Originals) into the import directory; there is no API or background-refreshable path — PhotoKit requires a signed app bundle and is not reachable from this connector, and the reverse-engineered Photos.sqlite path is out of scope by design." + }, + "public_listing": { + "tier": "development" + } + }, + "streams": [ + { + "name": "photos", + "required": true, + "description": "File-level metadata AND hydrated bytes for images and videos exported from Photos.app, within a configured size cap. Uses the same shared local-media blob hydration module as google_takeout's photos stream (src/local-media-blob-hydration.ts) — bounded local read, sha256 content hash, MIME-by-extension, and upload through the runtime's reference-blob-uploader when blob-upload bindings are configured. A record's `id` is derived from content_sha256 alone, so the same photo present in more than one Photos.app export collapses to one record rather than duplicating.", + "display": { + "label": "Your exported photos and videos", + "detail": "Filename, size, content hash, MIME type, file-modified timestamp, and (within the size cap) hydrated bytes for each file found in the Photos.app export directory. hydration_status/hydration_error explain when bytes are absent (oversized or unreadable) without dropping the record. EXIF/XMP fields (taken date, GPS, camera) are present in the schema but not populated in this cut — a bounded, explicit omission, not silently missing coverage." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "filename": { + "type": "string" + }, + "content_type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "content_sha256": { + "type": ["string", "null"] + }, + "hydration_status": { + "type": "string", + "enum": ["failed", "hydrated", "skipped_too_large", "unavailable"] + }, + "hydration_error": { + "type": ["string", "null"] + }, + "blob_ref": { + "type": ["object", "null"], + "properties": { + "blob_id": { + "type": "string" + }, + "mime_type": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "size_bytes": { + "type": "integer" + } + }, + "required": ["blob_id", "mime_type", "sha256", "size_bytes"] + }, + "taken_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "file_modified_at": { + "type": "string", + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "latitude": { + "type": ["number", "null"] + }, + "longitude": { + "type": ["number", "null"] + }, + "camera_make": { + "type": ["string", "null"] + }, + "camera_model": { + "type": ["string", "null"] + } + }, + "required": [ + "id", + "filename", + "content_type", + "hydration_status", + "file_modified_at" + ] + }, + "primary_key": ["id"], + "cursor_field": "file_modified_at", + "consent_time_field": "file_modified_at", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "query": { + "range_filters": { + "file_modified_at": ["gte", "gt", "lte", "lt"], + "taken_at": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["file_modified_at", "taken_at"], + "group_by": ["content_type"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "manual_as_of" + }, + { + "name": "coverage_diagnostics", + "required": false, + "description": "Safe coverage status for this connector's one known local store (the Photos.app export directory), so a local-collector run always leaves durable, honest coverage evidence even when the export directory is missing or empty — never a silent zero-evidence run.", + "display": { + "label": "Local collector coverage status", + "detail": "Whether the Photos.app export directory was found and collected on this run." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "stream": { + "type": ["string", "null"] + }, + "status": { + "type": "string" + }, + "reason": { + "type": "string", + "x_pdpp_role": "primary-title" + } + }, + "required": ["id", "store", "status", "reason"] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + } + ], + "reason_display_messages": { + "export_not_found": "We couldn't find an export to import" + } +} diff --git a/reference-implementation/test/fixtures/collector-profile-upgrade/codex-0.3.0.json b/reference-implementation/test/fixtures/collector-profile-upgrade/codex-0.3.0.json new file mode 100644 index 000000000..62d967652 --- /dev/null +++ b/reference-implementation/test/fixtures/collector-profile-upgrade/codex-0.3.0.json @@ -0,0 +1,908 @@ +{ + "protocol_version": "0.1.0", + "connector_id": "https://registry.pdpp.dev/connectors/codex", + "connector_key": "codex", + "manifest_uri": "https://registry.pdpp.dev/connectors/codex", + "version": "0.3.0", + "display_name": "OpenAI Codex CLI", + "brand": { + "icon": "icons/codex.svg", + "background_color": "#10A37F" + }, + "runtime_requirements": { + "bindings": { + "filesystem": { + "required": true + } + }, + "environment_variables": [ + "CODEX_RULES_DIR", + "CODEX_PROMPTS_DIR", + "CODEX_SKILLS_DIR" + ], + "local_paths": { + "home_env_override": "CODEX_HOME", + "home_default_relative_to_user_home": ".codex", + "paths": [ + { + "default_relative_to_home": "sessions", + "env_override": "CODEX_SESSIONS_DIR", + "label": "sessions directory", + "required_for_readiness": true + }, + { + "default_relative_to_home": "state_5.sqlite", + "env_override": "CODEX_STATE_DB", + "label": "state database", + "required_for_readiness": true + } + ] + } + }, + "capabilities": { + "human_interaction": [], + "proven": { + "local_collector": true + }, + "refresh_policy": { + "recommended_mode": "automatic", + "recommended_interval_seconds": 3600, + "minimum_interval_seconds": 300, + "maximum_staleness_seconds": 21600, + "interaction_posture": "none", + "rate_limit_sensitivity": "low", + "bot_detection_sensitivity": "low", + "background_safe": true, + "rationale": "Codex CLI session state lives on local disk; re-reading it is cheap." + }, + "public_listing": { + "tier": "supported" + } + }, + "streams": [ + { + "name": "sessions", + "required": true, + "description": "Codex CLI sessions. Canonical metadata comes from ~/.codex/state_5.sqlite#threads (title, archived, tokens_used, first_user_message, sandbox_policy, approval_mode), enriched with message/tool-call counts from the on-disk rollout JSONL.", + "compaction_class": "recurring_snapshot", + "compaction_class_note": "whole record is an evolving observation (message_count, last_event_at widen on growth); never compactable, never re-alarms as growth accumulates.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Codex CLI sessions", + "detail": "Title, working directory, git commit/branch, model provider, CLI version, approval mode, sandbox policy, token usage, and counts." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "cwd": { + "type": ["string", "null"] + }, + "originator": { + "type": ["string", "null"] + }, + "cli_version": { + "type": ["string", "null"] + }, + "model_provider": { + "type": ["string", "null"] + }, + "git_commit": { + "type": ["string", "null"] + }, + "git_branch": { + "type": ["string", "null"] + }, + "repository_url": { + "type": ["string", "null"] + }, + "started_at": { + "type": ["string", "null"], + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "last_event_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "message_count": { + "type": ["integer", "null"] + }, + "function_call_count": { + "type": ["integer", "null"] + }, + "title": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "archived": { + "type": ["boolean", "null"] + }, + "tokens_used": { + "type": ["integer", "null"] + }, + "first_user_message": { + "type": ["string", "null"] + }, + "sandbox_policy": { + "type": ["string", "null"] + }, + "approval_mode": { + "type": ["string", "null"] + }, + "rollout_path": { + "type": ["string", "null"] + } + }, + "required": ["id"] + }, + "primary_key": ["id"], + "cursor_field": "last_event_at", + "consent_time_field": "started_at", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "query": { + "search": { + "lexical_fields": [ + "cwd", + "originator", + "model_provider", + "git_branch", + "title", + "first_user_message", + "sandbox_policy", + "approval_mode" + ], + "semantic_fields": ["title", "first_user_message"] + }, + "range_filters": { + "started_at": ["gte", "gt", "lte", "lt"], + "last_event_at": ["gte", "gt", "lte", "lt"], + "message_count": ["gte", "gt", "lte", "lt"], + "function_call_count": ["gte", "gt", "lte", "lt"], + "tokens_used": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["last_event_at", "started_at"] + } + }, + "coverage_strategy": "checkpoint_window", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "messages", + "required": true, + "description": "Messages within Codex sessions: user prompts, assistant responses, and reasoning placeholders.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Codex messages", + "detail": "Role (user/assistant), text content, timestamp, session reference. Reasoning traces are referenced but content is encrypted and not stored." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "session_id": { + "type": "string" + }, + "role": { + "type": ["string", "null"], + "x_pdpp_role": "actor" + }, + "type": { + "type": ["string", "null"] + }, + "content": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time", + "x_pdpp_role": "event-time" + } + }, + "required": ["id", "session_id"] + }, + "primary_key": ["id"], + "cursor_field": "timestamp", + "consent_time_field": "timestamp", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "relationships": [ + { + "name": "session", + "stream": "sessions", + "foreign_key": "session_id", + "cardinality": "has_one" + } + ], + "query": { + "search": { + "lexical_fields": ["role", "type", "content"], + "semantic_fields": ["content"] + }, + "range_filters": { + "timestamp": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["timestamp"] + } + }, + "coverage_strategy": "checkpoint_window", + "state_stream": "sessions", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "function_calls", + "required": true, + "description": "Tool invocations and their outputs (shell commands, file ops) recorded in Codex sessions.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Codex tool calls", + "detail": "Tool name, arguments, call_id, and output preview. Includes shell commands Codex ran and their results." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "session_id": { + "type": "string" + }, + "call_id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "arguments": { + "type": ["string", "null"] + }, + "output_preview": { + "type": ["string", "null"] + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time", + "x_pdpp_role": "event-time" + } + }, + "required": ["id", "session_id"] + }, + "primary_key": ["id"], + "cursor_field": "timestamp", + "consent_time_field": "timestamp", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "relationships": [ + { + "name": "session", + "stream": "sessions", + "foreign_key": "session_id", + "cardinality": "has_one" + } + ], + "query": { + "search": { + "lexical_fields": ["name", "arguments", "output_preview"], + "semantic_fields": ["arguments", "output_preview"] + }, + "range_filters": { + "timestamp": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["timestamp"] + } + }, + "coverage_strategy": "checkpoint_window", + "state_stream": "sessions", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "rules", + "required": true, + "description": "Personal trust-registry rules recorded in ~/.codex/rules/*.rules — the de-facto log of approval decisions. One record per rule line. The stream is named `rules` to match the real local directory; `approval_rules` was backlog shorthand, not an on-disk source name.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Codex approval rules", + "detail": "Each rule line (e.g. prefix_rule / decision=allow) from your .rules files, with ruleset and position." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "ruleset": { + "type": "string" + }, + "rule_text": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "rule_index": { + "type": "integer" + }, + "path": { + "type": ["string", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + } + }, + "required": ["id", "ruleset", "rule_text", "rule_index"] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "search": { + "lexical_fields": ["ruleset", "rule_text", "path"], + "semantic_fields": ["rule_text"] + }, + "range_filters": { + "mtime_epoch": ["gte", "gt", "lte", "lt"] + } + }, + "coverage_strategy": "full_inventory", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "prompts", + "required": true, + "description": "User-authored prompt templates from ~/.codex/prompts/*.md. Frontmatter (name, description) is parsed; body is the prompt content.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Codex prompts", + "detail": "Name, description, body, and source path for each prompt template." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "description": { + "type": ["string", "null"], + "x_pdpp_role": "secondary" + }, + "content": { + "type": ["string", "null"] + }, + "path": { + "type": ["string", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"] + } + }, + "required": ["id", "name"] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "search": { + "lexical_fields": ["name", "description", "content", "path"], + "semantic_fields": ["description", "content"] + }, + "range_filters": { + "mtime_epoch": ["gte", "gt", "lte", "lt"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "skills", + "required": true, + "description": "User-authored skills from ~/.codex/skills//SKILL.md. Frontmatter (name, description) is parsed; body is the skill content.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Codex skills", + "detail": "Name, description, body, and source path for each skill." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "description": { + "type": ["string", "null"], + "x_pdpp_role": "secondary" + }, + "content": { + "type": ["string", "null"], + "x_pdpp_role": "secondary" + }, + "path": { + "type": ["string", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"] + } + }, + "required": ["id", "name"] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "search": { + "lexical_fields": ["name", "description", "content", "path"], + "semantic_fields": ["description", "content"] + }, + "range_filters": { + "mtime_epoch": ["gte", "gt", "lte", "lt"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "history", + "description": "Metadata-only inventory for Codex history.jsonl. Prompt-history payloads are not collected in this tranche.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "session_index", + "description": "Metadata-only inventory for Codex session_index.jsonl.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "coverage_diagnostics", + "description": "Safe coverage status for known Codex local stores, including excluded auth-adjacent stores without payloads or raw absolute paths.", + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "stream": { + "type": ["string", "null"] + }, + "status": { + "type": "string" + }, + "reason": { + "type": "string", + "x_pdpp_role": "primary-title" + } + }, + "required": ["id", "store", "status", "reason"] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "shell_snapshots", + "description": "Metadata-only inventory for Codex shell snapshots; shell content is not collected.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "config_inventory", + "description": "Metadata-only inventory for non-secret Codex configuration; raw config payloads are not collected.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "cache_inventory", + "description": "Metadata-only inventory for Codex cache stores; raw cache payloads are not collected.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + } + ], + "profiles": [ + { + "id": "full", + "label": "Your full Codex history", + "streams": [ + { + "name": "sessions" + }, + { + "name": "messages" + }, + { + "name": "function_calls" + }, + { + "name": "rules" + }, + { + "name": "prompts" + }, + { + "name": "skills" + } + ] + }, + { + "id": "messages_only", + "label": "Just prompts and responses", + "streams": [ + { + "name": "sessions" + }, + { + "name": "messages" + } + ] + }, + { + "id": "authoring", + "label": "Your prompts, skills, and approval rules", + "streams": [ + { + "name": "rules" + }, + { + "name": "prompts" + }, + { + "name": "skills" + } + ] + } + ], + "reason_display_messages": { + "symlink_skipped": "A symbolic link in your Codex data was skipped without reading its target", + "malformed_jsonl_line": "Part of this session's log file couldn't be read, so those entries were skipped. Nothing was lost on your machine — the original file is unchanged", + "non_object_jsonl_record": "A line in this Codex session log has an unexpected format and was skipped", + "rollout_source_read_error": "We couldn't read part of your Codex data. It will be checked again on the next run", + "shape_check_failed": "The data didn't look like we expected", + "truncated_jsonl_tail": "The last line of this Codex session log is incomplete and will be checked again on the next run" + } +} diff --git a/reference-implementation/test/fixtures/collector-profile-upgrade/google-takeout-0.1.0.json b/reference-implementation/test/fixtures/collector-profile-upgrade/google-takeout-0.1.0.json new file mode 100644 index 000000000..b942a406d --- /dev/null +++ b/reference-implementation/test/fixtures/collector-profile-upgrade/google-takeout-0.1.0.json @@ -0,0 +1,354 @@ +{ + "protocol_version": "0.1.0", + "connector_id": "https://registry.pdpp.dev/connectors/google-takeout", + "connector_key": "google-takeout", + "manifest_uri": "https://registry.pdpp.dev/connectors/google-takeout", + "version": "0.1.0", + "display_name": "Google Takeout", + "brand": { + "icon": "icons/google_takeout.svg", + "background_color": "#FBBC05" + }, + "runtime_requirements": { + "bindings": { + "network": { + "required": false + }, + "filesystem": { + "required": true + } + }, + "environment_variables": ["GOOGLE_TAKEOUT_DIR"] + }, + "capabilities": { + "human_interaction": [], + "proven": { + "local_collector": true + }, + "refresh_policy": { + "recommended_mode": "manual", + "interaction_posture": "manual_action_likely", + "rate_limit_sensitivity": "low", + "bot_detection_sensitivity": "low", + "background_safe": false, + "rationale": "Google Takeout requires the user to request and download a fresh archive; refresh cannot run in the background." + }, + "public_listing": { + "tier": "development" + } + }, + "streams": [ + { + "name": "location_history", + "description": "Location history from Google Maps Timeline.", + "display": { + "label": "Your Google location history", + "detail": "Places you've been with timestamps, lat/lon, accuracy, activity type when available. Parsed from Google Takeout Records.json." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "timestamp": { + "type": "string", + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "latitude": { + "type": ["number", "null"] + }, + "longitude": { + "type": ["number", "null"] + }, + "accuracy_meters": { + "type": ["number", "null"] + }, + "activity_type": { + "type": ["string", "null"] + }, + "velocity_mps": { + "type": ["number", "null"] + }, + "altitude_m": { + "type": ["number", "null"] + } + }, + "required": ["id", "timestamp"] + }, + "primary_key": ["id"], + "cursor_field": "timestamp", + "consent_time_field": "timestamp", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "query": { + "range_filters": { + "timestamp": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["timestamp"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "manual_as_of" + }, + { + "name": "youtube_watch_history", + "description": "YouTube watch history.", + "display": { + "label": "Your YouTube watch history", + "detail": "Videos watched with watch time, video URL, title, channel name." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "watched_at": { + "type": "string", + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "video_url": { + "type": ["string", "null"] + }, + "video_title": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "channel_name": { + "type": ["string", "null"], + "x_pdpp_role": "actor" + }, + "channel_url": { + "type": ["string", "null"] + } + }, + "required": ["id", "watched_at"] + }, + "primary_key": ["id"], + "cursor_field": "watched_at", + "consent_time_field": "watched_at", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "query": { + "range_filters": { + "watched_at": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["watched_at"] + }, + "search": { + "lexical_fields": ["video_title", "channel_name"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "manual_as_of" + }, + { + "name": "search_history", + "description": "Google search queries from My Activity.", + "display": { + "label": "Your Google search history", + "detail": "Search queries with timestamps and products (Search, Maps, Images, etc.)." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "timestamp": { + "type": "string", + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "query": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "product": { + "type": ["string", "null"] + } + }, + "required": ["id", "timestamp"] + }, + "primary_key": ["id"], + "cursor_field": "timestamp", + "consent_time_field": "timestamp", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "query": { + "range_filters": { + "timestamp": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["timestamp"] + }, + "search": { + "lexical_fields": ["query"], + "semantic_fields": ["query"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "manual_as_of" + }, + { + "name": "photos", + "description": "Photos and videos from Google Takeout.", + "display": { + "label": "Your Google photos and videos", + "detail": "Photos/videos with metadata: timestamps, title, description, geolocation when available. Full-resolution media. Parsed from Google Takeout Photos/ directory." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "filename": { + "type": "string" + }, + "event_time": { + "type": "string", + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "title": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "description": { + "type": ["string", "null"] + }, + "latitude": { + "type": ["number", "null"] + }, + "longitude": { + "type": ["number", "null"] + }, + "altitude": { + "type": ["number", "null"] + }, + "blob_ref": { + "type": ["object", "null"], + "properties": { + "blob_id": { + "type": "string" + }, + "mime_type": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "size_bytes": { + "type": "integer" + } + }, + "required": ["blob_id", "mime_type", "sha256", "size_bytes"] + }, + "content_sha256": { + "type": ["string", "null"] + }, + "size_bytes": { + "type": ["number", "null"] + }, + "hydration_status": { + "type": "string", + "enum": ["failed", "hydrated", "skipped_too_large", "unavailable"] + }, + "hydration_error": { + "type": ["string", "null"] + } + }, + "required": ["id", "filename", "event_time"] + }, + "required": true, + "primary_key": ["id"], + "cursor_field": "event_time", + "consent_time_field": "event_time", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "range_filters": { + "event_time": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["event_time"] + }, + "search": { + "lexical_fields": ["title", "description"], + "semantic_fields": ["title", "description"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "manual_as_of" + }, + { + "name": "coverage_diagnostics", + "description": "Safe coverage status for known Google Takeout local stores.", + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "stream": { + "type": ["string", "null"] + }, + "status": { + "type": "string" + }, + "reason": { + "type": "string", + "x_pdpp_role": "primary-title" + } + }, + "required": ["id", "store", "status", "reason"] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + } + ], + "reason_display_messages": { + "directory_read_failed": "We couldn't read a folder in your Google Takeout export", + "history_not_found": "We couldn't find any history to import", + "photos_not_found": "We couldn't find your Google Photos export", + "records_not_found": "We didn't find any records to import" + } +} diff --git a/reference-implementation/test/fixtures/collector-profile-upgrade/imessage-0.1.0.json b/reference-implementation/test/fixtures/collector-profile-upgrade/imessage-0.1.0.json new file mode 100644 index 000000000..ac221b9ac --- /dev/null +++ b/reference-implementation/test/fixtures/collector-profile-upgrade/imessage-0.1.0.json @@ -0,0 +1,226 @@ +{ + "protocol_version": "0.1.0", + "connector_id": "https://registry.pdpp.dev/connectors/imessage", + "connector_key": "imessage", + "manifest_uri": "https://registry.pdpp.dev/connectors/imessage", + "version": "0.1.0", + "display_name": "iMessage (macOS)", + "brand": { + "icon": "icons/imessage.svg", + "background_color": "#30D158" + }, + "runtime_requirements": { + "bindings": { + "filesystem": { + "required": true + } + }, + "environment_variables": ["IMESSAGE_DB_PATH", "IMESSAGE_ATTACHMENTS_ROOT"] + }, + "capabilities": { + "human_interaction": [], + "proven": { + "local_collector": true + }, + "public_listing": { + "tier": "development", + "rationale": "Hidden from the reference dashboard catalog until an operator proves a local macOS Messages database run on this deployment and explicitly opts it into listing." + }, + "refresh_policy": { + "recommended_mode": "automatic", + "recommended_interval_seconds": 3600, + "minimum_interval_seconds": 300, + "maximum_staleness_seconds": 21600, + "interaction_posture": "none", + "rate_limit_sensitivity": "low", + "bot_detection_sensitivity": "low", + "background_safe": true, + "rationale": "iMessage history is read from the operator's local macOS Messages database over a filesystem binding: no network provider, no interactive login, nothing to assist, so unattended refresh is mechanically safe. Whether the database is reachable is a runtime-requirements question (filesystem binding, IMESSAGE_DB_PATH) enforced by the pre-run deployment-readiness gate, not by refresh mode; Docker/provider runs must use a local collector or an explicitly mounted path." + } + }, + "streams": [ + { + "name": "messages", + "description": "iMessage + SMS messages from ~/Library/Messages/chat.db (macOS).", + "display": { + "label": "Your iMessages", + "detail": "Sender/recipient, text content, timestamp, service (iMessage or SMS), is_from_me flag, attachment presence." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "chat_id": { + "type": ["string", "null"] + }, + "handle": { + "type": ["string", "null"], + "x_pdpp_role": "actor" + }, + "service": { + "type": ["string", "null"] + }, + "is_from_me": { + "type": "boolean" + }, + "text": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "date": { + "type": "string", + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "date_read": { + "type": ["string", "null"] + }, + "has_attachments": { + "type": "boolean" + } + }, + "required": ["id", "date"] + }, + "primary_key": ["id"], + "cursor_field": "date", + "consent_time_field": "date", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "query": { + "search": { + "lexical_fields": ["text"], + "semantic_fields": ["text"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "manual_as_of" + }, + { + "name": "participants", + "required": false, + "description": "Group-chat and one-to-one chat membership from chat_handle_join (undocumented, reverse-engineered chat.db schema). Optional: older macOS chat.db schema versions may not expose chat_handle_join, in which case this stream degrades to a labeled SKIP_RESULT instead of failing the run.", + "display": { + "label": "Your iMessage chat participants", + "detail": "Chat and counterparty handle. is_from_me reports whether that handle has ever sent a message in the chat (a participant-level 'has this contact spoken here' marker) — not the per-message sender flag used in the messages stream, and best-effort only: the device owner's own handle is frequently absent from chat_handle_join entirely." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "chat_id": { + "type": "string" + }, + "handle": { + "type": ["string", "null"], + "x_pdpp_role": "actor" + }, + "is_from_me": { + "type": "boolean" + } + }, + "required": ["id", "chat_id"] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "manual_as_of" + }, + { + "name": "attachments", + "required": false, + "description": "Attachment metadata and bytes from ~/Library/Messages, joined via message_attachment_join (undocumented, reverse-engineered chat.db schema). Bytes are read only from inside a trusted attachments root (default ~/Library/Messages/Attachments, override via IMESSAGE_ATTACHMENTS_ROOT) — the raw path recorded in chat.db is canonicalized and verified to resolve inside that root before any read, rejecting `../` traversal, an absolute path outside the root, and a symlink that escapes the root, and uploaded as a BlobRef when the runtime has blob-upload bindings. The local filesystem path itself never leaves the connector process. Optional: older macOS chat.db schema versions may not expose attachment/message_attachment_join, in which case this stream degrades to a labeled SKIP_RESULT instead of failing the run.", + "display": { + "label": "Your iMessage attachments", + "detail": "Filename, MIME type, size, linked message/chat when detectable, and a blob reference when the local file was hydrated." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "message_id": { + "type": ["string", "null"] + }, + "chat_id": { + "type": ["string", "null"] + }, + "filename": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "content_type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "content_sha256": { + "type": ["string", "null"] + }, + "hydration_status": { + "type": "string", + "enum": ["deferred", "hydrated", "failed", "too_large", "missing"] + }, + "hydration_error": { + "type": ["string", "null"] + }, + "blob_ref": { + "type": ["object", "null"], + "properties": { + "blob_id": { + "type": "string" + }, + "mime_type": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "size_bytes": { + "type": "integer" + } + }, + "required": ["blob_id", "mime_type", "sha256", "size_bytes"] + } + }, + "required": ["id", "filename", "content_type", "hydration_status"] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "relationships": [ + { + "name": "message", + "stream": "messages", + "foreign_key": "message_id", + "cardinality": "has_one" + } + ], + "coverage_strategy": "parent_detail_accounting", + "freshness_strategy": "manual_as_of" + } + ], + "reason_display_messages": { + "attachment_tables_missing": "Your Messages database is missing attachment data we need", + "chat_handle_join_table_missing": "Your Messages database is missing chat-participant data we need", + "message_date_unusable": "We skipped a message with a date we couldn't read" + } +} From db68160207618c73cd5ae11d3cf41a8981d2af80 Mon Sep 17 00:00:00 2001 From: Tim Nunamaker Date: Fri, 6 Nov 2026 11:02:14 -0600 Subject: [PATCH 8/8] build: drop the pin script's flags and holds file The pin script is back to the shape it has on main: it moves every Collection Profile pin to the catalog's latest release. The only change is the call into the new installer core, which takes no context. The --latest and --version flags, the holds file and the selection module had no caller beyond this change. The hold reasons named release numbers that go stale, and the script took a nine-field injection object only so tests could drive it. Those tests are removed with it. The pins in this change stay where the re-pin put them. The catalog's latest releases of claude-code, codex, google-takeout and apple-photos drop the coverage_diagnostics stream that the collector requests, so running the script again moves those four pins to a release the collector cannot fully use. Re-pin by hand, with the same check, until a collector that matches the newer releases lands. The comment in scripts/resolve-connectors.js no longer points at the removed holds file. Assisted-by: AI Signed-off-by: Tim Nunamaker --- .../collection-profile-holds.json | 18 -- .../scripts/pin-collection-profiles.ts | 98 ++-------- .../local-collector/scripts/pin-selection.ts | 105 ----------- .../test/pin-collection-profiles.test.ts | 168 ------------------ .../test/pin-selection.test.ts | 80 --------- scripts/resolve-connectors.js | 5 - 6 files changed, 13 insertions(+), 461 deletions(-) delete mode 100644 packages/local-collector/collection-profile-holds.json delete mode 100644 packages/local-collector/scripts/pin-selection.ts delete mode 100644 packages/local-collector/test/pin-collection-profiles.test.ts delete mode 100644 packages/local-collector/test/pin-selection.test.ts diff --git a/packages/local-collector/collection-profile-holds.json b/packages/local-collector/collection-profile-holds.json deleted file mode 100644 index 99c6a4fe2..000000000 --- a/packages/local-collector/collection-profile-holds.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "claude-code": { - "version": "0.3.4", - "reason": "The catalog's latest release (0.3.29 when checked) drops the coverage_diagnostics stream, which this collector requests. 0.3.4 is the oldest release with a signed source declaration." - }, - "codex": { - "version": "0.3.3", - "reason": "The catalog's latest release (0.3.15 when checked) drops the coverage_diagnostics stream, which this collector requests. 0.3.3 is the oldest release with a signed source declaration." - }, - "google-takeout": { - "version": "0.1.3", - "reason": "The catalog's latest release (0.1.27 when checked) drops the coverage_diagnostics stream, which this collector requests. 0.1.3 is the oldest release with a signed source declaration." - }, - "apple-photos": { - "version": "0.1.3", - "reason": "The catalog's latest release (0.1.26 when checked) drops the coverage_diagnostics stream, which this collector requests. 0.1.3 is the oldest release with a signed source declaration." - } -} diff --git a/packages/local-collector/scripts/pin-collection-profiles.ts b/packages/local-collector/scripts/pin-collection-profiles.ts index c001f09f2..c17bfbc67 100644 --- a/packages/local-collector/scripts/pin-collection-profiles.ts +++ b/packages/local-collector/scripts/pin-collection-profiles.ts @@ -25,26 +25,12 @@ * Usage (network required): * * npm run pin:collection-profiles --workspace @pdpp/local-collector - * npm run pin:collection-profiles --workspace @pdpp/local-collector -- --latest - * npm run pin:collection-profiles --workspace @pdpp/local-collector -- --version claude-code=0.3.4 * - * With no flag the current pins stay where they are: the files are rewritten - * from the same releases, and a connector with no pin yet takes the catalog's - * latest. `--latest` moves every connector without a hold to the latest - * release; review the diff as a connector upgrade. `--version - * =` (repeatable) pins that connector to a named - * catalog release and beats a hold. Any other argument, an unknown connector - * key, or a malformed `--version` is an error. A release the run keeps or - * moves to (named, held, or already pinned) that the catalog does not list - * fails the run before anything is written; only a connector with no pin and - * no request may be left unpinned. - * - * A hold keeps a pin below the latest release, with the reason, in - * `collection-profile-holds.json`. Holds apply under `--latest`; change a hold - * in that file in the same change that moves the pin. + * Re-running it moves every pin to the catalog's latest release. Review the + * diff as a connector upgrade. */ -import { mkdirSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { mkdirSync, readdirSync, rmSync, writeFileSync } from "node:fs"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -54,7 +40,6 @@ import { fetchResolvedArtifact, } from "@opendatalabs/data-connectors-tools/installer-core"; -import { COLLECTION_PROFILE_PINS } from "../src/generated/collection-profile-pins.generated.ts"; import { LOCAL_COLLECTOR_DEFINITIONS } from "../src/generated/collector-definitions.generated.ts"; import { COLLECTION_PROFILE_ENTRYPOINT, @@ -62,11 +47,9 @@ import { type CollectionProfilePin, collectionProfileRepository, } from "../src/managed/collection-profiles.ts"; -import { choosePin, parseHolds, parsePinArguments } from "./pin-selection.ts"; const packageRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const pinsModule = path.join(packageRoot, "src", "generated", "collection-profile-pins.generated.ts"); -const holdsFile = path.join(packageRoot, "collection-profile-holds.json"); const REFERENCE_PROFILE_DIR = path.resolve( packageRoot, "..", @@ -76,15 +59,9 @@ const REFERENCE_PROFILE_DIR = path.resolve( "local-collector-profiles" ); -interface CatalogRelease { - readonly digest: string; - readonly version: string; -} - interface CatalogConnector { readonly connector_key: string; - readonly latest?: CatalogRelease; - readonly versions?: readonly CatalogRelease[]; + readonly latest?: { readonly digest: string; readonly version: string }; } interface ResolvedArtifact { @@ -98,59 +75,21 @@ function connectorKeyFor(connectorId: string): string { return connectorId.replaceAll("_", "-"); } -/** Everything the script touches outside its own logic, so a test can run it offline against a temp tree. */ -export interface PinScriptDeps { - readonly argv: readonly string[]; - readonly definitions: readonly { readonly connector_id: string }[]; - readonly currentPins: readonly { readonly connectorKey: string; readonly version: string }[]; - readonly holdsFile: string; - readonly pinsModule: string; - readonly referenceProfileDir: string; - readonly fetchCatalog: () => Promise<{ catalog: { connectors: readonly CatalogConnector[] } }>; - readonly fetchResolvedArtifact: ( - context: null, - artifact: Record, - options: Record - ) => Promise; - readonly log: (line: string) => void; -} - -export async function runPinScript(deps: PinScriptDeps): Promise { - const { pinsModule, referenceProfileDir: REFERENCE_PROFILE_DIR } = deps; - const knownKeys = new Set(deps.definitions.map((definition) => connectorKeyFor(definition.connector_id))); - const args = parsePinArguments(deps.argv, knownKeys); - const holds = parseHolds(JSON.parse(readFileSync(deps.holdsFile, "utf8"))); - const currentVersions = new Map(deps.currentPins.map((pin) => [pin.connectorKey, pin.version])); - const catalogResult = await deps.fetchCatalog(); +async function main(): Promise { + const catalogResult = (await fetchCatalog({})) as { catalog: { connectors: readonly CatalogConnector[] } }; const catalog = new Map(catalogResult.catalog.connectors.map((entry) => [entry.connector_key, entry])); const pins: CollectionProfilePin[] = []; const profiles = new Map(); - for (const definition of deps.definitions) { + for (const definition of LOCAL_COLLECTOR_DEFINITIONS) { const connectorKey = connectorKeyFor(definition.connector_id); - const listed = catalog.get(connectorKey); - const choice = choosePin({ - connectorKey, - args, - hold: holds.get(connectorKey), - currentVersion: currentVersions.get(connectorKey), - }); - const latest = - choice.kind === "latest" ? listed?.latest : listed?.versions?.find((release) => release.version === choice.version); + const latest = catalog.get(connectorKey)?.latest; if (!latest) { - // A release this run was asked to keep or move to must exist. Only a - // connector with no pin and no request may be left out, and then the - // collector refuses it by name. - if (choice.kind === "version" || currentVersions.has(connectorKey)) { - throw new Error( - `cannot pin ${connectorKey}: the signed catalog lists no ${choice.kind === "version" ? choice.version : "latest release"} (${choice.why}). Nothing was written.` - ); - } - deps.log(`unpinned ${definition.connector_id}: the signed catalog lists no ${connectorKey}`); + process.stdout.write(`unpinned ${definition.connector_id}: the signed catalog lists no ${connectorKey}\n`); continue; } const repository = collectionProfileRepository({ connectorKey }); - const resolved = await deps.fetchResolvedArtifact( + const resolved = (await fetchResolvedArtifact( null, { artifactKind: "pdpp-collection-profile", @@ -166,7 +105,7 @@ export async function runPinScript(deps: PinScriptDeps): Promise { ociCertificateIdentityResolver: ({ registry, repository: requested }: { registry: string; repository: string }) => registry === "ghcr.io" && requested === repository ? DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY : null, } - ); + )) as ResolvedArtifact; pins.push({ connectorId: definition.connector_id, connectorKey, @@ -176,7 +115,7 @@ export async function runPinScript(deps: PinScriptDeps): Promise { entrypointSha256: resolved.checksums.entrypoint, }); profiles.set(connectorKey, resolved.manifestBuffer); - deps.log(`pinned ${definition.connector_id} ${latest.version} ${resolved.oci.digest} (${choice.why})`); + process.stdout.write(`pinned ${definition.connector_id} ${latest.version} ${resolved.oci.digest}\n`); } writeFileSync(pinsModule, renderPinsModule(pins)); @@ -219,16 +158,5 @@ ${body} } if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { - await runPinScript({ - argv: process.argv.slice(2), - currentPins: COLLECTION_PROFILE_PINS, - definitions: LOCAL_COLLECTOR_DEFINITIONS, - fetchCatalog: async () => (await fetchCatalog({})) as { catalog: { connectors: readonly CatalogConnector[] } }, - fetchResolvedArtifact: async (context, artifact, options) => - (await fetchResolvedArtifact(context, artifact, options)) as ResolvedArtifact, - holdsFile, - log: (line) => process.stdout.write(`${line}\n`), - pinsModule, - referenceProfileDir: REFERENCE_PROFILE_DIR, - }); + await main(); } diff --git a/packages/local-collector/scripts/pin-selection.ts b/packages/local-collector/scripts/pin-selection.ts deleted file mode 100644 index c8f4795d6..000000000 --- a/packages/local-collector/scripts/pin-selection.ts +++ /dev/null @@ -1,105 +0,0 @@ -// Copyright The PDP-Connect Contributors -// SPDX-License-Identifier: Apache-2.0 - -/** - * Which catalog release `pin-collection-profiles.ts` pins for each connector. - * - * Kept apart from the script so the rules run without the network: a re-run - * must never move a pin the owner did not ask it to move. - */ - -/** A pin that stays where it is, with the reason, read from `collection-profile-holds.json`. */ -export interface CollectionProfileHold { - readonly version: string; - readonly reason: string; -} - -export interface PinArguments { - /** `--latest`: move connectors that have no hold to the catalog's latest release. */ - readonly latest: boolean; - /** `--version =`: a named release, which beats a hold. */ - readonly versions: ReadonlyMap; -} - -export type PinChoice = - | { readonly kind: "version"; readonly version: string; readonly why: string } - | { readonly kind: "latest"; readonly why: string }; - -/** - * Parse `--latest` and `--version =` (repeatable). - * Anything else is an error, so a mistyped flag cannot pass for a no-op. - */ -export function parsePinArguments(args: readonly string[], knownKeys: ReadonlySet): PinArguments { - let latest = false; - const versions = new Map(); - for (let index = 0; index < args.length; index += 1) { - const arg = args[index] ?? ""; - if (arg === "--latest") { - latest = true; - } else if (arg === "--version") { - const spec = args[index + 1] ?? ""; - index += 1; - const parts = spec.split("="); - const [key, version] = parts; - if (parts.length !== 2 || !key || !version) { - throw new Error(`--version takes exactly =, got "${spec}"`); - } - if (!knownKeys.has(key)) { - throw new Error(`--version names unknown connector "${key}"; known: ${[...knownKeys].sort().join(", ")}`); - } - if (versions.has(key)) { - throw new Error(`--version names ${key} twice`); - } - versions.set(key, version); - } else { - throw new Error(`unknown argument "${arg}"; accepted: --latest, --version =`); - } - } - return { latest, versions }; -} - -/** - * Choose a release for one connector, in this order: - * 1. `--version` names it; - * 2. a hold keeps it, even under `--latest`; - * 3. `--latest` takes the catalog's latest; - * 4. the current pin stays; - * 5. a connector with no pin yet takes the latest. - */ -export function choosePin(input: { - readonly connectorKey: string; - readonly args: PinArguments; - readonly hold: CollectionProfileHold | undefined; - readonly currentVersion: string | undefined; -}): PinChoice { - const requested = input.args.versions.get(input.connectorKey); - if (requested) { - return { kind: "version", version: requested, why: "named by --version" }; - } - if (input.hold) { - return { kind: "version", version: input.hold.version, why: `held: ${input.hold.reason}` }; - } - if (input.args.latest) { - return { kind: "latest", why: "--latest" }; - } - if (input.currentVersion) { - return { kind: "version", version: input.currentVersion, why: "current pin kept" }; - } - return { kind: "latest", why: "no pin yet" }; -} - -/** Validate the holds file's shape; a malformed hold must not be read as no hold. */ -export function parseHolds(raw: unknown): ReadonlyMap { - if (typeof raw !== "object" || raw === null || Array.isArray(raw)) { - throw new Error("collection-profile-holds.json must be an object keyed by connector key"); - } - const holds = new Map(); - for (const [key, value] of Object.entries(raw)) { - const hold = value as Partial | null; - if (!hold || typeof hold.version !== "string" || !hold.version || typeof hold.reason !== "string" || !hold.reason) { - throw new Error(`hold for "${key}" needs a non-empty version and reason`); - } - holds.set(key, { version: hold.version, reason: hold.reason }); - } - return holds; -} diff --git a/packages/local-collector/test/pin-collection-profiles.test.ts b/packages/local-collector/test/pin-collection-profiles.test.ts deleted file mode 100644 index 8e6727131..000000000 --- a/packages/local-collector/test/pin-collection-profiles.test.ts +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright The PDP-Connect Contributors -// SPDX-License-Identifier: Apache-2.0 - -/** - * Runs `pin-collection-profiles.ts` end to end against a temp tree and a fake - * catalog. A release the run was asked to keep or move to, and the catalog no - * longer lists, must fail the run with every existing output untouched; it - * must not drop the connector from the pins. - */ - -import assert from "node:assert/strict"; -import { mkdirSync, mkdtempSync, readdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import path from "node:path"; -import { test } from "node:test"; - -import { type PinScriptDeps, runPinScript } from "../scripts/pin-collection-profiles.ts"; - -const DEFINITIONS = [{ connector_id: "codex" }, { connector_id: "imessage" }]; - -interface Tree { - readonly deps: PinScriptDeps; - readonly logs: string[]; - readonly snapshot: () => Record; - readonly cleanup: () => void; -} - -function release(version: string) { - return { digest: `sha256:${version}`, version }; -} - -function tree(options: { - readonly argv: readonly string[]; - readonly catalog: Record; - readonly currentPins: readonly { connectorKey: string; version: string }[]; -}): Tree { - const root = mkdtempSync(path.join(tmpdir(), "pin-script-")); - const pinsModule = path.join(root, "pins.generated.ts"); - const referenceProfileDir = path.join(root, "profiles"); - const holdsFile = path.join(root, "holds.json"); - mkdirSync(referenceProfileDir); - writeFileSync(holdsFile, "{}\n"); - writeFileSync(pinsModule, "OLD PINS MODULE\n"); - for (const { connectorKey, version } of options.currentPins) { - writeFileSync(path.join(referenceProfileDir, `${connectorKey}.json`), `{"old":"${version}"}\n`); - writeFileSync(path.join(referenceProfileDir, `${connectorKey}.pin.json`), `{"old_pin":"${version}"}\n`); - } - const logs: string[] = []; - const deps: PinScriptDeps = { - argv: options.argv, - currentPins: options.currentPins, - definitions: DEFINITIONS, - fetchCatalog: async () => ({ - catalog: { - connectors: Object.entries(options.catalog).map(([connector_key, versions]) => ({ - connector_key, - latest: release(versions.at(-1) ?? ""), - versions: versions.map(release), - })), - }, - }), - fetchResolvedArtifact: async (_context, artifact) => ({ - checksums: { entrypoint: "entry", manifest: "manifest" }, - manifestBuffer: Buffer.from(`{"profile":"${String(artifact.connectorKey)} ${String(artifact.version)}"}\n`), - oci: { digest: `sha256:${String(artifact.version)}` }, - }), - holdsFile, - log: (line) => logs.push(line), - pinsModule, - referenceProfileDir, - }; - const snapshot = () => { - const files: Record = { "pins.generated.ts": readFileSync(pinsModule, "utf8") }; - for (const name of readdirSync(referenceProfileDir).sort()) { - files[`profiles/${name}`] = readFileSync(path.join(referenceProfileDir, name), "utf8"); - } - return files; - }; - return { cleanup: () => rmSync(root, { force: true, recursive: true }), deps, logs, snapshot }; -} - -const BOTH_PINNED = [ - { connectorKey: "codex", version: "0.3.3" }, - { connectorKey: "imessage", version: "0.1.25" }, -]; - -async function assertRefusedAndUntouched(options: Parameters[0], message: RegExp) { - const t = tree(options); - try { - const before = t.snapshot(); - await assert.rejects(runPinScript(t.deps), message); - assert.deepEqual(t.snapshot(), before, "a refused run leaves every pin output as it was"); - } finally { - t.cleanup(); - } -} - -test("--version naming a release the catalog does not list fails and keeps every output", async () => { - await assertRefusedAndUntouched( - { - argv: ["--version", "codex=99.99.99"], - catalog: { codex: ["0.3.3", "0.3.5"], imessage: ["0.1.25"] }, - currentPins: BOTH_PINNED, - }, - /cannot pin codex.*99\.99\.99/ - ); -}); - -test("a current pin whose release left the catalog fails and keeps every output", async () => { - await assertRefusedAndUntouched( - { - argv: [], - catalog: { codex: ["0.3.5"], imessage: ["0.1.25"] }, - currentPins: BOTH_PINNED, - }, - /cannot pin codex.*0\.3\.3/ - ); -}); - -test("--latest on a pinned connector the catalog no longer lists fails and keeps every output", async () => { - await assertRefusedAndUntouched( - { - argv: ["--latest"], - catalog: { imessage: ["0.1.25", "0.1.26"] }, - currentPins: BOTH_PINNED, - }, - /cannot pin codex/ - ); -}); - -test("a connector with no pin and no request that the catalog does not list is left out, and the rest is written", async () => { - const t = tree({ - argv: [], - catalog: { imessage: ["0.1.25"] }, - currentPins: [{ connectorKey: "imessage", version: "0.1.25" }], - }); - try { - await runPinScript(t.deps); - assert.ok(t.logs.some((line) => line.startsWith("unpinned codex"))); - const files = t.snapshot(); - assert.deepEqual(Object.keys(files), [ - "pins.generated.ts", - "profiles/imessage.json", - "profiles/imessage.pin.json", - ]); - assert.match(files["pins.generated.ts"] ?? "", /"connectorKey": "imessage"/); - assert.doesNotMatch(files["pins.generated.ts"] ?? "", /"connectorKey": "codex"/); - } finally { - t.cleanup(); - } -}); - -test("a bare run rewrites both connectors from the pinned releases", async () => { - const t = tree({ - argv: [], - catalog: { codex: ["0.3.3", "0.3.5"], imessage: ["0.1.25", "0.1.26"] }, - currentPins: BOTH_PINNED, - }); - try { - await runPinScript(t.deps); - const files = t.snapshot(); - assert.match(files["pins.generated.ts"] ?? "", /"version": "0\.3\.3"/); - assert.match(files["pins.generated.ts"] ?? "", /"version": "0\.1\.25"/); - assert.equal(files["profiles/codex.json"], '{"profile":"codex 0.3.3"}\n'); - } finally { - t.cleanup(); - } -}); diff --git a/packages/local-collector/test/pin-selection.test.ts b/packages/local-collector/test/pin-selection.test.ts deleted file mode 100644 index bfd6d83b1..000000000 --- a/packages/local-collector/test/pin-selection.test.ts +++ /dev/null @@ -1,80 +0,0 @@ -// Copyright The PDP-Connect Contributors -// SPDX-License-Identifier: Apache-2.0 - -/** - * `pin-collection-profiles.ts` decides which release each connector pins. A - * re-run must not move a pin nobody asked it to move, and a mistyped argument - * must fail instead of passing for a no-op. - */ - -import assert from "node:assert/strict"; -import { readFileSync } from "node:fs"; -import { test } from "node:test"; - -import { COLLECTION_PROFILE_PINS } from "../src/generated/collection-profile-pins.generated.ts"; -import { choosePin, parseHolds, parsePinArguments } from "../scripts/pin-selection.ts"; - -const KEYS = new Set(["claude-code", "codex", "imessage"]); -const hold = { version: "0.3.4", reason: "latest drops a stream" }; - -test("with no flag a connector keeps its current pin, even one below the latest release", () => { - const args = parsePinArguments([], KEYS); - assert.deepEqual(choosePin({ connectorKey: "codex", args, hold: undefined, currentVersion: "0.3.3" }), { - kind: "version", - version: "0.3.3", - why: "current pin kept", - }); -}); - -test("with no flag a connector that has no pin yet takes the latest release", () => { - const args = parsePinArguments([], KEYS); - assert.equal(choosePin({ connectorKey: "codex", args, hold: undefined, currentVersion: undefined }).kind, "latest"); -}); - -test("--latest moves an unheld connector and leaves a held one where its hold puts it", () => { - const args = parsePinArguments(["--latest"], KEYS); - assert.equal(choosePin({ connectorKey: "imessage", args, hold: undefined, currentVersion: "0.1.3" }).kind, "latest"); - const held = choosePin({ connectorKey: "claude-code", args, hold, currentVersion: "0.3.4" }); - assert.deepEqual(held, { kind: "version", version: "0.3.4", why: "held: latest drops a stream" }); -}); - -test("--version names a release and beats a hold", () => { - const args = parsePinArguments(["--version", "claude-code=0.3.9"], KEYS); - const choice = choosePin({ connectorKey: "claude-code", args, hold, currentVersion: "0.3.4" }); - assert.equal(choice.kind === "version" && choice.version, "0.3.9"); -}); - -test("an unknown connector key is an error", () => { - assert.throws(() => parsePinArguments(["--version", "claude_code=0.3.4"], KEYS), /unknown connector "claude_code"/); -}); - -test("a malformed --version is an error, including a spec with more than one =", () => { - for (const spec of ["claude-code", "claude-code=", "=0.3.4", "claude-code=0.3.4=0.3.5", ""]) { - assert.throws(() => parsePinArguments(["--version", spec], KEYS), /exactly =/, spec); - } - assert.throws(() => parsePinArguments(["--version"], KEYS), /exactly/); -}); - -test("an unknown argument and a repeated key are errors", () => { - assert.throws(() => parsePinArguments(["--lastest"], KEYS), /unknown argument "--lastest"/); - assert.throws( - () => parsePinArguments(["--version", "codex=0.3.3", "--version", "codex=0.3.5"], KEYS), - /names codex twice/ - ); -}); - -test("a hold needs a version and a reason", () => { - assert.throws(() => parseHolds({ codex: { version: "0.3.3" } }), /needs a non-empty version and reason/); - assert.throws(() => parseHolds([]), /must be an object/); -}); - -test("every hold in collection-profile-holds.json names the version this collector pins", () => { - const holds = parseHolds( - JSON.parse(readFileSync(new URL("../collection-profile-holds.json", import.meta.url), "utf8")) - ); - assert.ok(holds.size > 0); - const pinned = new Map(COLLECTION_PROFILE_PINS.map((pin) => [pin.connectorKey, pin.version])); - for (const [key, entry] of holds) { - assert.equal(pinned.get(key), entry.version, `${key} is held at ${entry.version} but pinned at ${pinned.get(key)}`); - } -}); diff --git a/scripts/resolve-connectors.js b/scripts/resolve-connectors.js index f2e6cd168..ce259a52d 100644 --- a/scripts/resolve-connectors.js +++ b/scripts/resolve-connectors.js @@ -101,11 +101,6 @@ export const LOCKED_ARTIFACT_SOURCE = Object.freeze({ // would need a registry fetch per entry on every install and `--check`, or a // new lock field. Delete the id here when #634 re-pins github-pdpp to a release // that carries a declaration. -// -// This is not the profile hold in -// packages/local-collector/collection-profile-holds.json. A hold keeps a -// collector pin below the catalog's latest release for a stated reason, and -// that pin installs through the current core. export const PRE_DECLARATION_OCI_CONNECTORS = Object.freeze( new Set(["github-pdpp"]) )