diff --git a/connectors/connector-dependencies.json b/connectors/connector-dependencies.json index fe988352c..b26f1fd3c 100644 --- a/connectors/connector-dependencies.json +++ b/connectors/connector-dependencies.json @@ -15,9 +15,9 @@ "wholefoods-playwright": "1.0.0", "youtube-playwright": "1.0.0", "claude-export-playwright": "2.0.1", - "apple-health-pdpp": "0.1.0", - "chatgpt-pdpp": "0.1.0", + "apple-health-pdpp": "0.1.17", + "chatgpt-pdpp": "0.2.5", "github-pdpp": "0.5.0", - "ynab-pdpp": "0.3.0" + "ynab-pdpp": "0.3.26" } } diff --git a/connectors/lock.json b/connectors/lock.json index c3d9584bf..becd52bc5 100644 --- a/connectors/lock.json +++ b/connectors/lock.json @@ -1,7 +1,7 @@ { "lockVersion": "2.0", "dependencyFile": "connectors/connector-dependencies.json", - "generatedAt": "2026-09-15T19:43:18.976Z", + "generatedAt": "2026-11-05T20:25:11.519Z", "sourceRepo": "https://github.com/PDP-Connect/data-connectors", "sourceTag": "mixed", "sourceCommit": "mixed", @@ -25,10 +25,10 @@ "wholefoods-playwright": "1.0.0", "youtube-playwright": "1.0.0", "claude-export-playwright": "2.0.1", - "apple-health-pdpp": "0.1.0", - "chatgpt-pdpp": "0.1.0", + "apple-health-pdpp": "0.1.17", + "chatgpt-pdpp": "0.2.5", "github-pdpp": "0.5.0", - "ynab-pdpp": "0.3.0" + "ynab-pdpp": "0.3.26" }, "connectors": [ { @@ -38,20 +38,20 @@ "manifestPath": "profile/collection-profile.json", "entrypointPath": "dist/collection-profile.mjs", "provenancePath": "provenance.json", - "version": "0.1.0", - "resolvedFrom": "0.1.0", + "version": "0.1.17", + "resolvedFrom": "0.1.17", "oci": { "registry": "ghcr.io", "repository": "pdp-connect/connector/apple-health", - "digest": "sha256:1e497ac3d5cbc868f27ddac42e792e9c2da929438ed7e38d24aa8ac84777ad8e", - "configDigest": "sha256:e1270b82e9ea0a24fc9d66c4f9cbfb775c74355c88928a0d3e51c95482b179fd" + "digest": "sha256:1555695184a65da4c8630ce47f18f7747cbdbfd6db14b09b48fe3ab1672299ec", + "configDigest": "sha256:e4c88e4a2688c4e9f86575f3f9567745a36e3839f1e60ca9402c8fa192a2cf6f" }, "company": "apple", "name": "Apple Health", "description": "Collects Apple Health export data through the PDPP Collection Profile protocol.", - "manifestSha256": "sha256:91d7e8af7ece79089c2e1b857bb15770fdccdcc5e80e767b3f6c7d072a5bc600", - "entrypointSha256": "sha256:7513de5db4e94deb75d82c7fdaae08add1689732d66952b156d4ecf21419cea7", - "provenanceSha256": "sha256:682be3f0a126e8ab148f43dd5ff8fa13cdc490ee78c0cd519d33a32c8b34de3b" + "manifestSha256": "sha256:83e71487ef1905966f00bd286f8139d5232d98830f722cfda4fe9770814dee9f", + "entrypointSha256": "sha256:6227d0fda6985887c2472d62d2bfd4eb7478e5257986d9287a3a2bee9208de8d", + "provenanceSha256": "sha256:d3e097df9d79c2aa522012b2c55460a98918ab2b78b73d582b6fb681dfe1cb46" }, { "connectorId": "chatgpt-pdpp", @@ -60,20 +60,20 @@ "manifestPath": "profile/collection-profile.json", "entrypointPath": "dist/collection-profile.mjs", "provenancePath": "provenance.json", - "version": "0.1.0", - "resolvedFrom": "0.1.0", + "version": "0.2.5", + "resolvedFrom": "0.2.5", "oci": { "registry": "ghcr.io", "repository": "pdp-connect/connector/chatgpt", - "digest": "sha256:811daaf8f2a9a346f1ff5109c118ea404212c8e00790cc5d448e89ad9fa9c4a4", - "configDigest": "sha256:8f1bc246633ae908636d841f8cf737d17daf0144207a375b694d8eea9ee1e30d" + "digest": "sha256:727b4469ad71ba3cd9f71d0e72191214c5439c189acf02c1f6949a41378c1ea5", + "configDigest": "sha256:aa81039f0ef1bd8997bebbdd49e3fd675e015a5d22aa346beef9d2436a05aeec" }, "company": "openai", "name": "ChatGPT (PDPP Collection Profile)", "description": "Collects ChatGPT conversations, messages, memories, custom GPTs, custom instructions, and shared conversations through the PDPP Collection Profile protocol.", - "manifestSha256": "sha256:eb491a782ba61925340c5334d3b35a6d0111757f63ccda72bc466f4f84b8f417", - "entrypointSha256": "sha256:e50f93dee4d48925f623f3be04acaed87f77783f1255579c0db7618fabe1a963", - "provenanceSha256": "sha256:d9695809c157e431a725aea21d1f65b48b652c5f9cc3b2b18ac925d122e2dbc5" + "manifestSha256": "sha256:4901f84e8b07874d47405fcb5618ed2867ebcbb0050193d18625045889f3a92c", + "entrypointSha256": "sha256:0b3dcbeb58422874da31a621ed36a78e0f690431d8a63970b1082089bba1b8ac", + "provenanceSha256": "sha256:2a643baea69b48aa7c40ea665250df4d0841c392703343c051b96a37f42430ec" }, { "connectorId": "chatgpt-playwright", @@ -390,20 +390,20 @@ "manifestPath": "profile/collection-profile.json", "entrypointPath": "dist/collection-profile.mjs", "provenancePath": "provenance.json", - "version": "0.3.0", - "resolvedFrom": "0.3.0", + "version": "0.3.26", + "resolvedFrom": "0.3.26", "oci": { "registry": "ghcr.io", "repository": "pdp-connect/connector/ynab", - "digest": "sha256:9da6a382500368ef28a8c9f59351d1f3ab6871297bf27418c7c63e8d9e241fad", - "configDigest": "sha256:20cc66f94d44748af7cc0c7eac1a20fa10e795b4a89f1b376e1827be947b8388" + "digest": "sha256:863fe543e7774252af4bd64a815fabada12a9a45dffaaebfca175411ffa4478d", + "configDigest": "sha256:32df5499e9e23a5639767fe7f04931808e42d78238829bac7af0df37cafa0738" }, "company": "ynab", "name": "YNAB", "description": "Collects YNAB budgets, accounts, categories, transactions, and other financial data through the PDPP Collection Profile protocol.", - "manifestSha256": "sha256:ce2e9b6badc9ff6b0573d36a1dd3875d24eefd69da4184179354613f71ffde9f", - "entrypointSha256": "sha256:9f90880f8dca821b8929305ddcf1a7948a8fbea1df66910e08298774cf3cb4b8", - "provenanceSha256": "sha256:0cab8f965bfaa5e9e0df643e424b2cd29e77b44324c2d738f2d8590d8583dc1d" + "manifestSha256": "sha256:c23d2b3a6c68beac5a8d082ed2843833f755794faaf4e629229230d447bec328", + "entrypointSha256": "sha256:58ba4e7c97899fb25918e8e90e4ca49fb59c9c0aad4a199b081b8b3cf862ce8d", + "provenanceSha256": "sha256:d0d7dd10baa9854d57f3cf076aed2d035d20e438f11158557f554888293f78be" }, { "connectorId": "youtube-playwright", diff --git a/package-lock.json b/package-lock.json index e109d1536..f995ed823 100644 --- a/package-lock.json +++ b/package-lock.json @@ -42,7 +42,7 @@ }, "devDependencies": { "@eslint/js": "^10.0.1", - "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#6c71697a1523926fe8d475cd243a8219ea21aa5b", + "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#708a6cc17d6821fad222a652084f2bc270c40052", "@semantic-release/exec": "^7.1.0", "@stryker-mutator/core": "10.0.0", "@stryker-mutator/vitest-runner": "10.0.0", @@ -56,6 +56,7 @@ "@vitejs/plugin-react": "^6.1.2", "autoprefixer": "^10.6.1", "conventional-changelog-conventionalcommits": "^10.4.1", + "data-connectors-legacy-installer": "github:PDP-Connect/data-connectors#e1e497f3bca842c0489c472c1ca286630eef5078", "eslint": "^10.12.0", "eslint-config-prettier": "^10.1.8", "eslint-plugin-react-hooks": "^7.1.1", @@ -3531,9 +3532,8 @@ }, "node_modules/@opendatalabs/data-connectors-tools": { "version": "0.1.0", - "resolved": "git+ssh://git@github.com/PDP-Connect/data-connectors.git#6c71697a1523926fe8d475cd243a8219ea21aa5b", - "integrity": "sha512-JXPJDjfKhFeAJEn8TZLVzhNrangqTYWOFIEff6VmNj5KuKP2md76sM71YVnCx4u8uRFWcHmTWvgJEilkikBLvA==", - "dev": true, + "resolved": "git+ssh://git@github.com/PDP-Connect/data-connectors.git#708a6cc17d6821fad222a652084f2bc270c40052", + "integrity": "sha512-0J0LBwzLI3FcT/7bP1zwgzRnz5c41aVTEQEE9dhf5/EjP+DMx+gPb3GGDT9faO4MvjWQe0xc8rLSWEzeOi2gCQ==", "license": "Apache-2.0", "workspaces": [ "packages/connector-installer-core", @@ -3542,6 +3542,7 @@ "dependencies": { "@sigstore/bundle": "^5.0.0", "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "sigstore": "^5.0.0", "tar": "^7.5.22" }, @@ -3553,7 +3554,6 @@ "version": "8.20.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", - "dev": true, "license": "MIT", "dependencies": { "fast-deep-equal": "^3.1.3", @@ -3570,7 +3570,6 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", - "dev": true, "license": "MIT" }, "node_modules/@opendatalabs/remote-surface": { @@ -10450,6 +10449,51 @@ "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", "license": "MIT" }, + "node_modules/data-connectors-legacy-installer": { + "name": "@opendatalabs/data-connectors-tools", + "version": "0.1.0", + "resolved": "git+ssh://git@github.com/PDP-Connect/data-connectors.git#e1e497f3bca842c0489c472c1ca286630eef5078", + "integrity": "sha512-FOFkJkRFU173i9t1QFW4F5zcU2MwwM+4A9a6bnGnsuftCqF7p4xfmKxNI2Y0vRMB00V4hTItVjpWd/k8p5QtSQ==", + "dev": true, + "license": "Apache-2.0", + "workspaces": [ + "packages/connector-installer-core", + "packages/polyfill-connectors" + ], + "dependencies": { + "@sigstore/bundle": "^5.0.0", + "ajv": "^8.20.0", + "sigstore": "^5.0.0", + "tar": "^7.5.22" + }, + "bin": { + "connector-installer": "packages/connector-installer-cli/index.mjs" + } + }, + "node_modules/data-connectors-legacy-installer/node_modules/ajv": { + "version": "8.20.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", + "integrity": "sha512-Thbli+OlOj+iMPYFBVBfJ3OmCAnaSyNn4M1vz9T6Gka5Jt9ba/HIR56joy65tY6kx/FCF5VXNB819Y7/GUrBGA==", + "dev": true, + "license": "MIT", + "dependencies": { + "fast-deep-equal": "^3.1.3", + "fast-uri": "^3.0.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2" + }, + "funding": { + "type": "github", + "url": "https://github.com/sponsors/epoberezkin" + } + }, + "node_modules/data-connectors-legacy-installer/node_modules/json-schema-traverse": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==", + "dev": true, + "license": "MIT" + }, "node_modules/data-urls": { "version": "8.0.0", "resolved": "https://registry.npmjs.org/data-urls/-/data-urls-8.0.0.tgz", @@ -23716,6 +23760,7 @@ "dependencies": { "@sigstore/bundle": "^5.0.0", "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "sigstore": "^5.0.0", "tar": "^7.5.22", "zod": "^4.6.7" @@ -23724,7 +23769,7 @@ "pdpp-local-collector": "dist/local-collector/bin/pdpp-local-collector.js" }, "devDependencies": { - "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#6c71697a1523926fe8d475cd243a8219ea21aa5b", + "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#708a6cc17d6821fad222a652084f2bc270c40052", "@pdpp/collector-runtime": "0.0.1", "@pdpp/connector-protocol": "0.0.1", "playwright": "^1.64.1", @@ -23769,7 +23814,7 @@ "dependencies": { "@fastify/formbody": "^9.0.1", "@huggingface/transformers": "^4.3.1", - "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#06ed284e1b7ee92b2ebab6bc8ef25f0ae392f84a", + "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#708a6cc17d6821fad222a652084f2bc270c40052", "@pdpp/collector-runtime": "*", "@pdpp/connector-protocol": "*", "@pdpp/display": "*", @@ -23869,26 +23914,6 @@ "node": "^22.18.0 || >=24.11.0" } }, - "reference-implementation/node_modules/@opendatalabs/data-connectors-tools": { - "version": "0.1.0", - "resolved": "git+ssh://git@github.com/PDP-Connect/data-connectors.git#06ed284e1b7ee92b2ebab6bc8ef25f0ae392f84a", - "integrity": "sha512-lTcdI60BuD+pEAXcvmEb9iDATPrXWxGve/Ah08MmOXZ4emp8KaJgmKWZdtg5qcJmOrA8ZGX5kVNfcFeO2B62tg==", - "license": "Apache-2.0", - "workspaces": [ - "packages/connector-installer-core", - "packages/polyfill-connectors" - ], - "dependencies": { - "@sigstore/bundle": "^5.0.0", - "ajv": "^8.20.0", - "ajv-formats": "^3.0.1", - "sigstore": "^5.0.0", - "tar": "^7.5.22" - }, - "bin": { - "connector-installer": "packages/connector-installer-cli/index.mjs" - } - }, "reference-implementation/node_modules/ajv": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", diff --git a/package.json b/package.json index 0d4373aaa..b57e4a5ec 100644 --- a/package.json +++ b/package.json @@ -97,7 +97,7 @@ }, "devDependencies": { "@eslint/js": "^10.0.1", - "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#6c71697a1523926fe8d475cd243a8219ea21aa5b", + "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#708a6cc17d6821fad222a652084f2bc270c40052", "@semantic-release/exec": "^7.1.0", "@stryker-mutator/core": "10.0.0", "@stryker-mutator/vitest-runner": "10.0.0", @@ -111,6 +111,7 @@ "@vitejs/plugin-react": "^6.1.2", "autoprefixer": "^10.6.1", "conventional-changelog-conventionalcommits": "^10.4.1", + "data-connectors-legacy-installer": "github:PDP-Connect/data-connectors#e1e497f3bca842c0489c472c1ca286630eef5078", "eslint": "^10.12.0", "eslint-config-prettier": "^10.1.8", "eslint-plugin-react-hooks": "^7.1.1", diff --git a/packages/local-collector/package.json b/packages/local-collector/package.json index 32de4f9d4..f01d7c3f1 100644 --- a/packages/local-collector/package.json +++ b/packages/local-collector/package.json @@ -55,12 +55,13 @@ "dependencies": { "@sigstore/bundle": "^5.0.0", "ajv": "^8.20.0", + "ajv-formats": "^3.0.1", "sigstore": "^5.0.0", "tar": "^7.5.22", "zod": "^4.6.7" }, "devDependencies": { - "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#6c71697a1523926fe8d475cd243a8219ea21aa5b", + "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#708a6cc17d6821fad222a652084f2bc270c40052", "@pdpp/collector-runtime": "0.0.1", "@pdpp/connector-protocol": "0.0.1", "playwright": "^1.64.1", diff --git a/packages/local-collector/scripts/pin-collection-profiles.ts b/packages/local-collector/scripts/pin-collection-profiles.ts index 8bf5cdd05..c17bfbc67 100644 --- a/packages/local-collector/scripts/pin-collection-profiles.ts +++ b/packages/local-collector/scripts/pin-collection-profiles.ts @@ -90,7 +90,7 @@ async function main(): Promise { } const repository = collectionProfileRepository({ connectorKey }); const resolved = (await fetchResolvedArtifact( - { doc: {}, mode: "locked" }, + null, { artifactKind: "pdpp-collection-profile", connectorId: definition.connector_id, diff --git a/packages/local-collector/src/generated/collection-profile-pins.generated.ts b/packages/local-collector/src/generated/collection-profile-pins.generated.ts index 925445c30..2e5527db2 100644 --- a/packages/local-collector/src/generated/collection-profile-pins.generated.ts +++ b/packages/local-collector/src/generated/collection-profile-pins.generated.ts @@ -14,41 +14,41 @@ export const COLLECTION_PROFILE_PINS: readonly CollectionProfilePin[] = Object.f { "connectorId": "claude_code", "connectorKey": "claude-code", - "version": "0.3.0", - "digest": "sha256:5b8866392bb11f24cbc3827e37e96faae9b30c8863d7d82bf9cd21aa2e636c82", - "profileSha256": "sha256:b6398b12d41043706ac4701db92f577ed9964d86b6e4776d471eca65587ae79f", - "entrypointSha256": "sha256:2a76a5b518c474a2a342bd2511ad825aec289c1310b2575801a419813bb9bea3" + "version": "0.3.4", + "digest": "sha256:08a82d8da8ed4eb70551007141383d3afaa9dd9878afb4fa76543ecfd53ee447", + "profileSha256": "sha256:1d9f5132894c8c14e5aa296b55f90c60414ef772b0fa6f6b3c0d9b55525b1381", + "entrypointSha256": "sha256:bbc13637cd863ec774d92c4f43dd926ff5cec39d1abe25d2b881023353f05072" }, { "connectorId": "codex", "connectorKey": "codex", - "version": "0.3.0", - "digest": "sha256:bb75129071d1afcf60bf5e20097b84e1262d3cb2ce3ca5b3c228d23f00495d28", - "profileSha256": "sha256:3287a16b01517439443e8641e97c7f1bfcb358d01641d3b3490331b2ddc107ab", - "entrypointSha256": "sha256:0ae6336d89b88d50ae6a13d11e62d4b844a623d76cfeb7f83dcd29c889764b3b" + "version": "0.3.3", + "digest": "sha256:8cbea0c5b56578ab94cc830bd5c69dce77aa5927106f8401d44a84fd3e39fdab", + "profileSha256": "sha256:f8d1773bb2b856c278748f512aba16fbc993b9a833fbfa9d50c357b7a232776f", + "entrypointSha256": "sha256:20e26674be1ec1ea64ec13bfa6b62717c85a6e474502e70a9f0e8e225000b6e3" }, { "connectorId": "google_takeout", "connectorKey": "google-takeout", - "version": "0.1.0", - "digest": "sha256:38d4bd1312073b167b668cad1701ebc78fea8cf9c19a0edb37a98d11fd660562", - "profileSha256": "sha256:708f149f0efedae0b41cc3f67a065c541ec6ab5305edb322b9f983a9d0913353", - "entrypointSha256": "sha256:710d01455fa0cc146665de92612682b749810b54f8f72d5e235a5538c4f43fda" + "version": "0.1.3", + "digest": "sha256:9d1f4934c42cddd73a00b2792ee7edcc389058c566e462a00cd8210fcb014653", + "profileSha256": "sha256:d645fffba61b0a47a077dc4fe4d04a803481844b3281d333818ab4532d9f01cd", + "entrypointSha256": "sha256:8eafae144013735b8dc44b6ff40d873feed95cbc934d50c08ad68937306f7d99" }, { "connectorId": "imessage", "connectorKey": "imessage", - "version": "0.1.0", - "digest": "sha256:ace3798a8e222e73fa1555a39ba52b245e2566d31d84bb60fdec61c9ae1a9ca6", - "profileSha256": "sha256:654eb8abb9625554c19edcf7e1ef6416fca556929456454ba29277751cde8efb", - "entrypointSha256": "sha256:aee91cec10278cca82acc000d504420f2135eac2e44838d3b4693d08db5f3d84" + "version": "0.1.25", + "digest": "sha256:920246f08102aea9c7958d6b3a0bf89eb07e3b776a921333f6d7ae93f99394b0", + "profileSha256": "sha256:399958b7d89e1263b5f61e6b09de4c7f5d6a81c1084ff887e75a504278820ef4", + "entrypointSha256": "sha256:2875a87d04b77e52b9b880a651814112db1b38cd97bd94dab7948a72666670a1" }, { "connectorId": "apple_photos", "connectorKey": "apple-photos", - "version": "0.1.0", - "digest": "sha256:0952cef101b05b5416d0ea0620ee12c857f493bae547ce71119a243131741415", - "profileSha256": "sha256:1d941bd3fc02831618fcd29b899b091ed2ce51f25535f35f58c5a2aaa37ce596", - "entrypointSha256": "sha256:eccd685d0d5d9b9c675365b0b3073c46028067aac5bceb2719770e5069bf8ca2" + "version": "0.1.3", + "digest": "sha256:3e58cf196d7fc119fa7f462432d069a257f5e62036a7c70c6b542bb7b779f07f", + "profileSha256": "sha256:d290a1d201b9b38955225e4b5cae6054a84097723dac8d600e06c5db4676036a", + "entrypointSha256": "sha256:8a2f1b66106b3574d1b675db9ee754904ac52f00921dfb174ab4ca7ededbaff2" }, ]); diff --git a/packages/local-collector/src/generated/collector-definitions.generated.ts b/packages/local-collector/src/generated/collector-definitions.generated.ts index 0ffc988ce..f49e38cb3 100644 --- a/packages/local-collector/src/generated/collector-definitions.generated.ts +++ b/packages/local-collector/src/generated/collector-definitions.generated.ts @@ -37,7 +37,7 @@ export const LOCAL_COLLECTOR_DEFINITIONS: readonly LocalCollectorDefinition[] = "filesystem": { required: true }, }, protocol_capabilities: [], - streams: ["sessions", "messages", "attachments", "memory_notes", "skills", "slash_commands", "file_history", "cache_inventory", "coverage_diagnostics", "backup_inventory", "config_inventory"], + streams: ["sessions", "messages", "attachments", "memory_notes", "skills", "slash_commands", "usage", "file_history", "cache_inventory", "coverage_diagnostics", "backup_inventory", "config_inventory"], time_scopable_streams: ["sessions", "messages", "attachments"], source_root_scopable_streams: ["sessions", "messages", "attachments"], enforces_source_roots: true, diff --git a/packages/local-collector/src/managed/collection-profiles.ts b/packages/local-collector/src/managed/collection-profiles.ts index 2b4b35109..45400f05d 100644 --- a/packages/local-collector/src/managed/collection-profiles.ts +++ b/packages/local-collector/src/managed/collection-profiles.ts @@ -69,7 +69,7 @@ export interface CollectionProfilePin { /** The part of the installer core this module calls. */ export interface CollectionProfileInstallerCore { readonly DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY: string; - installFromLock(options: Record): Promise; + installFromLock(options: Record & { source: null }): Promise; } export interface EnsureCollectionProfileOptions { @@ -163,8 +163,6 @@ export async function ensureCollectionProfileInstalled( const identityResolver = ({ registry, repository: requested }: { registry: string; repository: string }) => registry === REGISTRY && requested === repository ? core.DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY : null; await core.installFromLock({ - // Tarball entries are not accepted from this lock. - artifactCertificateIdentityResolver: () => null, installRoot: staging, layout: "source", lock: { @@ -183,7 +181,7 @@ export async function ensureCollectionProfileInstalled( ], }, ociCertificateIdentityResolver: identityResolver, - source: { doc: {}, mode: "locked" }, + source: null, }); const staged = join(staging, "collection-profiles", pin.connectorId); diff --git a/packages/local-collector/src/managed/index.ts b/packages/local-collector/src/managed/index.ts index 120fa79b3..21dcfd0b0 100644 --- a/packages/local-collector/src/managed/index.ts +++ b/packages/local-collector/src/managed/index.ts @@ -6,35 +6,28 @@ * * **Status.** The published collector does not use {@link obtainManagedConnectors}. * It installs through `collection-profiles.ts`, which delegates fetch and - * signature verification to data-connectors' installer core. That core now - * has the OCI and Sigstore path the paragraphs below describe as missing. - * The verifier in this directory is kept, with its tests, until an owner - * decides whether to remove it. + * signature verification to data-connectors' installer core, which has the + * OCI and Sigstore path. The verifier in this directory is kept, with its + * tests, until an owner decides whether to remove it. * * One call, {@link obtainManagedConnectors}, walks a pinned lock to a set of * definitions the existing runner seam accepts: resolve a pinned digest, * verify its signature against host policy, install to a content-addressed * cache, and hand the installed artifact to the runner. * - * **Relationship to `@pdpp/connector-manager`.** That package exists on - * data-connectors#97 (`packages/connector-installer-core`, published name - * `@pdpp/connector-manager`) and is where this logic belongs. It does not - * export it yet: its surface today is the incumbent tarball installer — - * `loadConnectorIndex`, `generateLock`, `installFromLock`, `verifyInstalled` - * and friends — and it contains no OCI, ORAS or Cosign code at all. #97's OCI - * work is publisher-side: `scripts/build-connector-oci-artifact.mjs`, - * `scripts/verify-connector-oci-artifact.mjs` and the publish workflow. The - * manager's *grow* half is unwritten. - * - * So this module is written against the artifact contract rather than against - * an API that does not exist, and it is shaped to be deleted: every step is a - * named export with an injected dependency, so when the manager grows - * `resolve`/`verify`/`install`, this becomes a thin adapter over those calls - * and the runner seam does not move again. What must not happen is the runner + * **Relationship to `@pdpp/connector-manager`.** That package is + * data-connectors' `packages/connector-installer-core` (published here as + * `@opendatalabs/data-connectors-tools/installer-core`) and is where this logic + * belongs. It no longer has the tarball installer: it is OCI-only, and its + * `fetchCatalog`, `fetchResolvedArtifact`, `installFromLock` and + * `verifyInstalled` resolve a pinned digest, verify its Sigstore signature, + * check every blob and install a signed Collection Profile. It still does not + * export the {@link ConnectorManager} shape below, so this module stays + * written against the artifact contract and is shaped to be deleted: every step + * is a named export with an injected dependency, so a swap to the core is a + * substitution rather than a rewrite. What must not happen is the runner * growing a dependency on a shape only this repo knows — hence - * {@link ConnectorManager}, which is the interface the manager is expected to - * satisfy, stated here so the eventual swap is a substitution rather than a - * rewrite. + * {@link ConnectorManager}, the interface the core is expected to satisfy. */ import type { ConnectorLock, PinnedConnector } from "./artifact-contract.ts"; diff --git a/packages/local-collector/src/managed/installer-core.d.ts b/packages/local-collector/src/managed/installer-core.d.ts index 92bb8462f..b17b10e78 100644 --- a/packages/local-collector/src/managed/installer-core.d.ts +++ b/packages/local-collector/src/managed/installer-core.d.ts @@ -8,9 +8,9 @@ declare module "@opendatalabs/data-connectors-tools/installer-core" { export const DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY: string; export function fetchCatalog(options: Record): Promise; export function fetchResolvedArtifact( - source: Record, + source: null, entry: Record, options: Record ): Promise; - export function installFromLock(options: Record): Promise; + export function installFromLock(options: Record & { source: null }): Promise; } diff --git a/packages/local-collector/src/managed/registry-client.ts b/packages/local-collector/src/managed/registry-client.ts index 23fbf2529..cbb070ace 100644 --- a/packages/local-collector/src/managed/registry-client.ts +++ b/packages/local-collector/src/managed/registry-client.ts @@ -11,10 +11,10 @@ * * **Why these are injected rather than implemented here.** The decision's * phrase is "registry-neutral", and the shape neutrality actually takes in - * this codebase is already established: the incumbent installer takes - * `artifactCertificateIdentityResolver` as a caller callback rather than + * this codebase is already established: the installer core takes + * `ociCertificateIdentityResolver` as a caller callback rather than * hardcoding a signer, and data-connect supplies it from - * `scripts/resolve-connectors.js`. Signer policy is the consumer's to decide. + * `collection-profiles.ts`. Signer policy is the consumer's to decide. * This module keeps that arrangement and extends it to blob transport, so a * different publisher serving a different registry is a different injected * client, not a patch to this file. diff --git a/packages/local-collector/test/collection-profiles.test.ts b/packages/local-collector/test/collection-profiles.test.ts index 424840d4c..c1a7372c6 100644 --- a/packages/local-collector/test/collection-profiles.test.ts +++ b/packages/local-collector/test/collection-profiles.test.ts @@ -101,6 +101,24 @@ test("each pinned profile is the release it pins and declares its definition's b } }); +test("a collector's default streams include every required stream of its pinned profile", () => { + // A connector emits a stream only when it is requested, and the server counts + // a required stream that is never reported as unknown coverage. A pin that + // adds a required stream must therefore add it to the default stream set. + for (const pin of COLLECTION_PROFILE_PINS) { + const definition = LOCAL_COLLECTOR_DEFINITIONS.find((candidate) => candidate.connector_id === pin.connectorId); + assert.ok(definition, pin.connectorId); + const profile = JSON.parse( + readFileSync(new URL(`${pin.connectorKey}.json`, REFERENCE_PROFILE_DIR), "utf8") + ) as { streams: { name: string; required?: boolean }[] }; + const requested = new Set(definition.streams); + const missing = profile.streams + .filter((stream) => stream.required !== false && !requested.has(stream.name)) + .map((stream) => stream.name); + assert.deepEqual(missing, [], `${pin.connectorId} never requests its required streams`); + } +}); + test("the collector build compiles no connector code", () => { const tsconfig = readFileSync(new URL("../tsconfig.build.json", import.meta.url), "utf8"); assert.doesNotMatch(tsconfig, /polyfill-connectors\/(connectors|src)/); @@ -175,7 +193,7 @@ test("a fresh install lands the pinned files under the digest directory and pass const call = core.calls[0] as { lock: { connectors: { oci: unknown; version: string; artifactKind: string }[] }; ociCertificateIdentityResolver: (input: { registry: string; repository: string }) => string | null; - artifactCertificateIdentityResolver: () => unknown; + source: unknown; }; assert.deepEqual(call.lock.connectors[0]?.oci, { digest: fakePin().digest, @@ -190,7 +208,7 @@ test("a fresh install lands the pinned files under the digest directory and pass ); assert.equal(call.ociCertificateIdentityResolver({ registry: "ghcr.io", repository: "pdp-connect/connector/other" }), null); assert.equal(call.ociCertificateIdentityResolver({ registry: "docker.io", repository: "pdp-connect/connector/fake-connector" }), null); - assert.equal(call.artifactCertificateIdentityResolver(), null); + assert.equal(call.source, null); }); test("a cached release that matches its pin is used without the network", async () => { diff --git a/reference-implementation/package.json b/reference-implementation/package.json index 967752fe8..f27167821 100644 --- a/reference-implementation/package.json +++ b/reference-implementation/package.json @@ -71,7 +71,7 @@ "dependencies": { "@fastify/formbody": "^9.0.1", "@huggingface/transformers": "^4.3.1", - "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#06ed284e1b7ee92b2ebab6bc8ef25f0ae392f84a", + "@opendatalabs/data-connectors-tools": "github:PDP-Connect/data-connectors#708a6cc17d6821fad222a652084f2bc270c40052", "@pdpp/collector-runtime": "*", "@pdpp/connector-protocol": "*", "@pdpp/display": "*", diff --git a/reference-implementation/server/connector-install/index.ts b/reference-implementation/server/connector-install/index.ts index dcf2c284a..48bd559cc 100644 --- a/reference-implementation/server/connector-install/index.ts +++ b/reference-implementation/server/connector-install/index.ts @@ -94,13 +94,13 @@ export interface ConnectorCatalogSnapshot { interface InstallerCore { DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY: string; - fetchCatalog?: (options: Record) => Promise; - fetchResolvedArtifact?: ( - source: Record, + fetchCatalog: (options: Record) => Promise; + fetchResolvedArtifact: ( + source: null, entry: Record, options: Record ) => Promise; - installFromLock: (options: Record) => Promise; + installFromLock: (options: Record & { source: null }) => Promise; } export interface ConnectorInstallStore { @@ -1109,9 +1109,6 @@ export function createConnectorInstallService(options: { return null; } const core = await loadPinnedCore(); - if (!core.fetchResolvedArtifact) { - throw new Error("Pinned connector installer core does not expose fetchResolvedArtifact."); - } const identityResolver = ({ registry, repository }: { registry: string; repository: string }) => { if (registry === "ghcr.io" && repository === `pdp-connect/connector/${entry.connector_key}`) { return core.DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY; @@ -1120,7 +1117,7 @@ export function createConnectorInstallService(options: { }; const limitedTransport = createConfigLimitedFetch(fetch, entry.digest); const artifact = await core.fetchResolvedArtifact( - { doc: {}, mode: "locked" }, + null, { artifactKind: "pdpp-collection-profile", connectorId: entry.connector_id, @@ -1219,9 +1216,6 @@ async function loadPinnedCore(): Promise { async function loadCatalogFromPinnedCore(lastAcceptedGeneratedAt: string | null): Promise { const core = await loadPinnedCore(); - if (!core.fetchCatalog) { - throw new Error("Pinned connector installer core does not expose fetchCatalog."); - } const value = (await core.fetchCatalog({ identity: core.DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY, registry: "ghcr.io", @@ -1399,9 +1393,6 @@ async function installPinnedArtifact( const core = await loadPinnedCore(); const abortableFetch: FetchLike = (input, init) => fetch(input, { ...init, signal: init?.signal ? AbortSignal.any([init.signal, signal]) : signal }); - if (!core.fetchResolvedArtifact) { - throw new Error("Pinned connector installer core does not expose fetchResolvedArtifact."); - } const identityResolver = ({ registry, repository }: { registry: string; repository: string }) => { if (registry === "ghcr.io" && repository === `pdp-connect/connector/${entry.connector_key}`) { return core.DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY; @@ -1410,7 +1401,7 @@ async function installPinnedArtifact( }; const preflightTransport = createConfigLimitedFetch(abortableFetch, entry.digest); const preflight = await core.fetchResolvedArtifact( - { doc: {}, mode: "locked" }, + null, { artifactKind: "pdpp-collection-profile", connectorId: entry.connector_id, @@ -1447,7 +1438,6 @@ async function installPinnedArtifact( } const limitedTransport = createConfigLimitedFetch(abortableFetch, entry.digest); await core.installFromLock({ - artifactCertificateIdentityResolver: () => null, fetchImpl: limitedTransport.fetchImpl, installRoot: root, layout: "source", @@ -1476,7 +1466,7 @@ async function installPinnedArtifact( registry === "ghcr.io" && repository === `pdp-connect/connector/${entry.connector_key}` ? core.DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY : null, - source: { doc: {}, mode: "locked" }, + source: null, }); const installedConfigDigest = limitedTransport.configDigest(); if (!installedConfigDigest || installedConfigDigest !== configDigest) { diff --git a/reference-implementation/server/local-collector-profiles/apple-photos.json b/reference-implementation/server/local-collector-profiles/apple-photos.json index 42e5d2c8e..c9b26f3be 100644 --- a/reference-implementation/server/local-collector-profiles/apple-photos.json +++ b/reference-implementation/server/local-collector-profiles/apple-photos.json @@ -1,12 +1,18 @@ { "protocol_version": "0.1.0", "connector_id": "https://registry.pdpp.dev/connectors/apple-photos", + "source": { + "id": "https://registry.pdpp.dev/sources/apple_photos", + "display": { + "name": "Apple Photos" + } + }, "connector_key": "apple-photos", "manifest_uri": "https://registry.pdpp.dev/connectors/apple-photos", - "version": "0.1.0", + "version": "0.1.3", "display_name": "Apple Photos", "brand": { - "icon": "icons/apple_photos.svg", + "icon": "icon.svg", "background_color": "#FA633E" }, "runtime_requirements": { diff --git a/reference-implementation/server/local-collector-profiles/apple-photos.pin.json b/reference-implementation/server/local-collector-profiles/apple-photos.pin.json index c625dfd1f..afa614a7e 100644 --- a/reference-implementation/server/local-collector-profiles/apple-photos.pin.json +++ b/reference-implementation/server/local-collector-profiles/apple-photos.pin.json @@ -1,6 +1,6 @@ { "connector_key": "apple-photos", - "version": "0.1.0", - "digest": "sha256:0952cef101b05b5416d0ea0620ee12c857f493bae547ce71119a243131741415", - "profile_sha256": "sha256:1d941bd3fc02831618fcd29b899b091ed2ce51f25535f35f58c5a2aaa37ce596" + "version": "0.1.3", + "digest": "sha256:3e58cf196d7fc119fa7f462432d069a257f5e62036a7c70c6b542bb7b779f07f", + "profile_sha256": "sha256:d290a1d201b9b38955225e4b5cae6054a84097723dac8d600e06c5db4676036a" } diff --git a/reference-implementation/server/local-collector-profiles/claude-code.json b/reference-implementation/server/local-collector-profiles/claude-code.json index 11b6cf048..954725a6f 100644 --- a/reference-implementation/server/local-collector-profiles/claude-code.json +++ b/reference-implementation/server/local-collector-profiles/claude-code.json @@ -1,12 +1,18 @@ { "protocol_version": "0.1.0", "connector_id": "https://registry.pdpp.dev/connectors/claude-code", + "source": { + "id": "https://registry.pdpp.dev/sources/claude_code", + "display": { + "name": "Claude Code" + } + }, "connector_key": "claude-code", "manifest_uri": "https://registry.pdpp.dev/connectors/claude-code", - "version": "0.3.0", + "version": "0.3.4", "display_name": "Claude Code", "brand": { - "icon": "icons/claude_code.svg", + "icon": "icon.svg", "background_color": "#D97757" }, "runtime_requirements": { @@ -97,14 +103,25 @@ "message_count": { "type": ["integer", "null"] }, + "title": { + "type": ["string", "null"], + "x_pdpp_role": "secondary" + }, "user_type": { "type": ["string", "null"] }, "entrypoint": { "type": ["string", "null"] + }, + "kind": { + "type": "string", + "enum": ["session", "subagent"] + }, + "parent_session_id": { + "type": ["string", "null"] } }, - "required": ["id"] + "required": ["id", "kind"] }, "primary_key": ["id"], "cursor_field": "last_event_at", @@ -114,6 +131,14 @@ "resources": true }, "incremental": true, + "relationships": [ + { + "name": "parent_session", + "stream": "sessions", + "foreign_key": "parent_session_id", + "cardinality": "has_one" + } + ], "query": { "search": { "lexical_fields": [ @@ -121,9 +146,11 @@ "cwd", "git_branch", "version", + "title", "user_type", "entrypoint" - ] + ], + "semantic_fields": ["title"] }, "range_filters": { "started_at": ["gte", "gt", "lte", "lt"], @@ -186,9 +213,15 @@ }, "agent_id": { "type": ["string", "null"] + }, + "subagent_session_id": { + "type": ["string", "null"] + }, + "has_pasted_content": { + "type": "boolean" } }, - "required": ["id", "session_id"] + "required": ["id", "session_id", "has_pasted_content"] }, "primary_key": ["id"], "cursor_field": "timestamp", @@ -204,6 +237,12 @@ "stream": "sessions", "foreign_key": "session_id", "cardinality": "has_one" + }, + { + "name": "subagent_session", + "stream": "sessions", + "foreign_key": "subagent_session_id", + "cardinality": "has_one" } ], "query": { @@ -272,6 +311,27 @@ "type": ["string", "null"], "format": "date-time", "x_pdpp_role": "event-time" + }, + "content_binary_reason": { + "type": ["string", "null"] + }, + "blob_ref": { + "type": ["object", "null"], + "properties": { + "blob_id": { "type": "string" }, + "mime_type": { "type": "string" }, + "size_bytes": { "type": "integer" }, + "sha256": { "type": "string" } + }, + "required": ["blob_id", "mime_type", "size_bytes", "sha256"], + "additionalProperties": true + }, + "artifact_capture": { + "type": ["string", "null"], + "enum": ["captured", "failed", "unavailable", null] + }, + "artifact_sha256": { + "type": ["string", "null"] } }, "required": ["id", "session_id"] @@ -507,6 +567,140 @@ "coverage_strategy": "snapshot_import_receipt", "freshness_strategy": "device_heartbeat" }, + { + "name": "usage", + "required": true, + "description": "Aggregate Claude Code usage from ~/.claude/stats-cache.json: total sessions and messages, per-model token/cost totals, day-by-day activity, hour-of-day distribution, and the longest session.", + "compaction_class": "recurring_snapshot", + "compaction_class_note": "single aggregate record re-derived in full from stats-cache.json every run; never compactable, never re-alarms as growth accumulates.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Claude Code usage", + "detail": "Total sessions and messages, per-model token and cost totals, daily activity, hour-of-day distribution, and your longest session." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "total_sessions": { + "type": ["integer", "null"] + }, + "total_messages": { + "type": ["integer", "null"] + }, + "first_session_date": { + "type": ["string", "null"], + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "last_computed_date": { + "type": ["string", "null"], + "format": "date" + }, + "total_speculation_time_saved_ms": { + "type": ["integer", "null"] + }, + "total_cost_usd_cents": { + "type": "integer" + }, + "currency": { + "type": "string" + }, + "models": { + "type": "array", + "items": { + "type": "object", + "properties": { + "model": { "type": "string" }, + "input_tokens": { "type": "integer" }, + "output_tokens": { "type": "integer" }, + "cache_read_input_tokens": { "type": "integer" }, + "cache_creation_input_tokens": { "type": "integer" }, + "web_search_requests": { "type": "integer" }, + "cost_usd_cents": { "type": "integer" }, + "context_window": { "type": ["integer", "null"] }, + "max_output_tokens": { "type": ["integer", "null"] } + } + } + }, + "daily_activity": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { "type": "string", "format": "date" }, + "message_count": { "type": "integer" }, + "session_count": { "type": "integer" }, + "tool_call_count": { "type": "integer" } + } + } + }, + "daily_model_tokens": { + "type": "array", + "items": { + "type": "object", + "properties": { + "date": { "type": "string", "format": "date" }, + "tokens_by_model": { "type": "object" } + } + } + }, + "hour_counts": { + "type": "object" + }, + "longest_session": { + "type": ["object", "null"], + "properties": { + "session_id": { "type": ["string", "null"] }, + "duration_seconds": { "type": ["integer", "null"] }, + "message_count": { "type": ["integer", "null"] }, + "timestamp": { "type": ["string", "null"], "format": "date-time" } + } + }, + "source": { + "type": "string" + } + }, + "required": [ + "id", + "total_cost_usd_cents", + "currency", + "models", + "daily_activity", + "daily_model_tokens", + "hour_counts", + "source" + ] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "search": { + "lexical_fields": ["source"] + }, + "range_filters": { + "first_session_date": ["gte", "gt", "lte", "lt"], + "last_computed_date": ["gte", "gt", "lte", "lt"], + "total_sessions": ["gte", "gt", "lte", "lt"], + "total_messages": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["first_session_date"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, { "name": "file_history", "description": "Metadata-only inventory for Claude Code file-history stores. Does not emit file-history payload content.", @@ -815,6 +1009,9 @@ }, { "name": "slash_commands" + }, + { + "name": "usage" } ] }, diff --git a/reference-implementation/server/local-collector-profiles/claude-code.pin.json b/reference-implementation/server/local-collector-profiles/claude-code.pin.json index 430406a60..3af964134 100644 --- a/reference-implementation/server/local-collector-profiles/claude-code.pin.json +++ b/reference-implementation/server/local-collector-profiles/claude-code.pin.json @@ -1,6 +1,6 @@ { "connector_key": "claude-code", - "version": "0.3.0", - "digest": "sha256:5b8866392bb11f24cbc3827e37e96faae9b30c8863d7d82bf9cd21aa2e636c82", - "profile_sha256": "sha256:b6398b12d41043706ac4701db92f577ed9964d86b6e4776d471eca65587ae79f" + "version": "0.3.4", + "digest": "sha256:08a82d8da8ed4eb70551007141383d3afaa9dd9878afb4fa76543ecfd53ee447", + "profile_sha256": "sha256:1d9f5132894c8c14e5aa296b55f90c60414ef772b0fa6f6b3c0d9b55525b1381" } diff --git a/reference-implementation/server/local-collector-profiles/codex.json b/reference-implementation/server/local-collector-profiles/codex.json index 62d967652..41c04e98f 100644 --- a/reference-implementation/server/local-collector-profiles/codex.json +++ b/reference-implementation/server/local-collector-profiles/codex.json @@ -1,12 +1,18 @@ { "protocol_version": "0.1.0", "connector_id": "https://registry.pdpp.dev/connectors/codex", + "source": { + "id": "https://registry.pdpp.dev/sources/codex", + "display": { + "name": "OpenAI Codex CLI" + } + }, "connector_key": "codex", "manifest_uri": "https://registry.pdpp.dev/connectors/codex", - "version": "0.3.0", + "version": "0.3.3", "display_name": "OpenAI Codex CLI", "brand": { - "icon": "icons/codex.svg", + "icon": "icon.svg", "background_color": "#10A37F" }, "runtime_requirements": { diff --git a/reference-implementation/server/local-collector-profiles/codex.pin.json b/reference-implementation/server/local-collector-profiles/codex.pin.json index b168c290a..4d7d81d05 100644 --- a/reference-implementation/server/local-collector-profiles/codex.pin.json +++ b/reference-implementation/server/local-collector-profiles/codex.pin.json @@ -1,6 +1,6 @@ { "connector_key": "codex", - "version": "0.3.0", - "digest": "sha256:bb75129071d1afcf60bf5e20097b84e1262d3cb2ce3ca5b3c228d23f00495d28", - "profile_sha256": "sha256:3287a16b01517439443e8641e97c7f1bfcb358d01641d3b3490331b2ddc107ab" + "version": "0.3.3", + "digest": "sha256:8cbea0c5b56578ab94cc830bd5c69dce77aa5927106f8401d44a84fd3e39fdab", + "profile_sha256": "sha256:f8d1773bb2b856c278748f512aba16fbc993b9a833fbfa9d50c357b7a232776f" } diff --git a/reference-implementation/server/local-collector-profiles/google-takeout.json b/reference-implementation/server/local-collector-profiles/google-takeout.json index b942a406d..392ff2a84 100644 --- a/reference-implementation/server/local-collector-profiles/google-takeout.json +++ b/reference-implementation/server/local-collector-profiles/google-takeout.json @@ -1,14 +1,16 @@ { "protocol_version": "0.1.0", "connector_id": "https://registry.pdpp.dev/connectors/google-takeout", + "source": { + "id": "https://registry.pdpp.dev/sources/google_takeout", + "display": { + "name": "Google Takeout" + } + }, "connector_key": "google-takeout", "manifest_uri": "https://registry.pdpp.dev/connectors/google-takeout", - "version": "0.1.0", + "version": "0.1.3", "display_name": "Google Takeout", - "brand": { - "icon": "icons/google_takeout.svg", - "background_color": "#FBBC05" - }, "runtime_requirements": { "bindings": { "network": { diff --git a/reference-implementation/server/local-collector-profiles/google-takeout.pin.json b/reference-implementation/server/local-collector-profiles/google-takeout.pin.json index c9a40966c..b648bb441 100644 --- a/reference-implementation/server/local-collector-profiles/google-takeout.pin.json +++ b/reference-implementation/server/local-collector-profiles/google-takeout.pin.json @@ -1,6 +1,6 @@ { "connector_key": "google-takeout", - "version": "0.1.0", - "digest": "sha256:38d4bd1312073b167b668cad1701ebc78fea8cf9c19a0edb37a98d11fd660562", - "profile_sha256": "sha256:708f149f0efedae0b41cc3f67a065c541ec6ab5305edb322b9f983a9d0913353" + "version": "0.1.3", + "digest": "sha256:9d1f4934c42cddd73a00b2792ee7edcc389058c566e462a00cd8210fcb014653", + "profile_sha256": "sha256:d645fffba61b0a47a077dc4fe4d04a803481844b3281d333818ab4532d9f01cd" } diff --git a/reference-implementation/server/local-collector-profiles/imessage.json b/reference-implementation/server/local-collector-profiles/imessage.json index ac221b9ac..960489d23 100644 --- a/reference-implementation/server/local-collector-profiles/imessage.json +++ b/reference-implementation/server/local-collector-profiles/imessage.json @@ -1,12 +1,18 @@ { "protocol_version": "0.1.0", "connector_id": "https://registry.pdpp.dev/connectors/imessage", + "source": { + "id": "https://registry.pdpp.dev/sources/imessage", + "display": { + "name": "iMessage" + } + }, "connector_key": "imessage", "manifest_uri": "https://registry.pdpp.dev/connectors/imessage", - "version": "0.1.0", + "version": "0.1.25", "display_name": "iMessage (macOS)", "brand": { - "icon": "icons/imessage.svg", + "icon": "icon.svg", "background_color": "#30D158" }, "runtime_requirements": { diff --git a/reference-implementation/server/local-collector-profiles/imessage.pin.json b/reference-implementation/server/local-collector-profiles/imessage.pin.json index c978468a2..5215000f6 100644 --- a/reference-implementation/server/local-collector-profiles/imessage.pin.json +++ b/reference-implementation/server/local-collector-profiles/imessage.pin.json @@ -1,6 +1,6 @@ { "connector_key": "imessage", - "version": "0.1.0", - "digest": "sha256:ace3798a8e222e73fa1555a39ba52b245e2566d31d84bb60fdec61c9ae1a9ca6", - "profile_sha256": "sha256:654eb8abb9625554c19edcf7e1ef6416fca556929456454ba29277751cde8efb" + "version": "0.1.25", + "digest": "sha256:920246f08102aea9c7958d6b3a0bf89eb07e3b776a921333f6d7ae93f99394b0", + "profile_sha256": "sha256:399958b7d89e1263b5f61e6b09de4c7f5d6a81c1084ff887e75a504278820ef4" } diff --git a/reference-implementation/test/collection-report-projection.test.ts b/reference-implementation/test/collection-report-projection.test.ts index 4cd803802..4e2ea219b 100644 --- a/reference-implementation/test/collection-report-projection.test.ts +++ b/reference-implementation/test/collection-report-projection.test.ts @@ -2,6 +2,7 @@ // SPDX-License-Identifier: Apache-2.0 import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; import test from "node:test"; import { @@ -295,6 +296,33 @@ test("a required stream reading unknown drags the connection coverage axis off c assert.notEqual(override, "partial"); }); +test("the pinned claude-code profile: a collector that never reports its required usage stream leaves coverage unknown", () => { + // The pinned profile requires `usage`. A connector emits a stream only when + // it is requested, so a collector that does not list `usage` among its + // default streams reports every other stream complete and still cannot reach + // `complete`. The collector's definition lists `usage`; this keeps the server + // side of that contract visible. + const profile = JSON.parse( + readFileSync(new URL("../server/local-collector-profiles/claude-code.json", import.meta.url), "utf8") + ) as { streams: ManifestStreamFixture[] }; + const requiredNames = profile.streams.filter((stream) => stream.required !== false).map((stream) => stream.name); + assert.ok(requiredNames.includes("usage")); + const reportFor = (reported: string[]) => + buildCollectionReport({ + attentionOpen: false, + collectionFacts: { + streams: reported.map((stream) => fact({ considered: 1, covered: 1, stream })), + }, + freshness: "fresh", + manifestStreams: profile.streams, + refresh: null, + }); + const withoutUsage = reportFor(requiredNames.filter((name) => name !== "usage")); + assert.equal(rollupCollectionReportCoverageOverride("complete", withoutUsage, profile.streams), "unknown"); + const withUsage = reportFor(requiredNames); + assert.notEqual(rollupCollectionReportCoverageOverride("complete", withUsage, profile.streams), "unknown"); +}); + test("unreliable projection withholds required complete coverage but preserves optional policy outcomes", () => { const entries = buildCollectionReport({ attentionOpen: false, diff --git a/reference-implementation/test/collector-profile-upgrade.test.ts b/reference-implementation/test/collector-profile-upgrade.test.ts new file mode 100644 index 000000000..4136972ed --- /dev/null +++ b/reference-implementation/test/collector-profile-upgrade.test.ts @@ -0,0 +1,421 @@ +// Copyright The PDP-Connect Contributors +// SPDX-License-Identifier: Apache-2.0 + +/** + * Mixed versions of the claude-code Collection Profile. + * + * The collector pin and the profile this server registers move together + * (claude-code 0.3.0 to 0.3.4), but a device and a server do not upgrade at the + * same moment, and the server registers a pinned profile only when a device + * enrolls (`ensureReferenceConnectorCatalogEntry`). This test holds one enrolled + * device across both directions: + * + * - a new collector pushes `usage` to a server that still holds the previous + * profile: the push is refused at ingest, nothing is stored; + * - the server then registers the new profile: the same connection keeps its + * id, its checkpoint and its records, `usage` is accepted, the coverage proof + * stamped under the previous profile reads `generation_mismatch` (`unknown`), + * and the next run's coverage write brings the axis back. + * + * The same registration is then repeated for the other four re-pinned profiles + * (codex, google-takeout, apple-photos, imessage), whose streams and keys do not + * change: the connection id, the checkpoint and the records survive, and a + * local-coverage proof written under the previous profile is untrusted until + * the next run (the connectors that keep one). + * + * `fixtures/collector-profile-upgrade/-.json` is the verbatim + * profile the previous pin shipped. Only the device-side process is simulated; + * enrollment, registration, ingest, state and the health projection are the + * production paths. + */ + +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import test from "node:test"; +import { fileURLToPath } from "node:url"; + +import { expectedLocalCoverageStoreDescriptors } from "@pdpp/polyfill-connectors/local-source-inventory"; +import { registerConnector } from "../server/auth.ts"; +import { COLLECTOR_PROTOCOL_VERSION } from "../server/collector-protocol.ts"; +import { reconcileDirtyConnectorSummaryEvidence } from "../server/connector-summary-read-model.ts"; +import { getDb } from "../server/db.ts"; +import { startServer } from "../server/index.ts"; +import { readLocalCollectorProfile } from "../server/local-collector-profiles.ts"; +import { putSyncState, readCommittedLocalCoverageDiagnostics } from "../server/records.ts"; +import { deriveLocalCoverageAxis, listConnectorSummaries } from "../server/ref-control.ts"; +import { closeTestServer } from "./helpers/close-test-server.ts"; +import { fetchJson, type TestServer } from "./helpers/legacy-upgrade.ts"; + +const HERE = dirname(fileURLToPath(import.meta.url)); +const CONNECTOR_ID = "claude-code"; +const FIXTURES = join(HERE, "fixtures", "collector-profile-upgrade"); +const PROTOCOL_HEADERS = { "X-PDPP-Collector-Protocol": COLLECTOR_PROTOCOL_VERSION }; + +function postJson(url: string, body: unknown, headers: Record = {}) { + return fetchJson(url, { + body: JSON.stringify(body), + headers: { Accept: "application/json", "Content-Type": "application/json", ...headers }, + method: "POST", + }); +} + +function canonicalValue(value: unknown): unknown { + if (value === null || typeof value !== "object") { + return value; + } + if (Array.isArray(value)) { + return value.map(canonicalValue); + } + const record = value as Record; + return Object.fromEntries( + Object.keys(record) + .sort() + .filter((key) => record[key] !== undefined) + .map((key) => [key, canonicalValue(record[key])]) + ); +} + +function bodyHash(records: unknown): string { + return createHash("sha256") + .update(JSON.stringify(canonicalValue(records))) + .digest("hex"); +} + +/** The manifest the server registers for a local-collector connector (see `readReferenceLocalConnectorCatalogManifest`). */ +function enrollmentManifest(connectorId: string, profile: Record) { + return { ...profile, connector_id: connectorId, display_name: profile.display_name || connectorId }; +} + +function register(connectorId: string, profile: Record) { + return registerConnector(enrollmentManifest(connectorId, profile), { + backfillRetrievalIndexes: false, + replaceAcceptedDigest: true, + }); +} + +interface Device { + connector_instance_id: string; + device_id: string; + device_token: string; + source_instance_id: string; +} + +let batchSeq = 0; +function push(asUrl: string, device: Device, connectorId: string, records: readonly Record[]) { + batchSeq += 1; + return postJson( + `${asUrl}/_ref/device-exporters/${encodeURIComponent(device.device_id)}/ingest-batches`, + { + batch_id: `profile-upgrade-${batchSeq}`, + batch_seq: batchSeq, + body_hash: bodyHash(records), + connector_id: connectorId, + device_id: device.device_id, + records, + source_instance_id: device.source_instance_id, + }, + { Authorization: `Bearer ${device.device_token}`, ...PROTOCOL_HEADERS } + ); +} + +function emitted(stream: string, key: string, data: Record) { + return { data, emitted_at: "2026-11-05T10:00:00.000Z", record_key: key, stream }; +} + +/** One `coverage_diagnostics` state write, as the connector commits it at the end of a successful run. */ +const CLAUDE_CODE_STORE_STATUSES = new Map([ + ["projects", "collected"], + ["derived_messages", "collected"], + ["derived_attachments", "collected"], + ["derived_memory_notes", "collected"], + ["cache", "inventory_only"], + ["auth", "excluded"], +]); + +/** + * The stores the connector's fixed inventory declares. `statuses` says which are + * collected; the rest are inventory only. + */ +function coverageRows(connectorId: string, statuses: ReadonlyMap = new Map()) { + const expected = expectedLocalCoverageStoreDescriptors(connectorId); + assert.ok(expected, `the production ${connectorId} connector declares a fixed inventory`); + return expected.map(({ store, stream }) => ({ + status: statuses.get(store) ?? "inventory_only", + store, + stream, + })); +} + +async function commitCoverage( + connectorId: string, + connectorInstanceId: string, + rows: ReturnType +) { + for (const row of rows) { + getDb() + .prepare( + `INSERT INTO records(connector_id, connector_instance_id, stream, record_key, record_json, emitted_at, version, deleted) + VALUES (?, ?, 'coverage_diagnostics', ?, ?, ?, 1, 0) + ON CONFLICT DO NOTHING` + ) + .run( + connectorId, + connectorInstanceId, + `coverage:${row.store}`, + JSON.stringify({ id: `coverage:${row.store}`, ...row }), + "2026-11-05T10:00:00.000Z" + ); + } + await putSyncState( + { connector_id: connectorId, connector_instance_id: connectorInstanceId }, + { coverage_diagnostics: { fetched_at: new Date(Date.now() - 30_000).toISOString(), stores: rows } } + ); +} + +/** The local-coverage axis the server derives from the committed proof (`deriveLocalCoverageAxis`). */ +async function localCoverage(connectorId: string, connectorInstanceId: string) { + const proof = (await readCommittedLocalCoverageDiagnostics({ + connector_id: connectorId, + connector_instance_id: connectorInstanceId, + })) as Parameters[0]; + const { axis, reliable, unreliableReason } = deriveLocalCoverageAxis(proof); + return { axis, reliable, unreliableReason }; +} + +async function projectedConnectionIds(): Promise { + await reconcileDirtyConnectorSummaryEvidence(null); + return (await listConnectorSummaries()).map((summary) => summary.connector_instance_id); +} + +function readProfile(file: string): Record { + return JSON.parse(readFileSync(join(FIXTURES, file), "utf8")) as Record; +} + +function stateRow(connectorInstanceId: string, stream: string) { + return getDb() + .prepare("SELECT state_json FROM connector_state WHERE connector_instance_id = ? AND stream = ?") + .get<{ state_json: string }>(connectorInstanceId, stream)?.state_json; +} + +function recordCount(connectorInstanceId: string, stream: string): number { + return ( + getDb() + .prepare("SELECT COUNT(*) AS n FROM records WHERE connector_instance_id = ? AND stream = ?") + .get<{ n: number }>(connectorInstanceId, stream)?.n ?? 0 + ); +} + +test("a device enrolled on the previous claude-code profile survives the server registering the new profile, in both push directions", async () => { + const previous = readProfile("claude-code-0.3.0.json"); + const current = readLocalCollectorProfile(CONNECTOR_ID); + assert.ok(current, "claude-code must have a pinned Collection Profile"); + const names = (profile: Record) => + (profile.streams as { name: string }[]).map((stream) => stream.name); + assert.equal(previous.version, "0.3.0"); + assert.equal(names(previous).includes("usage"), false); + assert.equal(names(current).includes("usage"), true); + + const server = (await startServer({ + asPort: 0, + dbPath: ":memory:", + quiet: true, + reconcilePolyfillManifests: false, + rsPort: 0, + })) as TestServer; + try { + const asUrl = `http://localhost:${server.asPort}`; + const code = await postJson(`${asUrl}/_ref/device-exporters/enrollment-codes`, { + connector_id: CONNECTOR_ID, + local_binding_name: "laptop", + }); + assert.equal(code.status, 201, JSON.stringify(code.body)); + const enrolled = await postJson( + `${asUrl}/_ref/device-exporters/enroll`, + { enrollment_code: code.body.enrollment_code }, + PROTOCOL_HEADERS + ); + assert.equal(enrolled.status, 201, JSON.stringify(enrolled.body)); + const device = enrolled.body as Device; + const connectionId = device.connector_instance_id; + + // The server a previous release left behind: it holds the previous profile. + await register(CONNECTOR_ID, previous); + + const first = await push(asUrl, device, CONNECTOR_ID, [ + emitted("sessions", "s1", { id: "s1" }), + emitted("messages", "m1", { id: "m1", session_id: "s1" }), + ]); + assert.equal(first.status, 201, JSON.stringify(first.body)); + assert.equal(first.body.connector_instance_id, connectionId); + const checkpoint = { cursor: "sessions-checkpoint-1" }; + await putSyncState({ connector_id: CONNECTOR_ID, connector_instance_id: connectionId }, { sessions: checkpoint }); + await commitCoverage(CONNECTOR_ID, connectionId, coverageRows(CONNECTOR_ID, CLAUDE_CODE_STORE_STATUSES)); + assert.deepEqual(await localCoverage(CONNECTOR_ID, connectionId), { axis: "complete", reliable: true, unreliableReason: undefined }); + assert.deepEqual(await projectedConnectionIds(), [connectionId]); + + // New collector, previous server: `usage` is not declared, so the push is refused and stores nothing. + const usage = emitted("usage", "usage", { + currency: "USD", + daily_activity: [], + daily_model_tokens: [], + hour_counts: {}, + id: "usage", + models: [], + source: "stats-cache", + total_cost_usd_cents: 0, + }); + const refused = await push(asUrl, device, CONNECTOR_ID, [usage]); + assert.equal(refused.status, 400, JSON.stringify(refused.body)); + assert.equal(refused.body.error.message, "records[0].stream is not declared by the connector manifest"); + assert.equal(recordCount(connectionId, "usage"), 0); + + // The server registers the new profile. + await register(CONNECTOR_ID, current); + + const accepted = await push(asUrl, device, CONNECTOR_ID, [usage]); + assert.equal(accepted.status, 201, JSON.stringify(accepted.body)); + assert.equal(accepted.body.connector_instance_id, connectionId); + assert.equal(recordCount(connectionId, "usage"), 1); + // Identity, checkpoint and stored records are untouched by the profile change. + assert.equal(recordCount(connectionId, "sessions"), 1); + assert.equal(recordCount(connectionId, "messages"), 1); + assert.deepEqual(JSON.parse(stateRow(connectionId, "sessions") ?? "null"), checkpoint); + assert.deepEqual(await projectedConnectionIds(), [connectionId]); + // The coverage proof was stamped under the previous profile, so it is not + // trusted until the next run writes it again (`generation_mismatch`). + assert.deepEqual(await localCoverage(CONNECTOR_ID, connectionId), { + axis: "unknown", + reliable: false, + unreliableReason: "generation_mismatch", + }); + + await commitCoverage(CONNECTOR_ID, connectionId, coverageRows(CONNECTOR_ID, CLAUDE_CODE_STORE_STATUSES)); + assert.deepEqual(await localCoverage(CONNECTOR_ID, connectionId), { axis: "complete", reliable: true, unreliableReason: undefined }); + assert.deepEqual(await projectedConnectionIds(), [connectionId]); + } finally { + await closeTestServer(server); + } +}); + +interface StreamSchema { + properties?: Record; + required?: string[]; +} + +/** The smallest record the profile's own schema accepts for `stream`. */ +function minimalRecord(profile: Record, stream: string, key: string) { + const declared = (profile.streams as { name: string; schema: StreamSchema }[]).find((s) => s.name === stream); + assert.ok(declared, `${stream} is declared`); + const data: Record = {}; + for (const name of declared.schema.required ?? []) { + const property = declared.schema.properties?.[name] ?? {}; + const type = [property.type].flat().find((t) => t !== "null"); + if (name === "id") { + data[name] = key; + } else if (property.enum) { + data[name] = property.enum[0]; + } else if (property.format === "date-time") { + data[name] = "2026-11-05T10:00:00.000Z"; + } else if (type === "integer" || type === "number") { + data[name] = 1; + } else if (type === "boolean") { + data[name] = true; + } else { + data[name] = "x"; + } + } + return emitted(stream, key, data); +} + +/** + * The four other re-pinned profiles. Their streams, keys and cursors do not + * change, so the claude-code test above carries the stream-level behaviour; this + * one proves the registration itself is safe for each of them, against the + * verbatim profile the previous pin shipped. + */ +const OTHER_REPINNED = [ + { id: "codex", previous: "codex-0.3.0.json" }, + { id: "google-takeout", previous: "google-takeout-0.1.0.json" }, + { id: "apple-photos", previous: "apple-photos-0.1.0.json" }, + { id: "imessage", previous: "imessage-0.1.0.json" }, +]; + +for (const { id, previous: previousFile } of OTHER_REPINNED) { + test(`${id}: a device enrolled on the previous profile keeps its connection, checkpoint and records when the server registers the new profile`, async () => { + const previous = readProfile(previousFile); + const current = readLocalCollectorProfile(id); + assert.ok(current, `${id} must have a pinned Collection Profile`); + assert.notEqual(previous.version, current.version); + const streamNames = (profile: Record) => + (profile.streams as { name: string }[]).map((stream) => stream.name).sort(); + assert.deepEqual(streamNames(previous), streamNames(current), "the re-pin leaves the declared streams alone"); + const stream = (current.streams as { name: string }[]).find((s) => s.name !== "coverage_diagnostics")?.name; + assert.ok(stream); + const hasCoverageProof = expectedLocalCoverageStoreDescriptors(id) !== null; + + const server = (await startServer({ + asPort: 0, + dbPath: ":memory:", + quiet: true, + reconcilePolyfillManifests: false, + rsPort: 0, + })) as TestServer; + try { + const asUrl = `http://localhost:${server.asPort}`; + const code = await postJson(`${asUrl}/_ref/device-exporters/enrollment-codes`, { + connector_id: id, + local_binding_name: "laptop", + }); + assert.equal(code.status, 201, JSON.stringify(code.body)); + const enrolled = await postJson( + `${asUrl}/_ref/device-exporters/enroll`, + { enrollment_code: code.body.enrollment_code }, + PROTOCOL_HEADERS + ); + assert.equal(enrolled.status, 201, JSON.stringify(enrolled.body)); + const device = enrolled.body as Device; + const connectionId = device.connector_instance_id; + + await register(id, previous); + const first = await push(asUrl, device, id, [minimalRecord(previous, stream, "r1")]); + assert.equal(first.status, 201, JSON.stringify(first.body)); + assert.equal(first.body.connector_instance_id, connectionId); + const checkpoint = { cursor: `${id}-checkpoint-1` }; + await putSyncState({ connector_id: id, connector_instance_id: connectionId }, { [stream]: checkpoint }); + const proof = () => + coverageRows( + id, + new Map( + (expectedLocalCoverageStoreDescriptors(id) ?? []).filter((d) => d.stream).map((d) => [d.store, "collected"]) + ) + ); + if (hasCoverageProof) { + await commitCoverage(id, connectionId, proof()); + assert.equal((await localCoverage(id, connectionId)).reliable, true); + } + assert.deepEqual(await projectedConnectionIds(), [connectionId]); + + await register(id, current); + + const second = await push(asUrl, device, id, [minimalRecord(current, stream, "r2")]); + assert.equal(second.status, 201, JSON.stringify(second.body)); + assert.equal(second.body.connector_instance_id, connectionId); + assert.equal(recordCount(connectionId, stream), 2); + assert.deepEqual(JSON.parse(stateRow(connectionId, stream) ?? "null"), checkpoint); + assert.deepEqual(await projectedConnectionIds(), [connectionId]); + if (hasCoverageProof) { + assert.deepEqual(await localCoverage(id, connectionId), { + axis: "unknown", + reliable: false, + unreliableReason: "generation_mismatch", + }); + await commitCoverage(id, connectionId, proof()); + assert.equal((await localCoverage(id, connectionId)).reliable, true); + } + } finally { + await closeTestServer(server); + } + }); +} diff --git a/reference-implementation/test/fixtures/collector-profile-upgrade/apple-photos-0.1.0.json b/reference-implementation/test/fixtures/collector-profile-upgrade/apple-photos-0.1.0.json new file mode 100644 index 000000000..42e5d2c8e --- /dev/null +++ b/reference-implementation/test/fixtures/collector-profile-upgrade/apple-photos-0.1.0.json @@ -0,0 +1,185 @@ +{ + "protocol_version": "0.1.0", + "connector_id": "https://registry.pdpp.dev/connectors/apple-photos", + "connector_key": "apple-photos", + "manifest_uri": "https://registry.pdpp.dev/connectors/apple-photos", + "version": "0.1.0", + "display_name": "Apple Photos", + "brand": { + "icon": "icons/apple_photos.svg", + "background_color": "#FA633E" + }, + "runtime_requirements": { + "bindings": { + "filesystem": { + "required": true + } + } + }, + "capabilities": { + "human_interaction": [], + "proven": { + "local_collector": true + }, + "refresh_policy": { + "recommended_mode": "manual", + "interaction_posture": "manual_action_likely", + "rate_limit_sensitivity": "low", + "bot_detection_sensitivity": "low", + "background_safe": false, + "rationale": "Apple Photos imports require the user to manually export files from Photos.app (File → Export → Export Unmodified Originals) into the import directory; there is no API or background-refreshable path — PhotoKit requires a signed app bundle and is not reachable from this connector, and the reverse-engineered Photos.sqlite path is out of scope by design." + }, + "public_listing": { + "tier": "development" + } + }, + "streams": [ + { + "name": "photos", + "required": true, + "description": "File-level metadata AND hydrated bytes for images and videos exported from Photos.app, within a configured size cap. Uses the same shared local-media blob hydration module as google_takeout's photos stream (src/local-media-blob-hydration.ts) — bounded local read, sha256 content hash, MIME-by-extension, and upload through the runtime's reference-blob-uploader when blob-upload bindings are configured. A record's `id` is derived from content_sha256 alone, so the same photo present in more than one Photos.app export collapses to one record rather than duplicating.", + "display": { + "label": "Your exported photos and videos", + "detail": "Filename, size, content hash, MIME type, file-modified timestamp, and (within the size cap) hydrated bytes for each file found in the Photos.app export directory. hydration_status/hydration_error explain when bytes are absent (oversized or unreadable) without dropping the record. EXIF/XMP fields (taken date, GPS, camera) are present in the schema but not populated in this cut — a bounded, explicit omission, not silently missing coverage." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "filename": { + "type": "string" + }, + "content_type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "content_sha256": { + "type": ["string", "null"] + }, + "hydration_status": { + "type": "string", + "enum": ["failed", "hydrated", "skipped_too_large", "unavailable"] + }, + "hydration_error": { + "type": ["string", "null"] + }, + "blob_ref": { + "type": ["object", "null"], + "properties": { + "blob_id": { + "type": "string" + }, + "mime_type": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "size_bytes": { + "type": "integer" + } + }, + "required": ["blob_id", "mime_type", "sha256", "size_bytes"] + }, + "taken_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "file_modified_at": { + "type": "string", + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "latitude": { + "type": ["number", "null"] + }, + "longitude": { + "type": ["number", "null"] + }, + "camera_make": { + "type": ["string", "null"] + }, + "camera_model": { + "type": ["string", "null"] + } + }, + "required": [ + "id", + "filename", + "content_type", + "hydration_status", + "file_modified_at" + ] + }, + "primary_key": ["id"], + "cursor_field": "file_modified_at", + "consent_time_field": "file_modified_at", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "query": { + "range_filters": { + "file_modified_at": ["gte", "gt", "lte", "lt"], + "taken_at": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["file_modified_at", "taken_at"], + "group_by": ["content_type"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "manual_as_of" + }, + { + "name": "coverage_diagnostics", + "required": false, + "description": "Safe coverage status for this connector's one known local store (the Photos.app export directory), so a local-collector run always leaves durable, honest coverage evidence even when the export directory is missing or empty — never a silent zero-evidence run.", + "display": { + "label": "Local collector coverage status", + "detail": "Whether the Photos.app export directory was found and collected on this run." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "stream": { + "type": ["string", "null"] + }, + "status": { + "type": "string" + }, + "reason": { + "type": "string", + "x_pdpp_role": "primary-title" + } + }, + "required": ["id", "store", "status", "reason"] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + } + ], + "reason_display_messages": { + "export_not_found": "We couldn't find an export to import" + } +} diff --git a/reference-implementation/test/fixtures/collector-profile-upgrade/claude-code-0.3.0.json b/reference-implementation/test/fixtures/collector-profile-upgrade/claude-code-0.3.0.json new file mode 100644 index 000000000..11b6cf048 --- /dev/null +++ b/reference-implementation/test/fixtures/collector-profile-upgrade/claude-code-0.3.0.json @@ -0,0 +1,880 @@ +{ + "protocol_version": "0.1.0", + "connector_id": "https://registry.pdpp.dev/connectors/claude-code", + "connector_key": "claude-code", + "manifest_uri": "https://registry.pdpp.dev/connectors/claude-code", + "version": "0.3.0", + "display_name": "Claude Code", + "brand": { + "icon": "icons/claude_code.svg", + "background_color": "#D97757" + }, + "runtime_requirements": { + "bindings": { + "filesystem": { + "required": true + } + }, + "environment_variables": [ + "CLAUDE_CODE_PROJECT_INCLUDE", + "CLAUDE_CODE_PROJECT_EXCLUDE" + ], + "local_paths": { + "home_env_override": "CLAUDE_CODE_HOME", + "home_default_relative_to_user_home": ".claude", + "paths": [ + { + "default_relative_to_home": "projects", + "env_override": "CLAUDE_CODE_PROJECTS_DIR", + "label": "projects directory", + "required_for_readiness": true + } + ] + } + }, + "capabilities": { + "human_interaction": [], + "proven": { + "local_collector": true + }, + "refresh_policy": { + "recommended_mode": "automatic", + "recommended_interval_seconds": 3600, + "minimum_interval_seconds": 300, + "maximum_staleness_seconds": 21600, + "interaction_posture": "none", + "rate_limit_sensitivity": "low", + "bot_detection_sensitivity": "low", + "background_safe": true, + "rationale": "Claude Code session logs live on local disk; re-reading them is cheap and has no platform cost." + }, + "public_listing": { + "tier": "supported" + } + }, + "streams": [ + { + "name": "sessions", + "required": true, + "description": "Claude Code sessions. One session = one CLI invocation in a specific project directory.", + "compaction_class": "recurring_snapshot", + "compaction_class_note": "aggregated per-session record; message_count/last_event_at widen on growth; never compactable, never re-alarms as growth accumulates.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Claude Code sessions", + "detail": "Session start time, duration, project directory, git branch, Claude Code version, total message count." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "project_path": { + "type": ["string", "null"] + }, + "cwd": { + "type": ["string", "null"] + }, + "git_branch": { + "type": ["string", "null"] + }, + "version": { + "type": ["string", "null"] + }, + "started_at": { + "type": ["string", "null"], + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "last_event_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "message_count": { + "type": ["integer", "null"] + }, + "user_type": { + "type": ["string", "null"] + }, + "entrypoint": { + "type": ["string", "null"] + } + }, + "required": ["id"] + }, + "primary_key": ["id"], + "cursor_field": "last_event_at", + "consent_time_field": "started_at", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "query": { + "search": { + "lexical_fields": [ + "project_path", + "cwd", + "git_branch", + "version", + "user_type", + "entrypoint" + ] + }, + "range_filters": { + "started_at": ["gte", "gt", "lte", "lt"], + "last_event_at": ["gte", "gt", "lte", "lt"], + "message_count": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["last_event_at", "started_at"] + } + }, + "coverage_strategy": "checkpoint_window", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "messages", + "required": true, + "description": "Messages within Claude Code sessions: user prompts and assistant responses. Includes both top-level session transcripts and nested subagent (sidechain) transcripts.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Claude Code messages", + "detail": "Role (user/assistant), text content, timestamp, parent message (for threading), session reference, subagent id when the message came from a sidechain run." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "session_id": { + "type": "string" + }, + "parent_uuid": { + "type": ["string", "null"] + }, + "role": { + "type": ["string", "null"], + "x_pdpp_role": "actor" + }, + "type": { + "type": ["string", "null"] + }, + "content": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "is_sidechain": { + "type": ["boolean", "null"] + }, + "user_type": { + "type": ["string", "null"] + }, + "agent_id": { + "type": ["string", "null"] + } + }, + "required": ["id", "session_id"] + }, + "primary_key": ["id"], + "cursor_field": "timestamp", + "consent_time_field": "timestamp", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "relationships": [ + { + "name": "session", + "stream": "sessions", + "foreign_key": "session_id", + "cardinality": "has_one" + } + ], + "query": { + "search": { + "lexical_fields": [ + "role", + "type", + "content", + "user_type", + "agent_id" + ], + "semantic_fields": ["content"] + }, + "range_filters": { + "timestamp": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["timestamp"] + } + }, + "coverage_strategy": "checkpoint_window", + "state_stream": "sessions", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "attachments", + "required": true, + "description": "Non-message events: hook outputs, tool uses, file snapshots, plus tool-results blob files written to disk.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Claude Code events", + "detail": "System events attached to sessions: tool calls, hook outputs, file-history snapshots, permission-mode changes, and persisted tool-result files." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "session_id": { + "type": "string" + }, + "parent_uuid": { + "type": ["string", "null"] + }, + "event_type": { + "type": ["string", "null"] + }, + "hook_name": { + "type": ["string", "null"] + }, + "tool_use_id": { + "type": ["string", "null"] + }, + "content_preview": { + "type": ["string", "null"] + }, + "content_bytes": { + "type": ["integer", "null"] + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time", + "x_pdpp_role": "event-time" + } + }, + "required": ["id", "session_id"] + }, + "primary_key": ["id"], + "cursor_field": "timestamp", + "consent_time_field": "timestamp", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "relationships": [ + { + "name": "session", + "stream": "sessions", + "foreign_key": "session_id", + "cardinality": "has_one" + } + ], + "query": { + "search": { + "lexical_fields": ["event_type", "hook_name", "content_preview"], + "semantic_fields": ["content_preview"] + }, + "range_filters": { + "timestamp": ["gte", "gt", "lte", "lt"], + "content_bytes": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["timestamp"] + } + }, + "coverage_strategy": "checkpoint_window", + "state_stream": "sessions", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "skills", + "required": true, + "description": "User-authored Claude Code skills installed under ~/.claude/skills//SKILL.md. Each record captures frontmatter metadata and the full body (capped at 20k chars).", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Claude Code skills", + "detail": "Skill name, description, full SKILL.md body, frontmatter, and on-disk path." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "description": { + "type": ["string", "null"], + "x_pdpp_role": "secondary" + }, + "source": { + "type": ["string", "null"] + }, + "path": { + "type": ["string", "null"] + }, + "content": { + "type": ["string", "null"], + "x_pdpp_role": "secondary" + }, + "frontmatter": { + "type": ["object", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"] + } + }, + "required": ["id"] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "search": { + "lexical_fields": ["name", "description", "source", "content"], + "semantic_fields": ["description", "content"] + }, + "range_filters": { + "mtime_epoch": ["gte", "gt", "lte", "lt"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "memory_notes", + "required": true, + "description": "Project-scoped Claude Code memory notes under ~/.claude/projects//memory/*.md. Each record captures frontmatter metadata and the note body (capped at 20k chars).", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Claude Code memory notes", + "detail": "Project path, note path, name, description, body, frontmatter, and on-disk path." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "project_path": { + "type": "string" + }, + "note_path": { + "type": "string" + }, + "name": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "description": { + "type": ["string", "null"], + "x_pdpp_role": "secondary" + }, + "path": { + "type": ["string", "null"] + }, + "content": { + "type": ["string", "null"] + }, + "frontmatter": { + "type": ["object", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"] + } + }, + "required": ["id", "project_path", "note_path"] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "search": { + "lexical_fields": [ + "project_path", + "note_path", + "name", + "description", + "content" + ], + "semantic_fields": ["description", "content"] + }, + "range_filters": { + "mtime_epoch": ["gte", "gt", "lte", "lt"] + } + }, + "coverage_strategy": "checkpoint_window", + "state_stream": "sessions", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "slash_commands", + "required": true, + "description": "User-authored Claude Code slash commands under ~/.claude/commands/*.md. Each record captures frontmatter, the command body (capped at 20k chars), and the file's absolute path.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your custom slash commands", + "detail": "Command name, description, full command body, frontmatter, and on-disk path." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "description": { + "type": ["string", "null"], + "x_pdpp_role": "secondary" + }, + "path": { + "type": ["string", "null"] + }, + "content": { + "type": ["string", "null"] + }, + "frontmatter": { + "type": ["object", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"] + } + }, + "required": ["id"] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "search": { + "lexical_fields": ["name", "description", "content"], + "semantic_fields": ["description", "content"] + }, + "range_filters": { + "mtime_epoch": ["gte", "gt", "lte", "lt"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "file_history", + "description": "Metadata-only inventory for Claude Code file-history stores. Does not emit file-history payload content.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "cache_inventory", + "description": "Metadata-only inventory for Claude Code cache stores; raw cache payloads are not collected.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "coverage_diagnostics", + "description": "Safe coverage status for known Claude Code local stores, including excluded auth-adjacent stores without payloads or raw absolute paths.", + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "stream": { + "type": ["string", "null"] + }, + "status": { + "type": "string" + }, + "reason": { + "type": "string", + "x_pdpp_role": "primary-title" + } + }, + "required": ["id", "store", "status", "reason"] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "backup_inventory", + "description": "Metadata-only inventory for Claude Code backup stores; raw backup payloads are not collected.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "config_inventory", + "description": "Metadata-only inventory for non-secret Claude Code configuration; raw config payloads are not collected.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + } + ], + "profiles": [ + { + "id": "full", + "label": "Your full Claude Code history", + "streams": [ + { + "name": "sessions" + }, + { + "name": "messages" + }, + { + "name": "attachments" + }, + { + "name": "skills" + }, + { + "name": "memory_notes" + }, + { + "name": "slash_commands" + } + ] + }, + { + "id": "messages_only", + "label": "Just prompts and responses", + "streams": [ + { + "name": "sessions" + }, + { + "name": "messages" + } + ] + }, + { + "id": "customizations", + "label": "Your skills, memory notes, and slash commands", + "streams": [ + { + "name": "skills" + }, + { + "name": "memory_notes" + }, + { + "name": "slash_commands" + } + ] + } + ], + "reason_display_messages": { + "claude_dir_not_found": "We couldn't find your Claude Code data folder", + "malformed_jsonl_line": "A line in this Claude Code session log contains invalid JSON and was skipped", + "non_object_jsonl_record": "A line in this Claude Code session log has an unexpected format and was skipped", + "scope_matched_no_sources": "None of your configured project folders matched any Claude Code projects", + "source_directory_read_error": "We couldn't read this Claude Code folder. It will be checked again on the next run", + "source_read_error": "This Claude Code session log could not be collected and will be retried on the next run", + "source_unreadable": "We couldn't read this folder, so we don't know what's in it", + "symlink_skipped": "A symbolic link was skipped without reading its target. The skipped path is listed in the source diagnostics", + "truncated_jsonl_tail": "The last line of this Claude Code session log is incomplete and will be checked again on the next run" + }, + "options_schema": { + "type": "object", + "description": "Owner tuning knobs for the Claude Code connector, resolved from the connector_instance_config_revisions spine. declared_option_kind is this connector's CLAIM only; the platform registry in packages/polyfill-connectors/src/connector-config-option-kind-registry.ts decides what actually gates self-activation, and an option the registry does not classify is enforced as collection_scope.", + "properties": { + "CLAUDE_CODE_PROJECT_INCLUDE": { + "type": "array", + "items": { "type": "string" }, + "description": "Substrings of project folder paths to include. A folder is collected if it contains any of these. Empty list means every project folder is eligible.", + "default": [], + "declared_option_kind": "collection_scope" + }, + "CLAUDE_CODE_PROJECT_EXCLUDE": { + "type": "array", + "items": { "type": "string" }, + "description": "Substrings of project folder paths to exclude. A folder is dropped if it contains any of these, even when the include list matched it.", + "default": [], + "declared_option_kind": "collection_scope" + } + } + } +} diff --git a/reference-implementation/test/fixtures/collector-profile-upgrade/codex-0.3.0.json b/reference-implementation/test/fixtures/collector-profile-upgrade/codex-0.3.0.json new file mode 100644 index 000000000..62d967652 --- /dev/null +++ b/reference-implementation/test/fixtures/collector-profile-upgrade/codex-0.3.0.json @@ -0,0 +1,908 @@ +{ + "protocol_version": "0.1.0", + "connector_id": "https://registry.pdpp.dev/connectors/codex", + "connector_key": "codex", + "manifest_uri": "https://registry.pdpp.dev/connectors/codex", + "version": "0.3.0", + "display_name": "OpenAI Codex CLI", + "brand": { + "icon": "icons/codex.svg", + "background_color": "#10A37F" + }, + "runtime_requirements": { + "bindings": { + "filesystem": { + "required": true + } + }, + "environment_variables": [ + "CODEX_RULES_DIR", + "CODEX_PROMPTS_DIR", + "CODEX_SKILLS_DIR" + ], + "local_paths": { + "home_env_override": "CODEX_HOME", + "home_default_relative_to_user_home": ".codex", + "paths": [ + { + "default_relative_to_home": "sessions", + "env_override": "CODEX_SESSIONS_DIR", + "label": "sessions directory", + "required_for_readiness": true + }, + { + "default_relative_to_home": "state_5.sqlite", + "env_override": "CODEX_STATE_DB", + "label": "state database", + "required_for_readiness": true + } + ] + } + }, + "capabilities": { + "human_interaction": [], + "proven": { + "local_collector": true + }, + "refresh_policy": { + "recommended_mode": "automatic", + "recommended_interval_seconds": 3600, + "minimum_interval_seconds": 300, + "maximum_staleness_seconds": 21600, + "interaction_posture": "none", + "rate_limit_sensitivity": "low", + "bot_detection_sensitivity": "low", + "background_safe": true, + "rationale": "Codex CLI session state lives on local disk; re-reading it is cheap." + }, + "public_listing": { + "tier": "supported" + } + }, + "streams": [ + { + "name": "sessions", + "required": true, + "description": "Codex CLI sessions. Canonical metadata comes from ~/.codex/state_5.sqlite#threads (title, archived, tokens_used, first_user_message, sandbox_policy, approval_mode), enriched with message/tool-call counts from the on-disk rollout JSONL.", + "compaction_class": "recurring_snapshot", + "compaction_class_note": "whole record is an evolving observation (message_count, last_event_at widen on growth); never compactable, never re-alarms as growth accumulates.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Codex CLI sessions", + "detail": "Title, working directory, git commit/branch, model provider, CLI version, approval mode, sandbox policy, token usage, and counts." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "cwd": { + "type": ["string", "null"] + }, + "originator": { + "type": ["string", "null"] + }, + "cli_version": { + "type": ["string", "null"] + }, + "model_provider": { + "type": ["string", "null"] + }, + "git_commit": { + "type": ["string", "null"] + }, + "git_branch": { + "type": ["string", "null"] + }, + "repository_url": { + "type": ["string", "null"] + }, + "started_at": { + "type": ["string", "null"], + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "last_event_at": { + "type": ["string", "null"], + "format": "date-time" + }, + "message_count": { + "type": ["integer", "null"] + }, + "function_call_count": { + "type": ["integer", "null"] + }, + "title": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "archived": { + "type": ["boolean", "null"] + }, + "tokens_used": { + "type": ["integer", "null"] + }, + "first_user_message": { + "type": ["string", "null"] + }, + "sandbox_policy": { + "type": ["string", "null"] + }, + "approval_mode": { + "type": ["string", "null"] + }, + "rollout_path": { + "type": ["string", "null"] + } + }, + "required": ["id"] + }, + "primary_key": ["id"], + "cursor_field": "last_event_at", + "consent_time_field": "started_at", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "query": { + "search": { + "lexical_fields": [ + "cwd", + "originator", + "model_provider", + "git_branch", + "title", + "first_user_message", + "sandbox_policy", + "approval_mode" + ], + "semantic_fields": ["title", "first_user_message"] + }, + "range_filters": { + "started_at": ["gte", "gt", "lte", "lt"], + "last_event_at": ["gte", "gt", "lte", "lt"], + "message_count": ["gte", "gt", "lte", "lt"], + "function_call_count": ["gte", "gt", "lte", "lt"], + "tokens_used": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["last_event_at", "started_at"] + } + }, + "coverage_strategy": "checkpoint_window", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "messages", + "required": true, + "description": "Messages within Codex sessions: user prompts, assistant responses, and reasoning placeholders.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Codex messages", + "detail": "Role (user/assistant), text content, timestamp, session reference. Reasoning traces are referenced but content is encrypted and not stored." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "session_id": { + "type": "string" + }, + "role": { + "type": ["string", "null"], + "x_pdpp_role": "actor" + }, + "type": { + "type": ["string", "null"] + }, + "content": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time", + "x_pdpp_role": "event-time" + } + }, + "required": ["id", "session_id"] + }, + "primary_key": ["id"], + "cursor_field": "timestamp", + "consent_time_field": "timestamp", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "relationships": [ + { + "name": "session", + "stream": "sessions", + "foreign_key": "session_id", + "cardinality": "has_one" + } + ], + "query": { + "search": { + "lexical_fields": ["role", "type", "content"], + "semantic_fields": ["content"] + }, + "range_filters": { + "timestamp": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["timestamp"] + } + }, + "coverage_strategy": "checkpoint_window", + "state_stream": "sessions", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "function_calls", + "required": true, + "description": "Tool invocations and their outputs (shell commands, file ops) recorded in Codex sessions.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Codex tool calls", + "detail": "Tool name, arguments, call_id, and output preview. Includes shell commands Codex ran and their results." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "session_id": { + "type": "string" + }, + "call_id": { + "type": ["string", "null"] + }, + "name": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "arguments": { + "type": ["string", "null"] + }, + "output_preview": { + "type": ["string", "null"] + }, + "timestamp": { + "type": ["string", "null"], + "format": "date-time", + "x_pdpp_role": "event-time" + } + }, + "required": ["id", "session_id"] + }, + "primary_key": ["id"], + "cursor_field": "timestamp", + "consent_time_field": "timestamp", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "relationships": [ + { + "name": "session", + "stream": "sessions", + "foreign_key": "session_id", + "cardinality": "has_one" + } + ], + "query": { + "search": { + "lexical_fields": ["name", "arguments", "output_preview"], + "semantic_fields": ["arguments", "output_preview"] + }, + "range_filters": { + "timestamp": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["timestamp"] + } + }, + "coverage_strategy": "checkpoint_window", + "state_stream": "sessions", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "rules", + "required": true, + "description": "Personal trust-registry rules recorded in ~/.codex/rules/*.rules — the de-facto log of approval decisions. One record per rule line. The stream is named `rules` to match the real local directory; `approval_rules` was backlog shorthand, not an on-disk source name.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Codex approval rules", + "detail": "Each rule line (e.g. prefix_rule / decision=allow) from your .rules files, with ruleset and position." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "ruleset": { + "type": "string" + }, + "rule_text": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "rule_index": { + "type": "integer" + }, + "path": { + "type": ["string", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + } + }, + "required": ["id", "ruleset", "rule_text", "rule_index"] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "search": { + "lexical_fields": ["ruleset", "rule_text", "path"], + "semantic_fields": ["rule_text"] + }, + "range_filters": { + "mtime_epoch": ["gte", "gt", "lte", "lt"] + } + }, + "coverage_strategy": "full_inventory", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "prompts", + "required": true, + "description": "User-authored prompt templates from ~/.codex/prompts/*.md. Frontmatter (name, description) is parsed; body is the prompt content.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Codex prompts", + "detail": "Name, description, body, and source path for each prompt template." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "description": { + "type": ["string", "null"], + "x_pdpp_role": "secondary" + }, + "content": { + "type": ["string", "null"] + }, + "path": { + "type": ["string", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"] + } + }, + "required": ["id", "name"] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "search": { + "lexical_fields": ["name", "description", "content", "path"], + "semantic_fields": ["description", "content"] + }, + "range_filters": { + "mtime_epoch": ["gte", "gt", "lte", "lt"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "skills", + "required": true, + "description": "User-authored skills from ~/.codex/skills//SKILL.md. Frontmatter (name, description) is parsed; body is the skill content.", + "compaction_fingerprint": { + "exclude_keys": [] + }, + "display": { + "label": "Your Codex skills", + "detail": "Name, description, body, and source path for each skill." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "name": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "description": { + "type": ["string", "null"], + "x_pdpp_role": "secondary" + }, + "content": { + "type": ["string", "null"], + "x_pdpp_role": "secondary" + }, + "path": { + "type": ["string", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"] + } + }, + "required": ["id", "name"] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "search": { + "lexical_fields": ["name", "description", "content", "path"], + "semantic_fields": ["description", "content"] + }, + "range_filters": { + "mtime_epoch": ["gte", "gt", "lte", "lt"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "history", + "description": "Metadata-only inventory for Codex history.jsonl. Prompt-history payloads are not collected in this tranche.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "session_index", + "description": "Metadata-only inventory for Codex session_index.jsonl.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "coverage_diagnostics", + "description": "Safe coverage status for known Codex local stores, including excluded auth-adjacent stores without payloads or raw absolute paths.", + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "stream": { + "type": ["string", "null"] + }, + "status": { + "type": "string" + }, + "reason": { + "type": "string", + "x_pdpp_role": "primary-title" + } + }, + "required": ["id", "store", "status", "reason"] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "shell_snapshots", + "description": "Metadata-only inventory for Codex shell snapshots; shell content is not collected.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "config_inventory", + "description": "Metadata-only inventory for non-secret Codex configuration; raw config payloads are not collected.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + }, + { + "name": "cache_inventory", + "description": "Metadata-only inventory for Codex cache stores; raw cache payloads are not collected.", + "compaction_fingerprint": { + "exclude_keys": ["mtime_epoch", "size_bytes"] + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "relative_path": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "path_hash": { + "type": "string" + }, + "type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "mtime_epoch": { + "type": ["integer", "null"], + "x_pdpp_role": "event-time" + }, + "classification": { + "type": "string" + }, + "reason": { + "type": "string" + } + }, + "required": [ + "id", + "store", + "relative_path", + "path_hash", + "type", + "classification", + "reason" + ] + }, + "primary_key": ["id"], + "cursor_field": "mtime_epoch", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + } + ], + "profiles": [ + { + "id": "full", + "label": "Your full Codex history", + "streams": [ + { + "name": "sessions" + }, + { + "name": "messages" + }, + { + "name": "function_calls" + }, + { + "name": "rules" + }, + { + "name": "prompts" + }, + { + "name": "skills" + } + ] + }, + { + "id": "messages_only", + "label": "Just prompts and responses", + "streams": [ + { + "name": "sessions" + }, + { + "name": "messages" + } + ] + }, + { + "id": "authoring", + "label": "Your prompts, skills, and approval rules", + "streams": [ + { + "name": "rules" + }, + { + "name": "prompts" + }, + { + "name": "skills" + } + ] + } + ], + "reason_display_messages": { + "symlink_skipped": "A symbolic link in your Codex data was skipped without reading its target", + "malformed_jsonl_line": "Part of this session's log file couldn't be read, so those entries were skipped. Nothing was lost on your machine — the original file is unchanged", + "non_object_jsonl_record": "A line in this Codex session log has an unexpected format and was skipped", + "rollout_source_read_error": "We couldn't read part of your Codex data. It will be checked again on the next run", + "shape_check_failed": "The data didn't look like we expected", + "truncated_jsonl_tail": "The last line of this Codex session log is incomplete and will be checked again on the next run" + } +} diff --git a/reference-implementation/test/fixtures/collector-profile-upgrade/google-takeout-0.1.0.json b/reference-implementation/test/fixtures/collector-profile-upgrade/google-takeout-0.1.0.json new file mode 100644 index 000000000..b942a406d --- /dev/null +++ b/reference-implementation/test/fixtures/collector-profile-upgrade/google-takeout-0.1.0.json @@ -0,0 +1,354 @@ +{ + "protocol_version": "0.1.0", + "connector_id": "https://registry.pdpp.dev/connectors/google-takeout", + "connector_key": "google-takeout", + "manifest_uri": "https://registry.pdpp.dev/connectors/google-takeout", + "version": "0.1.0", + "display_name": "Google Takeout", + "brand": { + "icon": "icons/google_takeout.svg", + "background_color": "#FBBC05" + }, + "runtime_requirements": { + "bindings": { + "network": { + "required": false + }, + "filesystem": { + "required": true + } + }, + "environment_variables": ["GOOGLE_TAKEOUT_DIR"] + }, + "capabilities": { + "human_interaction": [], + "proven": { + "local_collector": true + }, + "refresh_policy": { + "recommended_mode": "manual", + "interaction_posture": "manual_action_likely", + "rate_limit_sensitivity": "low", + "bot_detection_sensitivity": "low", + "background_safe": false, + "rationale": "Google Takeout requires the user to request and download a fresh archive; refresh cannot run in the background." + }, + "public_listing": { + "tier": "development" + } + }, + "streams": [ + { + "name": "location_history", + "description": "Location history from Google Maps Timeline.", + "display": { + "label": "Your Google location history", + "detail": "Places you've been with timestamps, lat/lon, accuracy, activity type when available. Parsed from Google Takeout Records.json." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "timestamp": { + "type": "string", + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "latitude": { + "type": ["number", "null"] + }, + "longitude": { + "type": ["number", "null"] + }, + "accuracy_meters": { + "type": ["number", "null"] + }, + "activity_type": { + "type": ["string", "null"] + }, + "velocity_mps": { + "type": ["number", "null"] + }, + "altitude_m": { + "type": ["number", "null"] + } + }, + "required": ["id", "timestamp"] + }, + "primary_key": ["id"], + "cursor_field": "timestamp", + "consent_time_field": "timestamp", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "query": { + "range_filters": { + "timestamp": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["timestamp"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "manual_as_of" + }, + { + "name": "youtube_watch_history", + "description": "YouTube watch history.", + "display": { + "label": "Your YouTube watch history", + "detail": "Videos watched with watch time, video URL, title, channel name." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "watched_at": { + "type": "string", + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "video_url": { + "type": ["string", "null"] + }, + "video_title": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "channel_name": { + "type": ["string", "null"], + "x_pdpp_role": "actor" + }, + "channel_url": { + "type": ["string", "null"] + } + }, + "required": ["id", "watched_at"] + }, + "primary_key": ["id"], + "cursor_field": "watched_at", + "consent_time_field": "watched_at", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "query": { + "range_filters": { + "watched_at": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["watched_at"] + }, + "search": { + "lexical_fields": ["video_title", "channel_name"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "manual_as_of" + }, + { + "name": "search_history", + "description": "Google search queries from My Activity.", + "display": { + "label": "Your Google search history", + "detail": "Search queries with timestamps and products (Search, Maps, Images, etc.)." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "timestamp": { + "type": "string", + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "query": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "product": { + "type": ["string", "null"] + } + }, + "required": ["id", "timestamp"] + }, + "primary_key": ["id"], + "cursor_field": "timestamp", + "consent_time_field": "timestamp", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "query": { + "range_filters": { + "timestamp": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["timestamp"] + }, + "search": { + "lexical_fields": ["query"], + "semantic_fields": ["query"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "manual_as_of" + }, + { + "name": "photos", + "description": "Photos and videos from Google Takeout.", + "display": { + "label": "Your Google photos and videos", + "detail": "Photos/videos with metadata: timestamps, title, description, geolocation when available. Full-resolution media. Parsed from Google Takeout Photos/ directory." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "filename": { + "type": "string" + }, + "event_time": { + "type": "string", + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "title": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "description": { + "type": ["string", "null"] + }, + "latitude": { + "type": ["number", "null"] + }, + "longitude": { + "type": ["number", "null"] + }, + "altitude": { + "type": ["number", "null"] + }, + "blob_ref": { + "type": ["object", "null"], + "properties": { + "blob_id": { + "type": "string" + }, + "mime_type": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "size_bytes": { + "type": "integer" + } + }, + "required": ["blob_id", "mime_type", "sha256", "size_bytes"] + }, + "content_sha256": { + "type": ["string", "null"] + }, + "size_bytes": { + "type": ["number", "null"] + }, + "hydration_status": { + "type": "string", + "enum": ["failed", "hydrated", "skipped_too_large", "unavailable"] + }, + "hydration_error": { + "type": ["string", "null"] + } + }, + "required": ["id", "filename", "event_time"] + }, + "required": true, + "primary_key": ["id"], + "cursor_field": "event_time", + "consent_time_field": "event_time", + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "query": { + "range_filters": { + "event_time": ["gte", "gt", "lte", "lt"] + }, + "aggregations": { + "count": true, + "group_by_time": ["event_time"] + }, + "search": { + "lexical_fields": ["title", "description"], + "semantic_fields": ["title", "description"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "manual_as_of" + }, + { + "name": "coverage_diagnostics", + "description": "Safe coverage status for known Google Takeout local stores.", + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "store": { + "type": "string" + }, + "stream": { + "type": ["string", "null"] + }, + "status": { + "type": "string" + }, + "reason": { + "type": "string", + "x_pdpp_role": "primary-title" + } + }, + "required": ["id", "store", "status", "reason"] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "required": false, + "coverage_policy": "inventory_only", + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "device_heartbeat" + } + ], + "reason_display_messages": { + "directory_read_failed": "We couldn't read a folder in your Google Takeout export", + "history_not_found": "We couldn't find any history to import", + "photos_not_found": "We couldn't find your Google Photos export", + "records_not_found": "We didn't find any records to import" + } +} diff --git a/reference-implementation/test/fixtures/collector-profile-upgrade/imessage-0.1.0.json b/reference-implementation/test/fixtures/collector-profile-upgrade/imessage-0.1.0.json new file mode 100644 index 000000000..ac221b9ac --- /dev/null +++ b/reference-implementation/test/fixtures/collector-profile-upgrade/imessage-0.1.0.json @@ -0,0 +1,226 @@ +{ + "protocol_version": "0.1.0", + "connector_id": "https://registry.pdpp.dev/connectors/imessage", + "connector_key": "imessage", + "manifest_uri": "https://registry.pdpp.dev/connectors/imessage", + "version": "0.1.0", + "display_name": "iMessage (macOS)", + "brand": { + "icon": "icons/imessage.svg", + "background_color": "#30D158" + }, + "runtime_requirements": { + "bindings": { + "filesystem": { + "required": true + } + }, + "environment_variables": ["IMESSAGE_DB_PATH", "IMESSAGE_ATTACHMENTS_ROOT"] + }, + "capabilities": { + "human_interaction": [], + "proven": { + "local_collector": true + }, + "public_listing": { + "tier": "development", + "rationale": "Hidden from the reference dashboard catalog until an operator proves a local macOS Messages database run on this deployment and explicitly opts it into listing." + }, + "refresh_policy": { + "recommended_mode": "automatic", + "recommended_interval_seconds": 3600, + "minimum_interval_seconds": 300, + "maximum_staleness_seconds": 21600, + "interaction_posture": "none", + "rate_limit_sensitivity": "low", + "bot_detection_sensitivity": "low", + "background_safe": true, + "rationale": "iMessage history is read from the operator's local macOS Messages database over a filesystem binding: no network provider, no interactive login, nothing to assist, so unattended refresh is mechanically safe. Whether the database is reachable is a runtime-requirements question (filesystem binding, IMESSAGE_DB_PATH) enforced by the pre-run deployment-readiness gate, not by refresh mode; Docker/provider runs must use a local collector or an explicitly mounted path." + } + }, + "streams": [ + { + "name": "messages", + "description": "iMessage + SMS messages from ~/Library/Messages/chat.db (macOS).", + "display": { + "label": "Your iMessages", + "detail": "Sender/recipient, text content, timestamp, service (iMessage or SMS), is_from_me flag, attachment presence." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "chat_id": { + "type": ["string", "null"] + }, + "handle": { + "type": ["string", "null"], + "x_pdpp_role": "actor" + }, + "service": { + "type": ["string", "null"] + }, + "is_from_me": { + "type": "boolean" + }, + "text": { + "type": ["string", "null"], + "x_pdpp_role": "primary-title" + }, + "date": { + "type": "string", + "format": "date-time", + "x_pdpp_role": "event-time" + }, + "date_read": { + "type": ["string", "null"] + }, + "has_attachments": { + "type": "boolean" + } + }, + "required": ["id", "date"] + }, + "primary_key": ["id"], + "cursor_field": "date", + "consent_time_field": "date", + "selection": { + "fields": true, + "resources": true + }, + "incremental": true, + "query": { + "search": { + "lexical_fields": ["text"], + "semantic_fields": ["text"] + } + }, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "manual_as_of" + }, + { + "name": "participants", + "required": false, + "description": "Group-chat and one-to-one chat membership from chat_handle_join (undocumented, reverse-engineered chat.db schema). Optional: older macOS chat.db schema versions may not expose chat_handle_join, in which case this stream degrades to a labeled SKIP_RESULT instead of failing the run.", + "display": { + "label": "Your iMessage chat participants", + "detail": "Chat and counterparty handle. is_from_me reports whether that handle has ever sent a message in the chat (a participant-level 'has this contact spoken here' marker) — not the per-message sender flag used in the messages stream, and best-effort only: the device owner's own handle is frequently absent from chat_handle_join entirely." + }, + "semantics": "mutable_state", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "chat_id": { + "type": "string" + }, + "handle": { + "type": ["string", "null"], + "x_pdpp_role": "actor" + }, + "is_from_me": { + "type": "boolean" + } + }, + "required": ["id", "chat_id"] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "coverage_strategy": "snapshot_import_receipt", + "freshness_strategy": "manual_as_of" + }, + { + "name": "attachments", + "required": false, + "description": "Attachment metadata and bytes from ~/Library/Messages, joined via message_attachment_join (undocumented, reverse-engineered chat.db schema). Bytes are read only from inside a trusted attachments root (default ~/Library/Messages/Attachments, override via IMESSAGE_ATTACHMENTS_ROOT) — the raw path recorded in chat.db is canonicalized and verified to resolve inside that root before any read, rejecting `../` traversal, an absolute path outside the root, and a symlink that escapes the root, and uploaded as a BlobRef when the runtime has blob-upload bindings. The local filesystem path itself never leaves the connector process. Optional: older macOS chat.db schema versions may not expose attachment/message_attachment_join, in which case this stream degrades to a labeled SKIP_RESULT instead of failing the run.", + "display": { + "label": "Your iMessage attachments", + "detail": "Filename, MIME type, size, linked message/chat when detectable, and a blob reference when the local file was hydrated." + }, + "semantics": "append_only", + "schema": { + "type": "object", + "properties": { + "id": { + "type": "string" + }, + "message_id": { + "type": ["string", "null"] + }, + "chat_id": { + "type": ["string", "null"] + }, + "filename": { + "type": "string", + "x_pdpp_role": "primary-title" + }, + "content_type": { + "type": "string" + }, + "size_bytes": { + "type": ["integer", "null"] + }, + "content_sha256": { + "type": ["string", "null"] + }, + "hydration_status": { + "type": "string", + "enum": ["deferred", "hydrated", "failed", "too_large", "missing"] + }, + "hydration_error": { + "type": ["string", "null"] + }, + "blob_ref": { + "type": ["object", "null"], + "properties": { + "blob_id": { + "type": "string" + }, + "mime_type": { + "type": "string" + }, + "sha256": { + "type": "string" + }, + "size_bytes": { + "type": "integer" + } + }, + "required": ["blob_id", "mime_type", "sha256", "size_bytes"] + } + }, + "required": ["id", "filename", "content_type", "hydration_status"] + }, + "primary_key": ["id"], + "selection": { + "fields": true, + "resources": true + }, + "incremental": false, + "relationships": [ + { + "name": "message", + "stream": "messages", + "foreign_key": "message_id", + "cardinality": "has_one" + } + ], + "coverage_strategy": "parent_detail_accounting", + "freshness_strategy": "manual_as_of" + } + ], + "reason_display_messages": { + "attachment_tables_missing": "Your Messages database is missing attachment data we need", + "chat_handle_join_table_missing": "Your Messages database is missing chat-participant data we need", + "message_date_unusable": "We skipped a message with a date we couldn't read" + } +} diff --git a/reference-implementation/test/static-secret-run-credentials.test.ts b/reference-implementation/test/static-secret-run-credentials.test.ts index c989a1c23..b4053dc77 100644 --- a/reference-implementation/test/static-secret-run-credentials.test.ts +++ b/reference-implementation/test/static-secret-run-credentials.test.ts @@ -6,6 +6,7 @@ import test from "node:test"; import { readPolyfillManifests } from "@pdpp/polyfill-connectors/manifests"; import { closeDb, getDb, initDb } from "../server/db.ts"; +import { classifyConnectorSetupModality, type ConnectorManifestLike } from "../server/connection-setup-plan.ts"; const MANIFESTS: Record = { chatgpt: { @@ -669,3 +670,55 @@ test("optional capture may omit credentials, while wrong-kind and invalid bundle (error) => error instanceof StaticSecretRunCredentialError && error.code === "recovered_secret_bundle_invalid", ); }); + +// chatgpt 0.1.0 declared a sealed email-and-password setup; chatgpt 0.2.5 +// declares none. A connection that sealed credentials under 0.1.0 keeps its +// stored credential after the upgrade, but no run reads or injects it. +const CHATGPT_0_1_0_MANIFEST = { + capabilities: { human_interaction: ["manual_action"] }, + runtime_requirements: { bindings: { browser: { required: true }, network: { required: true } } }, + setup: { + credential_capture: { + fields: [ + { autocomplete: "username", env: ["CHATGPT_USERNAME"], label: "ChatGPT email", name: "username", required: true, secret: true, type: "email" }, + { autocomplete: "current-password", env: ["CHATGPT_PASSWORD"], label: "ChatGPT password", name: "password", required: true, secret: true, type: "password" }, + ], + kind: "username_password", + label: "ChatGPT sign-in details", + }, + modality: "static_secret", + }, +}; +const { setup: _removedSetup, ...CHATGPT_0_2_5_MANIFEST } = CHATGPT_0_1_0_MANIFEST; + +test( + "chatgpt 0.2.5 stops injecting a credential sealed under 0.1.0, leaves it stored, and does not fail the run", + withStore(async (store) => { + seedConnectorInstance({ connectorId: "chatgpt-pdpp", connectorInstanceId: "cin_chat", ownerSubjectId: "owner_1" }); + await store.capture({ + connectorInstanceId: "cin_chat", + credentialKind: "username_password", + now: NOW, + ownerSubjectId: "owner_1", + secret: JSON.stringify({ password: "chat-pw", username: "owner@example.test" }), + }); + const args = { connectorId: "chatgpt-pdpp", connectorInstanceId: "cin_chat", ownerSubjectId: "owner_1" }; + + const before = await resolveStaticSecretRunEnv({ ...args, credentialStore: store, manifest: CHATGPT_0_1_0_MANIFEST }); + assert.deepEqual(before, { CHATGPT_PASSWORD: "chat-pw", CHATGPT_USERNAME: "owner@example.test" }); + assert.equal(classifyConnectorSetupModality("chatgpt", CHATGPT_0_1_0_MANIFEST as unknown as ConnectorManifestLike), "static_secret"); + + let recovered = 0; + const spy = { + recoverSecret: async (recoverArgs: { connectorInstanceId: string; ownerSubjectId: string }) => { + recovered += 1; + return store.recoverSecret(recoverArgs); + }, + }; + const after = await resolveStaticSecretRunEnv({ ...args, credentialStore: spy, manifest: CHATGPT_0_2_5_MANIFEST }); + assert.equal(after, null, "no credential env reaches the 0.2.5 run"); + assert.equal(recovered, 0, "the sealed credential is not read"); + assert.notEqual(classifyConnectorSetupModality("chatgpt", CHATGPT_0_2_5_MANIFEST as unknown as ConnectorManifestLike), "static_secret"); + assert.equal(await store.hasActiveCredential("cin_chat"), true, "the sealed credential stays stored"); + }) +); diff --git a/scripts/resolve-connectors.js b/scripts/resolve-connectors.js index 9b4c263ac..ce259a52d 100644 --- a/scripts/resolve-connectors.js +++ b/scripts/resolve-connectors.js @@ -17,17 +17,29 @@ import { import { basename, dirname, join, relative, resolve, sep } from "path" import { fileURLToPath } from "url" import { - DEFAULT_CONNECTOR_INDEX_URL, DEFAULT_OCI_SIGSTORE_CERTIFICATE_IDENTITY, fetchResolvedArtifact, - generateLock, installFromLock, - loadConnectorIndex, readJson, - satisfies, sha256Buffer, verifyInstalled, } from "@opendatalabs/data-connectors-tools/installer-core" +// STOPGAP, removed when the legacy tarball connectors move to Collection +// Profiles (tracked in #633). The current installer core installs only signed +// OCI Collection Profiles that carry a signed source declaration. The bundled tarball +// connectors, and the one OCI release published before source declarations +// existed (see PRE_DECLARATION_OCI_CONNECTORS), still need the earlier core, +// which is pinned under its own name so it cannot be mistaken for the current +// one. +import { + DEFAULT_CONNECTOR_INDEX_URL, + fetchResolvedArtifact as legacyFetchResolvedArtifact, + generateLock, + installFromLock as legacyInstallFromLock, + loadConnectorIndex, + satisfies, + verifyInstalled as legacyVerifyInstalled, +} from "data-connectors-legacy-installer/installer-core" import { isMainModule } from "./is-main-module.js" const __dirname = dirname(fileURLToPath(import.meta.url)) @@ -55,11 +67,100 @@ const NON_CONNECTOR_FILES = new Set([ "types", ]) +// What the earlier installer core reads in place of a connector index. export const LOCKED_ARTIFACT_SOURCE = Object.freeze({ mode: "locked", doc: Object.freeze({}), }) +// STOPGAP. The earlier installer core installs exactly 13 lock entries, and +// this is the one place that names them. A guard test in +// resolve-connectors.test.mjs fails if connectors/lock.json and this list +// disagree. +// +// held back by connectors released by +// not an OCI artifact (tarball) chatgpt-playwright, #633 +// claude-export-playwright, +// github-playwright, +// heb-playwright, +// instagram-ads-playwright, +// instagram-playwright, +// linkedin-playwright, +// oura-playwright, +// shop-playwright, +// spotify-playwright, +// wholefoods-playwright, +// youtube-playwright +// no signed source declaration github-pdpp #634 +// +// The tarballs are picked out by `!entry.oci`. github-pdpp has to be named: +// nothing a lock entry or the catalog carries says its release lacks a signed +// source declaration. A lock entry holds the OCI digests and checksums, and a +// catalog release holds a digest and a version; the declaration is a layer of +// the artifact itself, and the catalog has no github entry at all. Reading it +// would need a registry fetch per entry on every install and `--check`, or a +// new lock field. Delete the id here when #634 re-pins github-pdpp to a release +// that carries a declaration. +export const PRE_DECLARATION_OCI_CONNECTORS = Object.freeze( + new Set(["github-pdpp"]) +) + +export const usesEarlierInstallerCore = entry => + !entry.oci || PRE_DECLARATION_OCI_CONNECTORS.has(entry.connectorId) + +function splitLockByInstallerCore(lock) { + const earlier = lock.connectors.filter(usesEarlierInstallerCore) + const current = lock.connectors.filter( + entry => !usesEarlierInstallerCore(entry) + ) + return { + earlier: { ...lock, connectors: earlier }, + current: { ...lock, connectors: current }, + } +} + +// Install each entry with the core that accepts it. The two cores write +// disjoint paths (`/...` and `collection-profiles//...`). +export async function installLockedConnectors({ lock, source, ...options }) { + const { earlier, current } = splitLockByInstallerCore(lock) + const results = [] + if (earlier.connectors.length > 0) { + results.push( + await legacyInstallFromLock({ lock: earlier, source, ...options }) + ) + } + if (current.connectors.length > 0) { + // The current core takes no connector index; `source` stays null. + results.push( + await installFromLock({ lock: current, source: null, ...options }) + ) + } + return { + connectorCount: results.reduce((n, r) => n + r.connectorCount, 0), + filesWritten: results.reduce((n, r) => n + r.filesWritten, 0), + } +} + +async function verifyLockedConnectors({ lock, source, ...options }) { + const { earlier, current } = splitLockByInstallerCore(lock) + const results = [] + if (earlier.connectors.length > 0) { + results.push( + await legacyVerifyInstalled({ lock: earlier, source, ...options }) + ) + } + if (current.connectors.length > 0) { + results.push( + await verifyInstalled({ lock: current, source: null, ...options }) + ) + } + return { + ok: results.every(r => r.ok), + missing: results.flatMap(r => r.missing), + mismatched: results.flatMap(r => r.mismatched), + } +} + // Absent an explicit `--index-url`, stay pinned to whatever the existing lock // already resolved against — for BOTH `--check` and a normal (re)generation. // A bare `node scripts/resolve-connectors.js` must not silently float the 12 @@ -280,7 +381,7 @@ export async function installConnectorsAtomically({ lock, source = LOCKED_ARTIFACT_SOURCE, installRoot, - install = installFromLock, + install = installLockedConnectors, ...options }) { recoverInterruptedInstall(installRoot) @@ -432,11 +533,18 @@ export async function authorOciProfile( repository: `pdp-connect/connector/${connectorKey}`, }, } - const artifact = await fetchResolvedArtifact( - LOCKED_ARTIFACT_SOURCE, - candidate, - { allowTagResolution: true, ociCertificateIdentityResolver, ...options } - ) + const fetchOptions = { + allowTagResolution: true, + ociCertificateIdentityResolver, + ...options, + } + const artifact = PRE_DECLARATION_OCI_CONNECTORS.has(connectorId) + ? await legacyFetchResolvedArtifact( + LOCKED_ARTIFACT_SOURCE, + candidate, + fetchOptions + ) + : await fetchResolvedArtifact(null, candidate, fetchOptions) const { registry, repository, digest, configDigest } = artifact.oci return { ...candidate, @@ -630,7 +738,7 @@ async function main() { } if ( !reportInstalledCheck( - await verifyInstalled({ + await verifyLockedConnectors({ lock, source, installRoot: CONNECTORS_DIR, diff --git a/scripts/resolve-connectors.test.mjs b/scripts/resolve-connectors.test.mjs index a62fb5852..0bd568778 100644 --- a/scripts/resolve-connectors.test.mjs +++ b/scripts/resolve-connectors.test.mjs @@ -17,7 +17,7 @@ import { tmpdir } from "node:os" import { dirname, join, resolve } from "node:path" import { pathToFileURL } from "node:url" import { describe, expect, it } from "vitest" -import { installFromLock } from "@opendatalabs/data-connectors-tools/installer-core" +import { installFromLock as legacyInstallFromLock } from "data-connectors-legacy-installer/installer-core" import { artifactCertificateIdentityResolver, authorOciLock, @@ -26,8 +26,10 @@ import { installConnectorsAtomically, ociCertificateIdentityResolver, LOCKED_ARTIFACT_SOURCE, + PRE_DECLARATION_OCI_CONNECTORS, resolveIndexUrl, recoverInterruptedInstall, + usesEarlierInstallerCore, } from "./resolve-connectors.js" const require = createRequire(import.meta.url) @@ -36,6 +38,9 @@ const installerCorePath = const { readTarGzEntries } = await import( pathToFileURL(join(dirname(installerCorePath), "tar-stream.mjs")).href ) +const { buildSourceDeclaration, serializeSourceDeclaration } = await import( + pathToFileURL(join(dirname(installerCorePath), "source-declaration.mjs")).href +) const lock = JSON.parse(readFileSync("connectors/lock.json", "utf8")) const npmPackage = JSON.parse(readFileSync("package.json", "utf8")) @@ -47,7 +52,11 @@ const legacyArtifacts = lock.connectors.filter(connector => // This is the reviewed data-connectors main commit containing the duplicate // normalized-member guard. Keep the ancestry check local and deterministic; // acceptance must not turn into a live git or network lookup. -const DATA_CONNECTORS_MAIN_ANCESTOR = "6c71697a1523926fe8d475cd243a8219ea21aa5b" +const DATA_CONNECTORS_MAIN_ANCESTOR = "708a6cc17d6821fad222a652084f2bc270c40052" +// Last commit whose installer core still reads the 12 retired tarball entries +// and the one OCI release that predates signed source declarations. +const EARLIER_INSTALLER_CORE_ANCESTOR = + "e1e497f3bca842c0489c472c1ca286630eef5078" describe("connector artifact signer identities", () => { it("trusts only the six exact legacy artifact URLs retained by the lock", () => { @@ -203,6 +212,32 @@ describe("resolved connector-installer archive acceptance", () => { expect(lockCommit).toBe(DATA_CONNECTORS_MAIN_ANCESTOR) }) + it("pins the earlier installer core to one commit in package.json and the lockfile", () => { + const dependency = "data-connectors-legacy-installer" + const packageCommit = + npmPackage.devDependencies[dependency]?.match(/#([0-9a-f]{40})$/)?.[1] + const lockCommit = npmLock.packages[`node_modules/${dependency}`]?.resolved?.match( + /#([0-9a-f]{40})$/ + )?.[1] + expect(packageCommit).toBe(EARLIER_INSTALLER_CORE_ANCESTOR) + expect(lockCommit).toBe(EARLIER_INSTALLER_CORE_ANCESTOR) + }) + + it("keeps one current-core pin across the root, reference implementation and collector packages", () => { + const dependency = "@opendatalabs/data-connectors-tools" + const read = path => JSON.parse(readFileSync(path, "utf8")) + const specs = [ + npmPackage.devDependencies[dependency], + read("reference-implementation/package.json").dependencies?.[dependency] ?? + read("reference-implementation/package.json").devDependencies?.[dependency], + read("packages/local-collector/package.json").dependencies?.[dependency] ?? + read("packages/local-collector/package.json").devDependencies?.[dependency], + ] + for (const spec of specs) { + expect(spec).toMatch(new RegExp(`#${DATA_CONNECTORS_MAIN_ANCESTOR}$`)) + } + }) + it("refuses two differing collection-profile.mjs archive members through the resolved reader", async () => { await temporary(async root => { const archive = archiveMembers(root, "duplicate-entrypoint", [ @@ -262,7 +297,7 @@ describe("resolved connector-installer archive acceptance", () => { } const installRoot = join(root, "installed") - await installFromLock({ + await legacyInstallFromLock({ lock: { connectors: [entry] }, source: { mode: "local", rootDir: root, doc: {} }, installRoot, @@ -294,17 +329,36 @@ describe("resolved connector-installer archive acceptance", () => { // Reproduce PR A's OCI wire fixture. The core still parses manifests, checks // digests, extracts archives and assembles the cosign bundle. Only registry I/O // and Fulcio/Rekor are replaced; the test verifier checks a real EC signature. -function fixture(root) { - const key = "ynab", - connectorId = `${key}-pdpp`, +function fixture(root, { key = "ynab", withDeclaration = true } = {}) { + const connectorId = `${key}-pdpp`, version = "0.3.0" const profile = json({ connector_key: key, connector_id: `https://github.com/PDP-Connect/data-connectors/connector/${key}`, version, protocol_version: "1.0", - display_name: "YNAB", + display_name: key, + source: { + id: `https://registry.pdpp.dev/sources/${key}`, + display: { name: key }, + }, + streams: [ + { + name: "budgets", + primary_key: ["id"], + semantics: "mutable_state", + selection: { fields: true, resources: true }, + schema: { + type: "object", + properties: { id: { type: "string" } }, + required: ["id"], + }, + }, + ], }) + const declaration = serializeSourceDeclaration( + buildSourceDeclaration([JSON.parse(profile)]) + ) const code = Buffer.from("export const collect = () => {};\n") const provenance = json({ connector_key: key, version }) const files = { @@ -316,7 +370,7 @@ function fixture(root) { const entry = { connectorId, connectorKey: key, - company: "YNAB", + company: key, version, resolvedFrom: version, artifactKind: "pdpp-collection-profile", @@ -337,6 +391,9 @@ function fixture(root) { ...JSON.parse(profile), config_version: "1.0", profile_digest: digest(profile), + ...(withDeclaration + ? { source_declaration_digest: digest(declaration) } + : {}), entrypoint: "code/collection-profile.mjs", }) const manifest = json({ @@ -354,6 +411,14 @@ function fixture(root) { "application/vnd.pdpp.connector.licenses.v1.tar+gzip" ), layer(provenance, "application/vnd.pdpp.connector.provenance.v1+json"), + ...(withDeclaration + ? [ + layer( + declaration, + "application/vnd.pdpp.connector.source-declaration.v1+json" + ), + ] + : []), ], }) const manifestDigest = digest(manifest) @@ -790,6 +855,186 @@ describe("OCI consumer acceptance", () => { } })) + it("B-T10 the earlier core installs exactly the 13 connectors resolve-connectors.js names", () => { + const named = [ + "chatgpt-playwright", + "claude-export-playwright", + "github-playwright", + "heb-playwright", + "instagram-ads-playwright", + "instagram-playwright", + "linkedin-playwright", + "oura-playwright", + "shop-playwright", + "spotify-playwright", + "wholefoods-playwright", + "youtube-playwright", + "github-pdpp", + ].sort() + const routed = lock.connectors + .filter(usesEarlierInstallerCore) + .map(connector => connector.connectorId) + .sort() + expect(routed).toEqual(named) + // The comment above PRE_DECLARATION_OCI_CONNECTORS is the one list of them. + const source = readFileSync("scripts/resolve-connectors.js", "utf8") + for (const id of named) expect(source).toContain(id) + }) + + it("B-T9 a mixed lock installs through the default install: tarball by the earlier core, declaration-bearing OCI by the current core, declaration-less listed OCI by the earlier core; an unlisted declaration-less OCI is refused", async () => + temporary(async root => { + // `ynab` stands for a lock-delivered connector that has no generated + // local-collector pin: it must install and resolve the same way. + const pins = readFileSync( + "packages/local-collector/src/generated/collection-profile-pins.generated.ts", + "utf8" + ) + expect(pins).not.toContain('"connectorKey": "ynab"') + const signed = fixture(join(root, "signed"), { key: "ynab" }) + const listed = fixture(join(root, "listed"), { + key: "github", + withDeclaration: false, + }) + expect(PRE_DECLARATION_OCI_CONNECTORS.has(listed.oci.connectorId)).toBe( + true + ) + + const tarballUrl = legacyArtifacts[0].artifactUrl + const bundleUrl = `${tarballUrl}.sigstore.json` + const manifest = json({ + connector_id: "acme-tarball", + version: "1.0.0", + name: "Acme", + }) + const script = Buffer.from("export const run = () => {};\n") + const tar = archive(join(root, "tarball"), "legacy", { + "manifest.json": manifest, + "script.js": script, + }) + const tarballEntry = { + connectorId: "acme-tarball", + company: "acme", + version: "1.0.0", + resolvedFrom: "1.0.0", + sourceFiles: { + metadata: "acme/acme-tarball.json", + script: "acme/acme-tarball.js", + }, + artifactUrl: tarballUrl, + artifactSha256: digest(tar), + artifactSignature: { + type: "sigstoreBundle", + bundlePath: "acme-tarball.tgz.sigstore.json", + bundleUrl, + }, + manifestSha256: digest(manifest), + scriptSha256: digest(script), + } + + const verified = [] + const options = { + fetchImpl: async url => { + if (url === tarballUrl) + return new Response(tar, { status: 200 }) + if (url === bundleUrl) + return new Response(JSON.stringify({ bundle: "legacy" }), { + status: 200, + }) + const repository = new URL(url).pathname + const owner = [signed, listed].find(f => + repository.startsWith(`/v2/${f.oci.oci.repository}/`) + ) + if (!owner) throw new Error(`Unexpected request: ${url}`) + return owner.options.fetchImpl(url) + }, + sigstoreVerifier: async (bundle, bytes, policy) => { + if (bundle.bundle === "legacy") { + expect(bytes).toEqual(tar) + verified.push("tarball") + return + } + let failure + for (const f of [signed, listed]) { + try { + await f.options.sigstoreVerifier(bundle, bytes, policy) + verified.push(f.oci.connectorId) + return + } catch (error) { + failure = error + } + } + throw failure + }, + } + const lock = { + lockVersion: "2.0", + dependencies: { + "acme-tarball": "1.0.0", + [signed.oci.connectorId]: "0.3.0", + [listed.oci.connectorId]: "0.3.0", + }, + connectors: [tarballEntry, signed.oci, listed.oci], + } + const installRoot = join(root, "installed") + + const result = await installConnectorsAtomically({ + lock, + installRoot, + ...options, + }) + + // Summed across both cores: 3 connectors, 2 tarball files + 4 files for + // the declaration-less release + 5 for the signed one. + expect(result.connectorCount).toBe(3) + const paths = tree(installRoot).map(([path]) => path) + expect(paths).toEqual([ + "acme/acme-tarball.js", + "acme/acme-tarball.json", + "collection-profiles/github-pdpp/dist/collection-profile.mjs", + "collection-profiles/github-pdpp/licenses/LICENSE", + "collection-profiles/github-pdpp/profile/collection-profile.json", + "collection-profiles/github-pdpp/provenance.json", + "collection-profiles/ynab-pdpp/dist/collection-profile.mjs", + "collection-profiles/ynab-pdpp/licenses/LICENSE", + "collection-profiles/ynab-pdpp/profile/collection-profile.json", + "collection-profiles/ynab-pdpp/provenance.json", + // Only the signed release carries a source declaration. + "collection-profiles/ynab-pdpp/source-declaration.json", + "lock.json", + ]) + expect(result.filesWritten).toBe(11) + expect(verified.sort()).toEqual( + ["github-pdpp", "tarball", "ynab-pdpp"].sort() + ) + expect( + checkInstalledLock({ + lock, + dependencies: { connectors: lock.dependencies }, + installRoot, + }).ok + ).toBe(true) + + // The same declaration-less release under an id the stopgap does not + // list is refused by the current core, and nothing is published. + const unlisted = fixture(join(root, "unlisted"), { + key: "gitlab", + withDeclaration: false, + }) + const before = tree(installRoot) + await expect( + installConnectorsAtomically({ + lock: { + lockVersion: "2.0", + dependencies: { [unlisted.oci.connectorId]: "0.3.0" }, + connectors: [unlisted.oci], + }, + installRoot, + ...unlisted.options, + }) + ).rejects.toThrow(/no signed source declaration layer or referrer/) + expect(tree(installRoot)).toEqual(before) + })) + it("B-T8 generate-platform-registry output is unchanged by a mixed v2 lock", async () => temporary(async root => { const f = fixture(root),