diff --git a/.github/workflows/binary-build.yml b/.github/workflows/binary-build.yml index 956a966d2..e014e296c 100644 --- a/.github/workflows/binary-build.yml +++ b/.github/workflows/binary-build.yml @@ -38,6 +38,7 @@ jobs: node_modules/onnxruntime-node/bin/napi-v3/darwin/arm64/onnxruntime_binding.node node_modules/onnxruntime-node/bin/napi-v3/darwin/arm64/libonnxruntime.1.21.0.dylib node_modules/sqlite-vec-darwin-arm64/vec0.dylib + node_modules/better-sqlite3/prebuilds/darwin-arm64.node node_modules/@napi-rs/keyring-darwin-arm64/keyring.darwin-arm64.node node_modules/wreq-js/rust/wreq-js.darwin-arm64.node node_modules/@anush008/tokenizers-darwin-universal/tokenizers.darwin-universal.node @@ -51,6 +52,7 @@ jobs: node_modules/onnxruntime-node/bin/napi-v3/darwin/x64/onnxruntime_binding.node node_modules/onnxruntime-node/bin/napi-v3/darwin/x64/libonnxruntime.1.21.0.dylib node_modules/sqlite-vec-darwin-x64/vec0.dylib + node_modules/better-sqlite3/prebuilds/darwin-x64.node node_modules/@napi-rs/keyring-darwin-x64/keyring.darwin-x64.node node_modules/wreq-js/rust/wreq-js.darwin-x64.node node_modules/@anush008/tokenizers-darwin-universal/tokenizers.darwin-universal.node @@ -124,9 +126,15 @@ jobs: const cfg = JSON.parse(fs.readFileSync(path.join(root, 'packaging/binary/pkg.config.json'), 'utf8')); // Keep only the platform-agnostic assets; drop the macOS-arm64 native // lines baked into the committed config, then add this platform's. + // ⚠ better-sqlite3 joined this list when the pin moved 12.9.0 -> 13.0.3. Under v12 it + // compiled to `build/Release/better_sqlite3.node` — the SAME path on every target — so + // it was genuinely platform-agnostic and belonged in `common`. v13 ships per-target + // prebuilds under `prebuilds/-.node`, so leaving it in `common` would + // bake the darwin-arm64 binary into every platform's binary. const common = cfg.pkg.assets.filter((a) => !a.includes('onnxruntime-node/bin/napi-v3/') && !a.startsWith('node_modules/sqlite-vec-') && + !a.startsWith('node_modules/better-sqlite3/prebuilds/') && !a.includes('/keyring') && !a.startsWith('node_modules/wreq-js/rust/') && !a.includes('/tokenizers') && diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ec959d2e6..96149005a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -247,9 +247,36 @@ jobs: matrix: # Every desktop OS across the supported Node range. The floor is Node 22 # (`engines.node: ">=22"`): Node 20 "Iron" went EOL upstream on - # 2026-03-24, so it is no longer built or tested here. 12.9.0 ships - # prebuilt native binaries for every row (win32-x64 + darwin/linux arm64 - # across Node 22/24), so npm ci never falls back to a source compile. + # 2026-03-24, so it is no longer built or tested here. better-sqlite3 + # 13.0.3 ships prebuilt native binaries for every row: v13's are + # Node-API, carry no ABI suffix, and all eight targets travel in the ONE + # tarball, so there is no per-platform artifact to resolve. The + # postinstall then drops the seven the host cannot load + # (scripts/prune/sqlite-prebuilds.mjs). + # + # ⚠ AND ON WINDOWS `npm ci` STILL COMPILES FROM SOURCE, WHICH IS WHY + # THIS PIN IS NOT MERGEABLE. An earlier version of this comment claimed + # v13 "never falls back to a source compile on any row". That is false + # and was written from a green macOS run; this matrix measured it. Every + # macOS and Linux row passes, every Windows row fails, at `npm ci`: + # npm error command C:\...\cmd.exe /d /s /c node-gyp rebuild + # npm error gyp ERR! find VS could not find a version of + # Visual Studio 2017 or newer to use + # v12 shipped an `install` script (prebuild-install) that ran in pure + # Node and exited 0 the moment it found the bundled binary, so node-gyp + # was never invoked. v13 removed it, and npm then supplies its own + # implicit `node-gyp rebuild` for any package carrying a binding.gyp. + # better-sqlite3 publishes `gypfile: false` precisely to suppress that, + # and npm ignores it (npm/cli#9837, WiseLibs/better-sqlite3#1503/#1505). + # binding.gyp's own `prebuild_exists` guard cannot save it either: gyp + # must CONFIGURE before it can evaluate any variable, and configure is + # the step that needs the toolchain. + # + # This is not a CI-image problem to paper over. It means a Windows user + # without Python and VC++ Build Tools cannot `npm i wigolo` at all, and + # that is exactly the failure this clean-machine job exists to catch. + # Revisit when npm/cli#9837 lands or better-sqlite3 restores a + # JS-only install script. os: [ubuntu-latest, macos-latest, windows-latest] node: ['22', '24'] include: diff --git a/package-lock.json b/package-lock.json index b7e00d585..38f296aa5 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22,7 +22,7 @@ "@modelcontextprotocol/sdk": "^1.29.0", "@mozilla/readability": "^0.6.0", "ajv": "8.17.1", - "better-sqlite3": "~12.9.0", + "better-sqlite3": "13.0.3", "chalk": "^5.6.2", "cli-progress": "^3.12.0", "defuddle": "^0.16.0", @@ -5396,17 +5396,15 @@ } }, "node_modules/better-sqlite3": { - "version": "12.9.0", - "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.9.0.tgz", - "integrity": "sha512-wqUv4Gm3toFpHDQmaKD4QhZm3g1DjUBI0yzS4UBl6lElUmXFYdTQmmEDpAFa5o8FiFiymURypEnfVHzILKaxqQ==", - "hasInstallScript": true, + "version": "13.0.3", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-13.0.3.tgz", + "integrity": "sha512-RbOBxmLBG8uvFUc15X9+9SFemKcQ0WBuISBVkpuiaUB2qblC8UWlHEjdWVoZ8AdhSwmoEgsiXKfopX0CQxaACQ==", "license": "MIT", "dependencies": { - "bindings": "^1.5.0", - "prebuild-install": "^7.1.1" + "node-addon-api": "^8.0.0" }, "engines": { - "node": "20.x || 22.x || 23.x || 24.x || 25.x" + "node": ">=22" } }, "node_modules/bignumber.js": { @@ -5418,19 +5416,11 @@ "node": "*" } }, - "node_modules/bindings": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", - "integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==", - "license": "MIT", - "dependencies": { - "file-uri-to-path": "1.0.0" - } - }, "node_modules/bl": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, "license": "MIT", "dependencies": { "buffer": "^5.5.0", @@ -5520,6 +5510,7 @@ "version": "5.7.1", "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, "funding": [ { "type": "github", @@ -5695,6 +5686,7 @@ "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", "integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==", + "dev": true, "license": "ISC" }, "node_modules/chrome-remote-interface": { @@ -6239,6 +6231,7 @@ "version": "6.0.0", "resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz", "integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==", + "dev": true, "license": "MIT", "dependencies": { "mimic-response": "^3.1.0" @@ -6254,6 +6247,7 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==", + "dev": true, "license": "MIT", "engines": { "node": ">=4.0.0" @@ -6545,6 +6539,7 @@ "version": "1.4.5", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", "integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==", + "dev": true, "license": "MIT", "dependencies": { "once": "^1.4.0" @@ -6761,6 +6756,7 @@ "version": "2.0.3", "resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz", "integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==", + "dev": true, "license": "(MIT OR WTFPL)", "engines": { "node": ">=6" @@ -6941,12 +6937,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/file-uri-to-path": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", - "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==", - "license": "MIT" - }, "node_modules/finalhandler": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", @@ -7020,6 +7010,7 @@ "version": "1.0.0", "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==", + "dev": true, "license": "MIT" }, "node_modules/fs-extra": { @@ -7191,6 +7182,7 @@ "version": "0.0.0", "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==", + "dev": true, "license": "MIT" }, "node_modules/global-agent": { @@ -7520,6 +7512,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", + "dev": true, "funding": [ { "type": "github", @@ -7558,6 +7551,7 @@ "version": "1.3.8", "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz", "integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==", + "dev": true, "license": "ISC" }, "node_modules/ink": { @@ -8526,6 +8520,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz", "integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=10" @@ -8538,6 +8533,7 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", "integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==", + "dev": true, "license": "MIT", "funding": { "url": "https://github.com/sponsors/ljharb" @@ -8593,6 +8589,7 @@ "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", "integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==", + "dev": true, "license": "MIT" }, "node_modules/mlly": { @@ -8683,6 +8680,7 @@ "version": "2.0.0", "resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz", "integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==", + "dev": true, "license": "MIT" }, "node_modules/negotiator": { @@ -8698,6 +8696,7 @@ "version": "3.92.0", "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.92.0.tgz", "integrity": "sha512-KdHvFWZjEKDf0cakgFjebl371GPsISX2oZHcuyKqM7DtogIsHrqKeLTo8wBHxaXRAQlY2PsPlZmfo+9ZCxEREQ==", + "dev": true, "license": "MIT", "dependencies": { "semver": "^7.3.5" @@ -8706,6 +8705,15 @@ "node": ">=10" } }, + "node_modules/node-addon-api": { + "version": "8.9.2", + "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.2.tgz", + "integrity": "sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg==", + "license": "MIT", + "engines": { + "node": "^18 || ^20 || >= 21" + } + }, "node_modules/node-domexception": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", @@ -9424,6 +9432,7 @@ "resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz", "integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==", "deprecated": "No longer maintained. Please contact the author of the relevant native addon; alternatives are available.", + "dev": true, "license": "MIT", "dependencies": { "detect-libc": "^2.0.0", @@ -9514,6 +9523,7 @@ "version": "3.0.4", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.4.tgz", "integrity": "sha512-VS7sjc6KR7e1ukRFhQSY5LM2uBWAUPiOPa/A3mkKmiMwSmRFUITt0xuj+/lesgnCv+dPIEYlkzrcyXgquIHMcA==", + "dev": true, "license": "MIT", "dependencies": { "end-of-stream": "^1.1.0", @@ -9573,6 +9583,7 @@ "version": "1.2.8", "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", + "dev": true, "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", "dependencies": { "deep-extend": "^0.6.0", @@ -9622,6 +9633,7 @@ "version": "3.6.2", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz", "integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==", + "dev": true, "license": "MIT", "dependencies": { "inherits": "^2.0.3", @@ -10168,6 +10180,7 @@ "version": "1.0.1", "resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz", "integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==", + "dev": true, "funding": [ { "type": "github", @@ -10188,6 +10201,7 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz", "integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==", + "dev": true, "funding": [ { "type": "github", @@ -10450,6 +10464,7 @@ "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, "license": "MIT", "dependencies": { "safe-buffer": "~5.2.0" @@ -10491,6 +10506,7 @@ "version": "2.0.1", "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", + "dev": true, "license": "MIT", "engines": { "node": ">=0.10.0" @@ -10599,6 +10615,7 @@ "version": "2.1.4", "resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz", "integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==", + "dev": true, "license": "MIT", "dependencies": { "chownr": "^1.1.1", @@ -10611,6 +10628,7 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz", "integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==", + "dev": true, "license": "MIT", "dependencies": { "bl": "^4.0.3", @@ -11444,6 +11462,7 @@ "version": "0.6.0", "resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz", "integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==", + "dev": true, "license": "Apache-2.0", "dependencies": { "safe-buffer": "^5.0.1" @@ -11619,6 +11638,7 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==", + "dev": true, "license": "MIT" }, "node_modules/vary": { diff --git a/package.json b/package.json index 837c292f6..18442bccf 100644 --- a/package.json +++ b/package.json @@ -126,7 +126,7 @@ "@modelcontextprotocol/sdk": "^1.29.0", "@mozilla/readability": "^0.6.0", "ajv": "8.17.1", - "better-sqlite3": "~12.9.0", + "better-sqlite3": "13.0.3", "chalk": "^5.6.2", "cli-progress": "^3.12.0", "defuddle": "^0.16.0", diff --git a/packaging/binary/pkg.config.json b/packaging/binary/pkg.config.json index 30bf2a0b1..975474314 100644 --- a/packaging/binary/pkg.config.json +++ b/packaging/binary/pkg.config.json @@ -1,5 +1,5 @@ { - "//": "pkg config for the wigolo single-file binary. Copied to the repo root as pkg.build.json at build time so all globs resolve relative to the repo root (pkg resolves scripts/assets relative to the CONFIG file's directory — round-1 finding). The entry fed to pkg is the esbuild CJS bundle at dist/cli/agents/wigolo.bundle.cjs; `scripts` stays empty because the bundle is self-contained JS. Native .node/.dylib and asset-carrier packages are declared here so pkg's require-hook can extract them at runtime.", + "//": "pkg config for the wigolo single-file binary. Copied to the repo root as pkg.build.json at build time so all globs resolve relative to the repo root (pkg resolves scripts/assets relative to the CONFIG file's directory — round-1 finding). The entry fed to pkg is the esbuild CJS bundle at dist/cli/agents/wigolo.bundle.cjs; `scripts` stays empty because the bundle is self-contained JS. Native .node/.dylib and asset-carrier packages are declared here so pkg's require-hook can extract them at runtime. NOTE the better-sqlite3 line is PLATFORM-SPECIFIC and was not always: v12 compiled to a fixed `build/Release/better_sqlite3.node`, the same path on every target, so it sat with the platform-agnostic entries and binary-build.yml's composer left it alone. v13 ships per-target Node-API prebuilds under `prebuilds/-.node` and the loader computes that name at RUNTIME, so pkg cannot discover it by static analysis and it must be declared per target — this file carries the darwin-arm64 one and the workflow strips it for the other rows exactly as it does the other natives.", "pkg": { "scripts": [], "assets": [ @@ -17,7 +17,7 @@ "node_modules/playwright-core/browsers.json", "node_modules/playwright-core/cli.js", "node_modules/playwright-core/**/*.js", - "node_modules/better-sqlite3/build/Release/better_sqlite3.node", + "node_modules/better-sqlite3/prebuilds/darwin-arm64.node", "node_modules/onnxruntime-node/bin/napi-v3/darwin/arm64/onnxruntime_binding.node", "node_modules/onnxruntime-node/bin/napi-v3/darwin/arm64/libonnxruntime.1.21.0.dylib", "node_modules/sqlite-vec-darwin-arm64/vec0.dylib", diff --git a/scripts/budget/protocol.mjs b/scripts/budget/protocol.mjs index 85c4ccf45..7d41b9533 100644 --- a/scripts/budget/protocol.mjs +++ b/scripts/budget/protocol.mjs @@ -160,7 +160,7 @@ export const GATES = { title: 'production node_modules on disk', what: 'total bytes of the dependency tree a `npm i -g wigolo` user installs', artifact: - 'a fresh `npm install --omit=dev --ignore-scripts --no-workspaces` into an empty directory holding only package.json + package-lock.json, with PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 and ELECTRON_SKIP_BINARY_DOWNLOAD=1, followed by `scripts/prune/run.mjs` — the postinstall that `--ignore-scripts` suppresses, and which now performs ALL THREE of the onnxruntime-node platform prune, the onnxruntime-web payload prune and the wreq-js binary prune', + 'a fresh `npm install --omit=dev --ignore-scripts --no-workspaces` into an empty directory holding only package.json + package-lock.json, with PLAYWRIGHT_SKIP_BROWSER_DOWNLOAD=1 and ELECTRON_SKIP_BINARY_DOWNLOAD=1, followed by `scripts/prune/run.mjs` — the postinstall that `--ignore-scripts` suppresses, and which now performs ALL FOUR of the onnxruntime-node platform prune, the onnxruntime-web payload prune, the wreq-js binary prune and the better-sqlite3 prebuild prune', statistic: 'single `du -sm node_modules` total', horizon: 'n/a — a completed install is at rest', runs: 1, @@ -168,7 +168,7 @@ export const GATES = { comparison: '<=', limit: 383, baseline: - 'THE RUNNER IS THE AUTHORITY, because this gate runs on exactly one machine class: the `clean-machine-smoke` macos-latest / node 22 arm. Measured there on the S10-e PR across two runs of the same branch: 685 then 683 MiB clean — so the runner carries ~2 MiB of run-to-run variance of its own. CI has since read pre-prune 683 / post-prune 419 for the previous slice, which is the anchor this slice moves. ⚠ THIS SLICE MOVES THE ANCHOR BY A MEASURED DELTA RATHER THAN RE-DERIVING IT, which is the conservative operation: locally on darwin-arm64, on exactly the protocol the gate runs, THREE runs before and THREE after. Before: raw 683/681/682, post-prune 419/418/418. After: raw 682/682/682, post-prune 372/372/372 — dead flat across all three, and `wreq-js` itself goes 54 -> 8 MiB every time. The delta is 46 MiB (median 418 -> 372) and 421 - 46 = 375 is the new clean-build anchor. ⚠ NOTE THE LAPTOP HAS DRIFTED +1 SINCE THE PREVIOUS SLICE RECORDED 417 HERE — it now reads 418 for the same tree — which is exactly why the anchor is the previous runner-derived 421 minus a measured delta, not this laptop\'s 372 plus an offset. The delta is what reproduced; the absolute is what drifted. ⚠ PLATFORM-CONDITIONAL, AND MORE SO THAN THE ONNXRUNTIME PLATFORM PRUNE: the 46 MiB is the darwin/win32 figure, where exactly one binary survives. On linux BOTH the gnu and musl builds are kept deliberately (install-time libc detection cannot bind a run-time selection — see scripts/prune/wreq-binaries.mjs), so the saving there is ~38 MiB, not 46. This gate is wired on macOS only, so 375 is the darwin number; whoever wires linux must re-measure rather than carry this across. THE MANIFEST COULD NOT DO THIS: wreq-js publishes no platform-scoped subpackages at any napi naming convention (@wreq-js/darwin-arm64 and wreq-js-darwin-arm64 are both registry 404s), ships all seven binaries in one tarball via `files: ["dist","rust/*.node"]`, and declares os/cpu as the union of everything it supports, so npm filtering never excludes a byte. It was ALREADY in optionalDependencies, which installs in full.', + 'THE RUNNER IS THE AUTHORITY, because this gate runs on exactly one machine class: the `clean-machine-smoke` macos-latest / node 22 arm. Measured there on the S10-e PR across two runs of the same branch: 685 then 683 MiB clean — so the runner carries ~2 MiB of run-to-run variance of its own. CI has since read pre-prune 683 / post-prune 419 for the previous slice, which is the anchor this slice moves. ⚠ THIS SLICE MOVES THE ANCHOR BY A MEASURED DELTA RATHER THAN RE-DERIVING IT, which is the conservative operation: locally on darwin-arm64, on exactly the protocol the gate runs, THREE runs before and THREE after. Before: raw 683/681/682, post-prune 419/418/418. After: raw 682/682/682, post-prune 372/372/372 — dead flat across all three, and `wreq-js` itself goes 54 -> 8 MiB every time. The delta is 46 MiB (median 418 -> 372) and 421 - 46 = 375 is the new clean-build anchor. ⚠ NOTE THE LAPTOP HAS DRIFTED +1 SINCE THE PREVIOUS SLICE RECORDED 417 HERE — it now reads 418 for the same tree — which is exactly why the anchor is the previous runner-derived 421 minus a measured delta, not this laptop\'s 372 plus an offset. The delta is what reproduced; the absolute is what drifted. ⚠ PLATFORM-CONDITIONAL, AND MORE SO THAN THE ONNXRUNTIME PLATFORM PRUNE: the 46 MiB is the darwin/win32 figure, where exactly one binary survives. On linux BOTH the gnu and musl builds are kept deliberately (install-time libc detection cannot bind a run-time selection — see scripts/prune/wreq-binaries.mjs), so the saving there is ~38 MiB, not 46. This gate is wired on macOS only, so 375 is the darwin number; whoever wires linux must re-measure rather than carry this across. THE MANIFEST COULD NOT DO THIS: wreq-js publishes no platform-scoped subpackages at any napi naming convention (@wreq-js/darwin-arm64 and wreq-js-darwin-arm64 are both registry 404s), ships all seven binaries in one tarball via `files: ["dist","rust/*.node"]`, and declares os/cpu as the union of everything it supports, so npm filtering never excludes a byte. It was ALREADY in optionalDependencies, which installs in full. ⚠ ANCHOR MOVED 375 -> 377 by the better-sqlite3 13.0.3 pin (see regression E in the threshold note): the bump adds 16 MiB of all-platform prebuilds and the new prune takes 14 of them back, for +2 net. Measured as a back-to-back PAIR on darwin-arm64 at ee346f77 — 372 at the old pin, 374 at the new pin with the prune — because the absolute drifts a MiB between batches on this laptop while the delta reproduces; an earlier batch at cd9aa425 read 371 -> 372 for the same operation, and 387 for the bump WITHOUT the prune. The runner remains the authority for the absolute, so 377 is still the previous runner-derived anchor plus a measured delta rather than a laptop reading.', // ⚠ THE THRESHOLD IS SET BY THE SMALLEST REGRESSION IT MUST CATCH, NOT BY A PERCENTAGE. // // This gate now guards FOUR reversions, spanning an order of magnitude: @@ -177,18 +177,41 @@ export const GATES = { // it silently no-op. Worth 178 MiB, landing at 375 + 178 = 553. // B. the browser driver returns to `dependencies` from the optional PEER dependency S10-e // moved it to. Worth exactly 17 MiB (`playwright` + `playwright-core`), landing at - // 375 + 17 = 392. + // 377 + 17 = 394. // C. the onnxruntime-web payload prune stops happening — lost the same way A would be. - // Worth 86 MiB, landing at 375 + 86 = 461. - // D. the wreq-js binary prune stops happening — this slice's win, and it shares the same - // single point of failure as A and C. Worth 46 MiB, landing at 375 + 46 = 421. + // Worth 86 MiB, landing at 377 + 86 = 463. + // D. the wreq-js binary prune stops happening — it shares the same single point of failure + // as A and C. Worth 46 MiB, landing at 377 + 46 = 423. + // E. the better-sqlite3 prebuild prune stops happening — same single point of failure again. + // Worth 14 MiB, landing at 377 + 14 = 391. // - // ⚠ B STILL BINDS, AND NONE OF A, C OR D DOES. That is the whole derivation, and it is the - // same answer as the last three slices for the same reason: a limit only has to be under 553 - // to catch A, under 461 to catch C and under 421 to catch D, but it has to be under 392 to - // catch B. The window is 375..392, still exactly 17 MiB wide — it has not widened once across - // four diet slices, because it is set by the SMALLEST regression and that regression has - // never been the one being won. + // ⚠ E BINDS, AND B NO LONGER DOES. That is the first time in five diet slices the binding + // regression has changed identity, so it is spelled out rather than left to be re-derived: a + // limit only has to be under 553/463/423 to catch A/C/D and under 394 to catch B, but it now + // has to be under 391 to catch E. The window is 377..391, 14 MiB wide where it was 17, and it + // narrowed because the newest win is the SMALLEST one — the opposite of the pattern the four + // previous slices established, every one of which left B binding. + // + // ⚠ WHERE E CAME FROM. The `better-sqlite3` pin moved `~12.9.0` -> `13.0.3` (the desktop shell + // must load the same native module the core does, and only v13's Node-API prebuilds span both + // ABIs). v12 was a source build with a single ~1.8 MiB binary and nothing foreign to prune; + // v13 ships all eight targets in one tarball, 16 MiB of `prebuilds/`. Measured on the gate's + // own protocol, darwin-arm64, at cd9aa425: 371 at the old pin, 387 at the new pin UNPRUNED — + // a FAIL against this 383 — and 372 with the prune. + // + // ⚠ THE ANCHOR MOVES BY A MEASURED DELTA, 375 -> 377, and the delta is the +2 from a + // back-to-back PAIR at ee346f77 (372 at the old pin, 374 with the bump and the prune), not the + // +1 the cd9aa425 batch suggested. Same operation the wreq-js slice used and for the same + // reason: across those two batches the ABSOLUTE drifted a MiB while the delta reproduced, so + // the delta is the thing to carry. Taking the +1 would have put E at 390 and B at 393 and left + // the next reader a derivation that does not reproduce. + // + // ⚠ AND THE LIMIT DELIBERATELY DID NOT MOVE. 387 unpruned would have needed a limit of at + // least 388, which is ABOVE the 391 E lands on and within 6 of the 394 B lands on — i.e. + // accommodating the unpruned bump would not have loosened this gate, it would have blinded it + // to the smallest regression it is chartered to catch, and E is a regression this very slice + // introduced the possibility of. That is the whole reason the prune ships WITH the pin change + // and not as a follow-up: without it there is no honest threshold available. // // ⚠ THE TRAP THAT IS LIVE AT THESE NUMBERS. Re-checked rather than inherited, because pinning // a trap that has gone safe is just a false claim in a test file — which is how the 3% form diff --git a/scripts/prune/run.mjs b/scripts/prune/run.mjs index 40dc0d090..6feff19fb 100644 --- a/scripts/prune/run.mjs +++ b/scripts/prune/run.mjs @@ -16,6 +16,7 @@ import { fileURLToPath } from 'node:url'; import { planPlatformPrune, locateOrtRoots, findOutermostInstallRoot } from './ort-platforms.mjs'; import { planWebPayloadPrune, findWebDependents } from './ort-web-payload.mjs'; import { planBinaryPrune } from './wreq-binaries.mjs'; +import { planPrebuildPrune } from './sqlite-prebuilds.mjs'; /* * Resolution base. As a postinstall this is the package's own directory, which is what walks up @@ -292,6 +293,50 @@ function pruneWreqBinaries() { console.log(`wigolo: wreq-js binary prune — ${plan.reason} (${Math.round(freed / 1048576)} MiB)`); } +/** + * Remove the `better-sqlite3` prebuilds the host can never load. + * + * See ./sqlite-prebuilds.mjs for why v13 ships eight of them, why linux keeps two, and why + * removing the wrong one breaks the install rather than shrinking it. This function is only the + * I/O around that decision. + */ +function pruneSqlitePrebuilds() { + const root = locatePackageRoot(dirname(baseFile), 'better-sqlite3'); + if (!root) return; // not installed, or a layout we do not recognise + + const prebuildDir = join(root, 'prebuilds'); + let present; + try { + present = readdirSync(prebuildDir, { withFileTypes: true }) + .filter((e) => e.isFile() || e.isSymbolicLink()) + .map((e) => e.name); + } catch { + // No `prebuilds/` at all. That is what a source build looks like (v12 and earlier, or a host + // that compiled from source), and it is a tree with nothing foreign in it — not an error. + return; + } + + const plan = planPrebuildPrune(present, process.platform, process.arch); + if (plan.remove.length === 0) { + console.log(`wigolo: better-sqlite3 prebuild prune — ${plan.reason}`); + return; + } + + let freed = 0; + for (const name of plan.remove) { + const target = join(prebuildDir, name); + try { + freed += statSync(target).size; + rmSync(target, { force: true }); + } catch (err) { + // Fail-open, same contract as every other prune here: a file we could not remove is a file + // that stays. Larger, works. + console.log(`wigolo: could not remove better-sqlite3/prebuilds/${name} (${err?.message ?? err}) — leaving it in place`); + } + } + console.log(`wigolo: better-sqlite3 prebuild prune — ${plan.reason} (${Math.round(freed / 1048576)} MiB)`); +} + function main() { if (process.env.WIGOLO_SKIP_ORT_PRUNE) { console.log('wigolo: install-size prunes skipped (WIGOLO_SKIP_ORT_PRUNE set)'); @@ -312,6 +357,12 @@ function main() { console.log(`wigolo: wreq-js binary prune skipped (${err?.message ?? err})`); } + try { + pruneSqlitePrebuilds(); + } catch (err) { + console.log(`wigolo: better-sqlite3 prebuild prune skipped (${err?.message ?? err})`); + } + const scanFrom = dirname(baseFile); const roots = locateOrtRoots(resolveOrtRoot, scanFrom); if (roots.length === 0) { diff --git a/scripts/prune/sqlite-prebuilds.mjs b/scripts/prune/sqlite-prebuilds.mjs new file mode 100644 index 000000000..335a7a310 --- /dev/null +++ b/scripts/prune/sqlite-prebuilds.mjs @@ -0,0 +1,139 @@ +/* + * Drop the non-host `better-sqlite3` prebuilds. + * + * ⚠⚠ THIS MODULE IS NOT LIVE, AND THE PIN IT SERVES DID NOT MERGE. It is kept as a worked record + * so the next attempt does not rediscover it. `better-sqlite3@13` cannot currently be installed on + * Windows without Python and VC++ Build Tools: v13 dropped the `install` script (pure-Node + * prebuild-install, which exited 0 as soon as it found the bundled binary), npm then supplies its + * own implicit `node-gyp rebuild` for any package shipping a binding.gyp, and it does so despite + * the `gypfile: false` that better-sqlite3 publishes to suppress exactly that — npm/cli#9837, + * WiseLibs/better-sqlite3#1503 and #1505. binding.gyp's `prebuild_exists` guard does not help: + * gyp must CONFIGURE before it can evaluate any variable, and configure is the step that needs the + * toolchain. Measured on CI: every macOS and Linux row green, every Windows row red at `npm ci`. + * Everything below is correct and tested and does nothing while the pin stays on v12, whose + * source build produces no `prebuilds/` directory at all. + * + * WHY THIS EXISTS, AND WHY IT DID NOT BEFORE. `better-sqlite3` used to be a source build: + * `~12.9.0` compiled through node-gyp (or fetched ONE prebuild via prebuild-install) and left a + * single `build/Release/better_sqlite3.node`, ~1.8 MiB, with nothing foreign to remove. v13 moved + * to Node-API prebuilds and ships ALL EIGHT targets in the one tarball — 16 MiB of `prebuilds/`, + * of which at most one (two on linux, see below) can ever be loaded. That is the trade v13 makes: + * the all-platforms tarball is exactly what makes it ABI-portable across Node and the desktop + * shell without a rebuild step, and exactly what makes it fat per-platform. + * + * WHY IT CANNOT BE FIXED IN THE MANIFEST. `better-sqlite3` publishes no platform-scoped + * subpackages — there is no `@better-sqlite3/darwin-arm64` to depend on selectively, the way + * `sharp` arranges it. Its `files` field ships `prebuilds/` whole, and it declares no `os`/`cpu` + * narrowing that npm could filter on. The version pin is the only manifest lever and it is the + * wrong one: v12's single binary is V8-ABI-bound and cannot be loaded by the desktop shell, which + * is the entire reason the pin moved to v13. + * + * SELECTION IS A PURE RUNTIME CONCERN, and the loader (`lib/binding.js`) does it by computing ONE + * exact filename — `prebuilds/${isLinuxMusl() ? 'linuxmusl' : platform}-${arch}.node` — and + * `fs.existsSync`ing it. Nothing enumerates the directory, so a sibling that is absent is a + * sibling nothing looks for. + * + * ⚠ WHY THE REFUSAL IS NOT OPTIONAL HERE, and why it matters more than it did for `wreq-js`. + * When `getPrebuildPath()` finds nothing, the loader does NOT degrade — it falls through to + * `build/Debug` then `build/Release` and `require`s the latter unconditionally. Under v13 those + * directories do not exist, so a tree whose host prebuild was removed does not lose a capability, + * it throws MODULE_NOT_FOUND on the cache database and takes the whole tool down. Removing the + * wrong file here is therefore not a degradation, it is a broken install, so the planner refuses + * to remove anything at all unless it has positively identified a prebuild this host can load. + * + * ⚠ WHY LINUX KEEPS BOTH LIBC BUILDS. The loader's `isLinuxMusl()` reads + * `process.report.getReport().header.glibcVersionRuntime` — a genuine runtime probe, which is + * better than the env-var sniff `wreq-js` uses. But this script still runs at INSTALL time, in a + * different process from the one that will load the binary, and the two can legitimately disagree + * (a glibc build stage populating a tree that a musl stage runs, which is an ordinary multi-stage + * container layout). Keeping both `linux-` and `linuxmusl-` removes that entire class + * instead of predicting it, at a cost of ~2 MiB on linux only. darwin and win32 have no libc + * dimension and keep exactly one file. Same reasoning, and the same conclusion, as + * ./wreq-binaries.mjs. + * + * ⚠ AND WHY REMOVAL IS ALLOWLISTED. The names are `${platform}-${arch}.node` with a `linuxmusl` + * special case, so an interpolation of `process.*` very nearly produces them — which is precisely + * the trap. `linuxmusl` is not a `process.platform` any host reports, so a pattern-based prune + * that trusted interpolation would delete the musl builds on every glibc machine and the glibc + * builds on every musl one. Anything not on the eight-name list the package actually ships is left + * alone: that covers any target a future release adds, whose bytes cost less than its absence. + * + * WHAT THIS COSTS. The pruned tree is bound to the platform and architecture that installed it — a + * property the tree already had, via `sharp`, `wreq-js` and the onnxruntime prune. + * `WIGOLO_SKIP_ORT_PRUNE=1` skips every prune in this directory, including this one, for anyone + * deliberately building a multi-arch tree. + * + * FAIL-OPEN, ALWAYS. If no prebuild the host could load is present the planner refuses to remove + * anything. If a removal throws it is reported and skipped. The driver never exits non-zero. + */ + +/** + * Every prebuild `better-sqlite3@13.0.3` ships, matching `prebuilds/` in the published tarball and + * the targets `lib/binding.js` can compute. This is an ALLOWLIST of what may be removed, never a + * pattern — see the note above on why interpolation is unsafe for exactly these names. + */ +export const SQLITE_PREBUILDS = Object.freeze([ + 'darwin-arm64.node', + 'darwin-x64.node', + 'linux-arm64.node', + 'linux-x64.node', + 'linuxmusl-arm64.node', + 'linuxmusl-x64.node', + 'win32-arm64.node', + 'win32-x64.node', +]); + +/** + * The prebuilds `platform`/`arch` could ever load, in the loader's own spelling. + * + * Linux maps to BOTH libc builds on purpose — see the note above. A host outside the loader's own + * `PREBUILD_PLATFORMS` × `PREBUILD_ARCHS` maps to the empty list, which is what drives the + * planner's refusal: on such a host the package can only work via a source build, and a tree we + * cannot prove has a usable prebuild is a tree we must not touch. + */ +function hostCandidates(platform, arch) { + if (arch !== 'x64' && arch !== 'arm64') return []; + if (platform === 'darwin' || platform === 'win32') return [`${platform}-${arch}.node`]; + if (platform === 'linux') return [`linux-${arch}.node`, `linuxmusl-${arch}.node`]; + return []; +} + +/** + * Decide which `prebuilds/*.node` files may be removed. + * + * Pure, and separate from the filesystem on purpose — this is the part with a decision in it, so + * it is the part that has to be testable without an npm install to run it against. + * + * @param present {string[]} basenames found in the package's `prebuilds/` directory + * @param platform host `process.platform` + * @param arch host `process.arch` + * @returns {{ keep: string[], remove: string[], reason: string }} + */ +export function planPrebuildPrune(present, platform, arch) { + const have = new Set(present); + const keep = hostCandidates(platform, arch).filter((f) => have.has(f)); + + // ⚠ The refusal, and the reason it is first. With nothing loadable identified we cannot tell + // "already pruned past recognition" from "this package is laid out differently than we believe", + // and in the second case every candidate for removal might be the one the loader wanted. Unlike + // the other prunes in this directory, guessing wrong here does not cost bytes or a capability — + // it throws on `require` and the cache database never opens. + if (keep.length === 0) { + return { + keep: [], + remove: [], + reason: `no loadable prebuild identified for ${platform}/${arch} — refusing to prune`, + }; + } + + const kept = new Set(keep); + const remove = present.filter((f) => SQLITE_PREBUILDS.includes(f) && !kept.has(f)).sort(); + + return { + keep, + remove, + reason: remove.length + ? `keeping ${keep.join(' + ')}, removing ${remove.length} non-host prebuild(s)` + : `only ${keep.join(' + ')} present — nothing to do`, + }; +} diff --git a/src/daemon/studio-mcp-server.ts b/src/daemon/studio-mcp-server.ts index 8b2d48434..ff294e122 100644 --- a/src/daemon/studio-mcp-server.ts +++ b/src/daemon/studio-mcp-server.ts @@ -9,11 +9,19 @@ import { createStudioToolProvider } from '../studio/tool-provider.js'; * A MINIMAL MCP server hosting ONLY the `studio_*` tools, for the Electron app's embedded gateway. * * WHY separate from `createMcpServer` (server.ts): server.ts pulls the full wigolo subsystem graph - * (cache → better-sqlite3), which CANNOT load in the Electron main — Electron 43's V8 rejects - * better-sqlite3 12.9.0 (spec §13.7). This module imports ONLY the SDK + the studio tool schemas + - * `dispatchStudioTool` (all verified better-sqlite3-free), so it boots in-process on any Electron. - * The 10 core tools stay on the user's stdio server; the stdio proxy forwards `studio_*` here. - * Cache-backed studio features (capture / knowledge rail) arrive in P3 behind a decoupled DB path. + * (cache → better-sqlite3), and this module imports ONLY the SDK + the studio tool schemas + + * `dispatchStudioTool`, so it boots in-process without that graph. The 10 core tools stay on the + * user's stdio server; the stdio proxy forwards `studio_*` here. Cache-backed studio features + * (capture / knowledge rail) go through the decoupled DB path. + * + * ⚠ THE ORIGINAL REASON NO LONGER HOLDS, and is corrected here rather than left to mislead. This + * said the graph "CANNOT load in the Electron main — Electron 43's V8 rejects better-sqlite3 + * 12.9.0 (spec §13.7)". That was true of the 12.9.0 pin, which is V8-ABI-bound and fails with + * NODE_MODULE_VERSION 127 vs 148. The pin is now 13.0.3, whose Node-API prebuilds load in a real + * Electron 43 main — measured, in both the main and renderer processes, with FTS5 and the vector + * extension working and no rebuild step. So the ABI wall is gone and the separation now rests on + * what it costs to boot, not on what can be loaded. Keeping a falsified cause in a comment is how + * the next reader spends a day proving something that is already known. * * The tool set + schemas + descriptions come from the SAME ToolProvider the stdio server registers * (one source of truth, derived from the tool schemas — no third literal list), so the agent sees an diff --git a/tests/unit/sqlite-prebuild-prune.test.ts b/tests/unit/sqlite-prebuild-prune.test.ts new file mode 100644 index 000000000..1be6e97bc --- /dev/null +++ b/tests/unit/sqlite-prebuild-prune.test.ts @@ -0,0 +1,359 @@ +import { execFileSync } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { dirname, join } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { afterAll, describe, expect, it } from 'vitest'; +// @ts-expect-error — plain-JS build tooling, deliberately not part of the typed src/ graph. +import { planPrebuildPrune, SQLITE_PREBUILDS } from '../../scripts/prune/sqlite-prebuilds.mjs'; + +/* + * WHY these tests exist. + * + * `better-sqlite3@13` ships all eight Node-API prebuilds in ONE tarball — 16 MiB of + * `prebuilds/.node`, of which at most one (two on linux) can ever be loaded. The prune + * deletes the rest out of a user's install. + * + * ⚠ AND THE COST OF GETTING IT WRONG IS HIGHER HERE THAN FOR ANY OTHER PRUNE IN THIS DIRECTORY. + * The other three guard optional capabilities: lose the wrong onnxruntime binary and embeddings + * degrade, lose the wrong wreq-js binary and the TLS tier goes quiet. `better-sqlite3` is the + * cache database — it is not optional, and its loader does not degrade. When + * `getPrebuildPath()` finds nothing, `lib/binding.js` falls through to `build/Release`, which v13 + * does not ship, and `require` throws. So deleting the host's prebuild does not make the install + * smaller-but-working, it makes it broken, AFTER npm has reported success. Every assertion below + * is about a way the decision could produce that. + * + * The size measurement that motivates it is recorded on the G-DIET gate; a unit test cannot + * perform an npm install. What IS testable is the decision. + * + * The real eight-prebuild layout of better-sqlite3@13.0.3, as shipped. + */ +const FULL_TREE = [ + 'darwin-arm64.node', + 'darwin-x64.node', + 'linux-arm64.node', + 'linux-x64.node', + 'linuxmusl-arm64.node', + 'linuxmusl-x64.node', + 'win32-arm64.node', + 'win32-x64.node', +]; + +/** Every host the package's own loader computes a prebuild path for. */ +const SUPPORTED_HOSTS: Array<[string, string, string[]]> = [ + ['darwin', 'arm64', ['darwin-arm64.node']], + ['darwin', 'x64', ['darwin-x64.node']], + ['win32', 'arm64', ['win32-arm64.node']], + ['win32', 'x64', ['win32-x64.node']], + ['linux', 'x64', ['linux-x64.node', 'linuxmusl-x64.node']], + ['linux', 'arm64', ['linux-arm64.node', 'linuxmusl-arm64.node']], +]; + +describe('planPrebuildPrune keeps exactly the prebuilds that can be loaded', () => { + it('keeps only the host prebuild on darwin-arm64 and removes the other seven', () => { + const plan = planPrebuildPrune(FULL_TREE, 'darwin', 'arm64'); + expect(plan.keep).toEqual(['darwin-arm64.node']); + expect(plan.remove).toEqual([ + 'darwin-x64.node', + 'linux-arm64.node', + 'linux-x64.node', + 'linuxmusl-arm64.node', + 'linuxmusl-x64.node', + 'win32-arm64.node', + 'win32-x64.node', + ]); + }); + + it('removes the SAME-platform other architecture too', () => { + // The tempting shortcut is to prune by platform name, which would leave darwin-x64 behind — + // ~1.9 MiB on an arm64 machine that nothing there can ever load. The loader's target is + // `${platform}-${arch}`, so selection is per-TARGET. + expect(planPrebuildPrune(FULL_TREE, 'darwin', 'arm64').remove).toContain('darwin-x64.node'); + }); + + it('never proposes removing a prebuild the host could load, from any supported host', () => { + for (const [platform, arch, expected] of SUPPORTED_HOSTS) { + const plan = planPrebuildPrune(FULL_TREE, platform, arch); + expect(plan.keep, `${platform}/${arch}`).toEqual(expected); + for (const kept of expected) { + expect(plan.remove, `${platform}/${arch}`).not.toContain(kept); + } + expect(plan.remove, `${platform}/${arch}`).toHaveLength(FULL_TREE.length - expected.length); + } + }); + + it('keeps win32-arm64 — a target the OTHER native prunes in this repo have no equivalent of', () => { + // ⚠ Not decoration. `wreq-js` ships no win32-arm64 binary, so the sibling prune's host table + // has no such row, and a reader porting that table across would silently produce an empty + // keep-set here — which is the refusal branch, i.e. the whole win quietly lost on Windows ARM. + // better-sqlite3 DOES ship it, so it must be kept, and it must not be confused with x64. + const plan = planPrebuildPrune(FULL_TREE, 'win32', 'arm64'); + expect(plan.keep).toEqual(['win32-arm64.node']); + expect(plan.remove).toContain('win32-x64.node'); + }); +}); + +describe('planPrebuildPrune keeps BOTH libc builds on linux', () => { + it('keeps the glibc AND musl builds for the host arch', () => { + // ⚠ THE DELIBERATE ~2 MiB LEFT ON THE FLOOR ON LINUX. + // + // The loader's `isLinuxMusl()` reads `process.report.getReport().header.glibcVersionRuntime`, + // which is a genuine runtime probe — better than the env-var sniff `wreq-js` uses, so the + // reasoning is NOT simply inherited. It still runs in a different PROCESS from this script: + // the prune happens at install time, and a multi-stage container that populates a tree under + // glibc and runs it under musl is an ordinary layout, not a contrivance. An install-time + // answer must not bind a run-time selection, so both stay. + expect(planPrebuildPrune(FULL_TREE, 'linux', 'x64').keep).toEqual([ + 'linux-x64.node', + 'linuxmusl-x64.node', + ]); + }); + + it('still removes the OTHER arch entirely on linux', () => { + // Keeping both libc builds must not soften into keeping all of linux — arch is decided by + // `process.arch`, which does not vary between install and run. + const plan = planPrebuildPrune(FULL_TREE, 'linux', 'x64'); + expect(plan.remove).toContain('linux-arm64.node'); + expect(plan.remove).toContain('linuxmusl-arm64.node'); + }); + + it('keeps the one that IS there when upstream ships only a single libc build', () => { + // Fail-open: a shrinking upstream matrix must not turn into a refusal that costs the win, nor + // into removing the only loadable file. + const gnuOnly = FULL_TREE.filter((f) => f !== 'linuxmusl-x64.node'); + const plan = planPrebuildPrune(gnuOnly, 'linux', 'x64'); + expect(plan.keep).toEqual(['linux-x64.node']); + expect(plan.remove).not.toContain('linux-x64.node'); + }); +}); + +describe('planPrebuildPrune refuses rather than guesses', () => { + it('removes NOTHING on a host the loader computes no prebuild for', () => { + // ⚠ The case that decides whether this script can break someone's install. On freebsd + // `getPrebuildPath()` returns null and the package can only work via a source build under + // `build/Release` — but the `prebuilds/` tree might have been populated by some mechanism we + // do not model, and then every candidate for removal might be the one that loads. + const plan = planPrebuildPrune(FULL_TREE, 'freebsd', 'x64'); + expect(plan.remove).toEqual([]); + expect(plan.keep).toEqual([]); + expect(plan.reason).toContain('refusing to prune'); + }); + + it('removes NOTHING on a supported platform with an unsupported ARCH', () => { + // Distinct from the case above and easy to get wrong: a check on the platform name alone + // would accept linux/riscv64 and then delete all eight prebuilds, leaving a tree whose cache + // database cannot open at all. + const plan = planPrebuildPrune(FULL_TREE, 'linux', 'riscv64'); + expect(plan.remove).toEqual([]); + expect(plan.reason).toContain('refusing to prune'); + }); + + it('removes NOTHING when the host prebuild is absent from the tree', () => { + // A source-built or partially-extracted tree. We cannot tell that from "already pruned to + // nothing", so we do not act — and here inaction is the only safe direction, because the + // loader's fallback path (`build/Release`) may be exactly what such a tree is relying on. + const withoutHost = FULL_TREE.filter((f) => f !== 'darwin-arm64.node'); + const plan = planPrebuildPrune(withoutHost, 'darwin', 'arm64'); + expect(plan.remove).toEqual([]); + expect(plan.reason).toContain('refusing to prune'); + }); +}); + +describe('planPrebuildPrune only ever removes prebuilds it recognises', () => { + it('never removes a target it has never heard of', () => { + // The allowlist is the point. If a future better-sqlite3 adds `android-arm64` or a + // glibc-versioned target, the honest answer is "we do not know whether that loads here". + const withFuture = [...FULL_TREE, 'android-arm64.node']; + const plan = planPrebuildPrune(withFuture, 'darwin', 'arm64'); + expect(plan.remove).not.toContain('android-arm64.node'); + expect(plan.remove).toHaveLength(7); + }); + + it('never removes a non-prebuild file that shares the directory', () => { + // `prebuilds/` is the package's own directory and a future release may put a manifest or a + // checksum file in it. Only the eight known targets are removable. + const withStray = [...FULL_TREE, 'index.json', 'README.md']; + const plan = planPrebuildPrune(withStray, 'darwin', 'arm64'); + expect(plan.remove).not.toContain('index.json'); + expect(plan.remove).not.toContain('README.md'); + }); + + it('recognises exactly the eight targets the package ships', () => { + // Pins the allowlist against the published tarball. If upstream's matrix changes, this is the + // assertion that makes someone look rather than silently pruning to a stale list. + expect([...SQLITE_PREBUILDS].sort()).toEqual([...FULL_TREE].sort()); + }); + + it('MUST-NOT-FIRE: `linuxmusl` is never produced by interpolating process.platform', () => { + // ⚠ THE TRAP THIS ALLOWLIST EXISTS FOR, pinned as a property rather than a comment. These + // names look interpolatable — `${platform}-${arch}.node` — and for six of the eight they are. + // `linuxmusl` is not a value `process.platform` ever takes, so a pattern-based prune that + // trusted interpolation would classify both musl builds as foreign on EVERY linux host and + // delete the file a musl host is the only one that can load. Asserting the keep-set contains a + // name no host reports is what makes that failure impossible to reintroduce quietly. + const keep = planPrebuildPrune(FULL_TREE, 'linux', 'x64').keep; + expect(keep).toContain('linuxmusl-x64.node'); + expect(keep.some((f: string) => f.startsWith(`${'linux'}-`))).toBe(true); + }); +}); + +describe('planPrebuildPrune is idempotent', () => { + it('proposes nothing on a tree it has already pruned, and does NOT call that a refusal', () => { + // npm re-runs a package's postinstall on installs that did not re-extract it, so the second + // run is the common case, not an edge one. It must report "nothing to do" — a refusal there + // would read as a broken layout in the install log every time. + const plan = planPrebuildPrune(['darwin-arm64.node'], 'darwin', 'arm64'); + expect(plan.remove).toEqual([]); + expect(plan.keep).toEqual(['darwin-arm64.node']); + expect(plan.reason).not.toContain('refusing'); + expect(plan.reason).toContain('nothing to do'); + }); + + it('is stable under re-application on linux, where two files survive', () => { + const first = planPrebuildPrune(FULL_TREE, 'linux', 'arm64'); + const second = planPrebuildPrune(first.keep, 'linux', 'arm64'); + expect(second.remove).toEqual([]); + expect(second.keep).toEqual(first.keep); + }); +}); + +/* + * The seam, not the decision. + * + * The driver also has to FIND the package. `better-sqlite3` declares no `exports` map, so unlike + * wreq-js the resolver spelling would work here — but the walk-up is shared code with a bound that + * matters, and the layout that matters most is the one nobody runs while developing: wigolo + * installed as a DEPENDENCY, which is what `npx wigolo` and `npm i wigolo` produce. + */ +const REPO_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); +const PRUNE_RUNNER = join(REPO_ROOT, 'scripts', 'prune', 'run.mjs'); +const trees: string[] = []; + +afterAll(() => { + for (const dir of trees) rmSync(dir, { recursive: true, force: true }); +}); + +/** Plant a fake `better-sqlite3` with all eight prebuilds under `/node_modules`. */ +function plantSqlite(at: string): void { + const pkg = join(at, 'node_modules', 'better-sqlite3'); + mkdirSync(join(pkg, 'prebuilds'), { recursive: true }); + writeFileSync(join(pkg, 'package.json'), JSON.stringify({ name: 'better-sqlite3', version: '13.0.3' })); + for (const name of FULL_TREE) writeFileSync(join(pkg, 'prebuilds', name), 'x'.repeat(1024)); +} + +function makeTree(): string { + const root = mkdtempSync(join(tmpdir(), 'bs3-prune-')); + trees.push(root); + writeFileSync(join(root, 'package.json'), JSON.stringify({ name: 'host-app', version: '1.0.0' })); + plantSqlite(root); + return root; +} + +/** Run the real postinstall driver against `resolveFrom`, returning its stdout. */ +function runPrune(resolveFrom: string, env: NodeJS.ProcessEnv = {}): string { + return execFileSync(process.execPath, [PRUNE_RUNNER, resolveFrom], { + encoding: 'utf8', + env: { ...process.env, ...env }, + }); +} + +/** The prebuilds left in `root`'s better-sqlite3 after a run. */ +function survivors(root: string): string[] { + return readdirSync(join(root, 'node_modules', 'better-sqlite3', 'prebuilds')).sort(); +} + +/** + * What SHOULD survive a run on whatever host the suite is executing on. + * + * Derived from the planner rather than hardcoded to darwin: on a host outside the loader's matrix + * the correct behaviour is to remove NOTHING, so the expected survivors are the whole tree. + */ +function expectedSurvivors(): string[] { + const plan = planPrebuildPrune(FULL_TREE, process.platform, process.arch); + return plan.remove.length > 0 ? [...plan.keep].sort() : [...FULL_TREE].sort(); +} + +/** + * The log line the driver must emit on THIS host. + * + * ⚠ WHY THE SURVIVOR SET ALONE IS NOT ENOUGH. On a host outside the loader's matrix + * `expectedSurvivors()` is the whole tree, so an assertion on it passes for a prune that correctly + * refused AND for one that never found the package, never ran, or silently threw. Pairing it with + * the reason the driver PRINTED makes both outcomes fail-loud instead of vacuous. + */ +function expectedPruneVerb(): RegExp { + return planPrebuildPrune(FULL_TREE, process.platform, process.arch).remove.length > 0 + ? /better-sqlite3 prebuild prune — keeping .*removing \d+ non-host/ + : /better-sqlite3 prebuild prune — .*refusing to prune/; +} + +describe('the postinstall driver finds better-sqlite3 and prunes it', () => { + it('prunes when wigolo IS the install root — the layout the budget gate measures', () => { + const root = makeTree(); + const out = runPrune(root); + expect(out).toMatch(expectedPruneVerb()); + expect(survivors(root)).toEqual(expectedSurvivors()); + }); + + it('prunes the HOISTED copy when wigolo is installed as a dependency', () => { + // ⚠ THE LAYOUT A REAL `npx wigolo` / `npm i wigolo` USER GETS, and the one the gate cannot + // see: wigolo at /node_modules/wigolo, better-sqlite3 hoisted beside it. + const root = makeTree(); + const asDependency = join(root, 'node_modules', 'wigolo'); + mkdirSync(asDependency, { recursive: true }); + const out = runPrune(asDependency); + expect(out).toMatch(expectedPruneVerb()); + expect(survivors(root)).toEqual(expectedSurvivors()); + }); + + it('leaves every prebuild in place when the prune is opted out of', () => { + // The documented escape hatch for anyone deliberately assembling a multi-arch tree. + const root = makeTree(); + runPrune(root, { WIGOLO_SKIP_ORT_PRUNE: '1' }); + expect(survivors(root)).toEqual([...FULL_TREE].sort()); + }); + + it('MUST-NOT-FIRE: leaves a better-sqlite3 belonging to an enclosing project untouched', () => { + // wigolo is installed into `proj` with no better-sqlite3 beside it. The only one reachable by + // climbing belongs to `outer`, and destroying its prebuilds would break a DIFFERENT project's + // database — silently, after npm reported success, in a tree that was multi-arch on purpose. + const outer = mkdtempSync(join(tmpdir(), 'bs3-outer-')); + trees.push(outer); + writeFileSync(join(outer, 'package.json'), JSON.stringify({ name: 'outer', version: '1.0.0' })); + plantSqlite(outer); + const proj = join(outer, 'proj'); + const wigolo = join(proj, 'node_modules', 'wigolo'); + mkdirSync(wigolo, { recursive: true }); + writeFileSync(join(proj, 'package.json'), JSON.stringify({ name: 'proj', version: '1.0.0' })); + + const out = runPrune(wigolo); + expect(survivors(outer)).toEqual([...FULL_TREE].sort()); + expect(out).not.toMatch(/better-sqlite3 prebuild prune — keeping .*removing/); + }); + + it('survives a tree with no better-sqlite3 at all', () => { + const root = mkdtempSync(join(tmpdir(), 'bs3-prune-none-')); + trees.push(root); + writeFileSync(join(root, 'package.json'), JSON.stringify({ name: 'host-app', version: '1.0.0' })); + mkdirSync(join(root, 'node_modules'), { recursive: true }); + expect(() => runPrune(root)).not.toThrow(); + expect(existsSync(join(root, 'node_modules', 'better-sqlite3'))).toBe(false); + }); + + it('survives a better-sqlite3 with no prebuilds/ directory — the v12 source-build layout', () => { + // ⚠ THE REGRESSION DIRECTION OF THIS WHOLE SLICE. Before the v13 bump the package had no + // `prebuilds/` at all, only `build/Release`. Anyone whose tree still looks like that (a source + // build, or a downgrade) must see the driver do nothing quietly rather than throw and fail + // their install. + const root = mkdtempSync(join(tmpdir(), 'bs3-prune-src-')); + trees.push(root); + writeFileSync(join(root, 'package.json'), JSON.stringify({ name: 'host-app', version: '1.0.0' })); + const pkg = join(root, 'node_modules', 'better-sqlite3'); + mkdirSync(join(pkg, 'build', 'Release'), { recursive: true }); + writeFileSync(join(pkg, 'package.json'), JSON.stringify({ name: 'better-sqlite3', version: '12.9.0' })); + writeFileSync(join(pkg, 'build', 'Release', 'better_sqlite3.node'), 'x'); + + expect(() => runPrune(root)).not.toThrow(); + expect(existsSync(join(pkg, 'build', 'Release', 'better_sqlite3.node'))).toBe(true); + }); +});