diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..481f184 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,14 @@ +name: ci +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + test: + # ci-workflows python-ci @ main + uses: img2threejs/ci-workflows/.github/workflows/python-ci.yml@4b7a0612d2c52815792124a18ae3f9032a4f0a88 + with: + python-version: '3.12' + test-command: npm test diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..7a82705 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,21 @@ +name: publish +on: + push: + tags: ['v*'] +permissions: + contents: read +jobs: + publish: + # ci-workflows npm-publish (feat/reusable-npm-publish); re-pin to the merged main SHA once img2threejs/ci-workflows#2 lands + uses: img2threejs/ci-workflows/.github/workflows/npm-publish.yml@27da7a68505591492cb4ef7358968890c6e39844 + permissions: + contents: read + id-token: write + with: + tag: ${{ github.ref_name }} + version-file: package.json + node-version: '24' + python-version: '3.12' + test-command: npm test + secrets: + npm-token: ${{ secrets.NPM_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 6502cb4..9d8d318 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,39 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 change that breaks a conforming plugin bumps one of those and states the migration in `docs/PLUGIN_CONTRACT.md` itself. +## [Unreleased] + +### Added + +- **`npm:` plugin source.** `img2 add npm:[@]` fetches a plugin from npm instead of + git — `npm view` resolves the version and `dist.integrity`, `npm pack` fetches the tarball, and + the harness extracts it itself (stripping the tarball's `package/` prefix) into the same + staging → validate → link flow git installs already go through. Default trust is the + `@img2threejs` scope, mirroring the `img2threejs/*` git org; anything else needs + `--allow-any-source`. The registry row keeps its four keys: `repo` becomes `npm:`, `ref` is + the resolved version, `resolvedSha` is npm's `dist.integrity` (a `sha512-…` string, not a git SHA). +- **`img2 update [] [--check]`.** Re-checks every registered plugin's source for something + newer — the newest npm version or the newest reachable git tag — and re-fetches it in place if + so, backing up the previous clone. A `link:` row is a local dev checkout and is left alone. + `--check` reports what is pending without fetching anything, exiting non-zero if there is any. +- Audited every git-specific assumption doctor/list/add made about a plugin directory: the + `.gitignore` covers `_img2_local.py` check now only runs when the plugin dir is actually a git + checkout (an npm-fetched or bare `--link`'d directory has no `.git` and no such hazard), and the + short-SHA display in `img2 add`/`img2 list` now shows a useful slice of an npm `sha512-…` + integrity string instead of just the literal `sha512-` prefix. +- `package.json`'s `files` field now excludes `__pycache__/`, `*.pyc`, and any stray `.omc/` + directory wherever they occur under `docs/`/`img2_core/` — `npm pack` does not consult + `.gitignore` for a directory explicitly listed in `files`, so without this a local dev artifact + left in the working tree would have shipped in the published tarball. +- CI: `.github/workflows/ci.yml` and `publish.yml` call the org's shared `img2threejs/ci-workflows` + reusable workflows (pinned by commit SHA, per org policy) instead of carrying inline workflow + logic — `ci.yml` runs the full test suite on every PR and push to `main`; `publish.yml` + publishes `img2` to npm (via an `NPM_TOKEN` secret, not OIDC trusted publishing) on a `vX.Y.Z` + tag push, after validating the tag against `package.json` and refusing install-time lifecycle + scripts. Both are pinned to a feature branch of `ci-workflows` pending + [img2threejs/ci-workflows#2](https://github.com/img2threejs/ci-workflows/pull/2); re-pin once + it merges. + ## [0.2.3] — 2026-09-03 ### Added diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1fe450b..5fa1e29 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -52,3 +52,24 @@ it demonstrably catches the bug it guards against. Tag `vX.Y.Z` (annotated) matching `package.json`; `img2 add` and self-installs resolve the newest semver tag. Keep `docs/PLAN.md` phase status current when a release changes what is true. + +## Publishing + +The harness itself ships to npm as the `img2` package. `.github/workflows/ci.yml` and +`publish.yml` are thin callers into the org's shared `img2threejs/ci-workflows` repo, pinned by +full commit SHA (org policy — never a branch or tag ref); re-pin to `ci-workflows`'s merged +`main` SHA once [img2threejs/ci-workflows#2](https://github.com/img2threejs/ci-workflows/pull/2) +lands, and again whenever a reusable workflow there changes in a way this repo needs. + +To release: bump `package.json`'s `version` and add a `CHANGELOG.md` entry, tag `vX.Y.Z` +matching it, and push the tag. The reusable `npm-publish.yml` workflow validates the tag against +`package.json`, refuses `preinstall`/`install`/`postinstall`/`prepare` lifecycle scripts, runs +`npm test` in a job with no access to the npm token, then publishes with +`npm publish --provenance --access public`, authenticated by the **`NPM_TOKEN`** secret (a +granular npm automation token — this is a plain token, not OIDC trusted publishing). It no-ops if +the tagged version is already on the registry, and a prerelease tag (`v0.3.0-beta.1`) publishes +under its prerelease dist-tag (`beta`). + +`NPM_TOKEN` must exist as an org or repo Actions secret before the first tag push — the reusable +workflow publishes straight from CI, so unlike a trusted-publishing setup there is no separate +manual first-publish step once the token is in place. diff --git a/README.md b/README.md index 095ab4f..daa6cae 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,9 @@ npx github:img2threejs/img2 install # $IMG2_HOME (~/.img2), harness checkout # (~/.local/bin, /opt/homebrew/bin, or /usr/local/bin — # if none qualifies, install prints an alias to use instead) img2 add img2threejs/plugin-img2glb # clone @ newest tag, pin SHA, link ~/.claude/skills/img2-img2glb +img2 add npm:@img2threejs/plugin-cs2 # npm instead of git: fetch @ newest version, pin dist.integrity img2 list # registered plugins: id, version, ref, sha +img2 update # re-check every row's source for a newer version, fetch it img2 doctor # fail-loud static audit of every row (--json for tooling) img2 sync --check # generated index == manifests (CI-able) img2 remove img2glb # unlink every host, move clone to backups, drop the row @@ -87,14 +89,23 @@ Developing a plugin locally: img2 add --link ~/src/plugin-hello-cube # symlink, no clone; ref/sha recorded as "local" ``` +A plugin can also be distributed as an npm package instead of a git repo — `img2 add npm:` +or `img2 add npm:@` (default source trust is the `@img2threejs` scope, same rule as +the `img2threejs/*` git org: anything else needs `--allow-any-source`). `ref` records the resolved +version and `resolvedSha` records npm's own `dist.integrity` (a `sha512-…` string) rather than a git +SHA — the harness's own notion of "pinned content hash" for a package it fetched instead of cloned. +`img2 update [] [--check]` re-checks every row's source (npm version, git's newest semver tag) for +something newer and fetches it in place; `--check` reports what is pending without changing anything. + Layout under `$IMG2_HOME` (default `~/.img2`; the deprecated `IMG2THREEJS_HOME` is honoured for one release with a warning): ``` harness/ canonical harness checkout (img2_core lives here) -plugins// one clone (or --link symlink) per registry row +plugins// one clone (or --link symlink, or npm fetch) per registry row generated/ index.md + routes.json, regenerated by `img2 sync` plugins.json the registry: flat rows {id, repo, ref, resolvedSha, addedAt} + repo is "org/repo", a git URL, "npm:", or "link:" receipts.json what was linked where backups/ displaced directories; nothing is ever deleted in place ``` diff --git a/bin/img2.mjs b/bin/img2.mjs index c7045a1..988c0b6 100755 --- a/bin/img2.mjs +++ b/bin/img2.mjs @@ -126,6 +126,29 @@ function isGitRepo(dir) { } } +// ---------------------------------------------------------------- npm & tar + +// npm and tar warn on stderr for reasons that have nothing to do with success (e.g. a machine-wide +// NODE_TLS_REJECT_UNAUTHORIZED=0 makes npm print a TLS warning) -- only a non-zero exit is a failure, +// stderr content on its own is not, same as `git` above. Neither needs git's cwd/env handling, so +// they share one wrapper instead of each repeating the ENOENT/failure-message shape. +function runTool(bin, args, execOpts = {}) { + try { + return execFileSync(bin, args, { stdio: ['ignore', 'pipe', 'pipe'], ...execOpts }) + } catch (err) { + if (err.code === 'ENOENT') throw new CliError(EXIT.FAIL, bin + ' is not on PATH; install ' + bin + ' first') + throw new CliError(EXIT.FAIL, bin + ' ' + args.join(' ') + ' failed', String(err.stderr || '').trim()) + } +} + +function npm(args) { + return runTool('npm', args, { encoding: 'utf8' }).trim() +} + +function tar(args) { + runTool('tar', args) +} + // ---------------------------------------------------------------- semver export function parseSemver(s) { @@ -174,6 +197,35 @@ export function resolveSource(spec, allowAny) { return { url: spec, label: spec, defaultOrg } } +// An `npm:` spec is `npm:` or `npm:@`; `` may itself start with a +// scope ("@scope/pkg"), so the version separator is the first "@" AFTER that leading one, not the +// first "@" in the string. +export function parseNpmSpec(spec) { + const rest = spec.slice('npm:'.length) + const at = rest.startsWith('@') ? rest.indexOf('@', 1) : rest.indexOf('@') + const name = at === -1 ? rest : rest.slice(0, at) + const version = at === -1 ? null : rest.slice(at + 1) + if (!name) throw new CliError(EXIT.REFUSED, 'npm: source needs a package name', spec) + if (version === '') throw new CliError(EXIT.REFUSED, 'npm: source has an empty version after "@"', spec) + return { name, version } +} + +// The npm equivalent of `resolveSource`'s org check: the default-trusted scope is "@", +// matching the "@img2threejs/plugin-*" packages this harness's own plugins are published as. +// Unscoped packages are never default-trusted, same as a bare non-shorthand git URL. +export function resolveNpmSource(name, allowAny) { + const scoped = /^@([A-Za-z0-9_.-]+)\/([A-Za-z0-9_.-]+)$/.exec(name) + const defaultOrg = Boolean(scoped && scoped[1] === DEFAULT_ORG) + if (!defaultOrg && !allowAny) { + throw new CliError( + EXIT.REFUSED, + 'source npm package "' + name + '" is outside the default @' + DEFAULT_ORG + ' scope', + 'pass --allow-any-source to accept npm:' + name, + ) + } + return { defaultOrg } +} + // ---------------------------------------------------------------- manifest export function validateManifest(manifest, where) { @@ -418,6 +470,15 @@ function ensureLink(target, dest, H) { return c.state === 'absent' ? 'linked' : 'relinked' } +// A git resolvedSha is a 40-hex commit SHA, where the first 7 chars are the conventional short +// form. An npm resolvedSha is `sha512-` (dist.integrity) -- slicing the first 7 chars of +// THAT would just print "sha512-", so this shows the first 7 chars of whichever part actually +// varies between installs. +function shortSha(resolvedSha) { + const s = String(resolvedSha) + return s.startsWith('sha512-') ? s.slice(7, 14) : s.slice(0, 7) +} + const stamp = () => new Date().toISOString().replace(/[:.]/g, '-') + '-' + process.pid function moveToBackups(H, src, label) { @@ -1160,24 +1221,36 @@ async function cmdInstall(opts) { } } +// Newest reachable semver tag on a remote, or null if it has none (untagged, or tags that don't +// parse as semver). Shared by `add` (pick a ref when none is pinned) and `update` (decide whether a +// newer tag exists). +function latestTagFor(url) { + const tags = git(['ls-remote', '--tags', url]) + .split('\n') + .map((line) => line.split('refs/tags/')[1]) + .filter((t) => t && !t.endsWith('^{}')) + .map((t) => ({ tag: t, key: parseSemver(t.replace(/^v/, '')) })) + .filter((t) => t.key) + .sort((a, b) => cmpSemver(b.key, a.key)) + return tags.length ? tags[0].tag : null +} + +// Clears a stale leftover from a previous crashed run under the same pid and ensures the parent +// (pluginsDir(H)) exists; the dir itself is left for the caller to create when it actually has +// content to put there. Shared by the git and npm fetch paths so "staging directory" has one +// lifecycle, not one hand-written per source kind. +function freshStagingDir(dir) { + fs.rmSync(dir, { recursive: true, force: true }) + fs.mkdirSync(path.dirname(dir), { recursive: true }) + return dir +} + async function resolveRefAndClone(H, spec, opts) { const { url, label, defaultOrg } = resolveSource(spec, opts.allowAnySource) await confirmOutOfOrgSource(url, defaultOrg, opts) - const staging = path.join(pluginsDir(H), '.staging-' + process.pid) - fs.rmSync(staging, { recursive: true, force: true }) - fs.mkdirSync(pluginsDir(H), { recursive: true }) - - let ref = opts.ref || null - if (!ref) { - const tags = git(['ls-remote', '--tags', url]) - .split('\n') - .map((line) => line.split('refs/tags/')[1]) - .filter((t) => t && !t.endsWith('^{}')) - .map((t) => ({ tag: t, key: parseSemver(t.replace(/^v/, '')) })) - .filter((t) => t.key) - .sort((a, b) => cmpSemver(b.key, a.key)) - if (tags.length) ref = tags[0].tag - } + const staging = freshStagingDir(path.join(pluginsDir(H), '.staging-' + process.pid)) + + let ref = opts.ref || latestTagFor(url) git(['clone', '-q', url, staging]) if (ref) { @@ -1194,6 +1267,69 @@ async function resolveRefAndClone(H, spec, opts) { return { staging, label, ref, resolvedSha } } +// One `npm view` call, parsed and validated -- shared by `resolveNpmAndFetch` (add) and `cmdUpdate` +// (check for a newer version), so update never has to view a package a second time just to fetch +// the version it already viewed. +function npmViewInfo(nameAtSpec) { + const viewed = npm(['view', nameAtSpec, 'version', 'dist.integrity', '--json']) + let info + try { + info = JSON.parse(viewed) + } catch (err) { + throw new CliError(EXIT.FAIL, 'npm view ' + nameAtSpec + ' did not return JSON', viewed) + } + // A version range matching more than one release comes back as an array; a concrete version or + // the default "latest" tag (our only two spec forms) always resolves to exactly one, but take the + // newest rather than crash if npm's own resolution ever disagrees. + if (Array.isArray(info)) info = info[info.length - 1] + if (!info || typeof info.version !== 'string' || !info.dist || typeof info.dist.integrity !== 'string') { + throw new CliError(EXIT.FAIL, 'npm view ' + nameAtSpec + ' did not return a version and dist.integrity', viewed) + } + return { version: info.version, integrity: info.dist.integrity } +} + +// Fetches one already-resolved npm package version into a fresh staging dir under pluginsDir(H). +// Version/integrity resolution is npmViewInfo's job, kept separate for the same reason. +function fetchNpmTarball(H, name, version) { + const staging = freshStagingDir(path.join(pluginsDir(H), '.staging-' + process.pid)) + const packStaging = freshStagingDir(path.join(pluginsDir(H), '.staging-pack-' + process.pid)) + fs.mkdirSync(packStaging, { recursive: true }) + try { + npm(['pack', name + '@' + version, '--pack-destination', packStaging]) + const tarballs = fs.readdirSync(packStaging).filter((f) => f.endsWith('.tgz')) + if (tarballs.length !== 1) { + throw new CliError( + EXIT.FAIL, + 'npm pack ' + name + '@' + version + ' produced ' + tarballs.length + ' tarball(s) in ' + packStaging + ', expected 1', + ) + } + fs.mkdirSync(staging, { recursive: true }) + // A published tarball's content sits under a "package/" prefix; strip it so `staging` mirrors + // exactly what a git checkout of the plugin repo root would put there. + tar(['-xzf', path.join(packStaging, tarballs[0]), '-C', staging, '--strip-components=1']) + } finally { + fs.rmSync(packStaging, { recursive: true, force: true }) + } + return staging +} + +// npm's counterpart to resolveRefAndClone: same return shape ({staging, label, ref, resolvedSha}), +// so `cmdAdd` needs only pick which of the two to call, never branch afterwards. `ref` is the +// resolved version; `resolvedSha` is npm's own `dist.integrity` (a `sha512-...` string, not a git +// SHA -- this harness has no other notion of "pinned content hash" for a package it did not clone, +// and integrity is exactly that for a tarball). +async function resolveNpmAndFetch(H, spec, opts) { + const { name, version: wantVersion } = parseNpmSpec(spec) + const { defaultOrg } = resolveNpmSource(name, opts.allowAnySource) + const label = 'npm:' + name + await confirmOutOfOrgSource(label, defaultOrg, opts) + + const nameAtSpec = wantVersion ? name + '@' + wantVersion : name + const { version, integrity } = npmViewInfo(nameAtSpec) + const staging = fetchNpmTarball(H, name, version) + return { staging, label, ref: version, resolvedSha: integrity } +} + async function cmdAdd(opts, spec) { if (!spec && !opts.link) throw new CliError(EXIT.REFUSED, 'add needs , a URL, or --link ') if (spec && opts.link) throw new CliError(EXIT.REFUSED, 'add takes either a source spec or --link, not both') @@ -1214,7 +1350,7 @@ async function cmdAdd(opts, spec) { manifest = readManifest(source) row = { id: manifest.name, repo: 'link:' + source, ref: 'local', resolvedSha: 'local', addedAt: new Date().toISOString() } } else { - const cloned = await resolveRefAndClone(H, spec, opts) + const cloned = spec.startsWith('npm:') ? await resolveNpmAndFetch(H, spec, opts) : await resolveRefAndClone(H, spec, opts) staging = cloned.staging manifest = readManifest(staging) row = { id: manifest.name, repo: cloned.label, ref: cloned.ref, resolvedSha: cloned.resolvedSha, addedAt: new Date().toISOString() } @@ -1277,7 +1413,7 @@ async function cmdAdd(opts, spec) { } syncAll(H, false) - console.log('added ' + id + ' ' + manifest.version + ' (' + row.ref + ' @ ' + String(row.resolvedSha).slice(0, 7) + ')') + console.log('added ' + id + ' ' + manifest.version + ' (' + row.ref + ' @ ' + shortSha(row.resolvedSha) + ')') return EXIT.OK } finally { if (staging) fs.rmSync(staging, { recursive: true, force: true }) @@ -1333,7 +1469,7 @@ async function cmdList(opts) { try { version = JSON.parse(fs.readFileSync(path.join(cloneDir(H, row.id), 'plugin.json'), 'utf8')).version || '?' } catch { /* listed anyway; doctor reports the cause */ } - console.log(row.id.padEnd(24) + String(version).padEnd(10) + String(row.ref).padEnd(14) + String(row.resolvedSha).slice(0, 7)) + console.log(row.id.padEnd(24) + String(version).padEnd(10) + String(row.ref).padEnd(14) + shortSha(row.resolvedSha)) } return EXIT.OK } @@ -1402,12 +1538,18 @@ async function cmdDoctor(opts) { ) } - let ignored = false - try { - git(['check-ignore', '-q', '--', '_img2_local.py'], dir) - ignored = true - } catch { /* not ignored, or not a git work tree */ } - if (!ignored) err(row.id, '.gitignore does not cover _img2_local.py (contract section 4)') + // Only meaningful for a git checkout: the rule guards against _img2_local.py landing in a + // commit, which cannot happen in an npm-fetched or bare --link'd directory that has no .git + // at all. Requiring a .gitignore entry there would be a git assumption failing a plugin for a + // hazard that does not exist for it. + if (isGitRepo(dir)) { + let ignored = false + try { + git(['check-ignore', '-q', '--', '_img2_local.py'], dir) + ignored = true + } catch { /* not ignored */ } + if (!ignored) err(row.id, '.gitignore does not cover _img2_local.py (contract section 4)') + } for (const localFile of localFiles(dir)) { const rel = path.relative(dir, localFile) @@ -1602,6 +1744,94 @@ async function cmdSync(opts) { } } +// ---------------------------------------------------------------- update + +// A `link:` row is a symlink to a local dev checkout -- nothing to fetch, the working tree IS the +// current version. An `npm:` row's newest version comes from `npm view`; everything else is a git +// remote, whose newest reachable semver tag comes from `latestTagFor` (same helper `add` uses). +// Re-confirming an already-registered non-default-org source on every update, rather than trusting +// it forever once accepted, matches the existing trust boundary: `add --force` re-confirms too. +async function cmdUpdate(opts, args) { + const H = resolveImg2Home(opts.home) + const onlyId = args[0] || null + acquireLock(H) + let staging = null + try { + const reg = readRegistry(H) + const rows = (onlyId ? reg.plugins.filter((r) => r.id === onlyId) : [...reg.plugins]).sort((a, b) => a.id.localeCompare(b.id)) + if (onlyId && !rows.length) throw new CliError(EXIT.FAIL, 'no registered plugin "' + onlyId + '"') + + let updated = 0 + let pending = 0 + for (const row of rows) { + if (row.repo.startsWith('link:')) { + console.log(' local ' + row.id + ' (' + row.repo + '); nothing to update') + continue + } + + let latest, fetch + if (row.repo.startsWith('npm:')) { + const name = row.repo.slice('npm:'.length) + // One view up front decides whether there is anything to do; fetch (below) reuses its + // result instead of viewing the same package a second time just to learn what it just did. + const info = npmViewInfo(name) + latest = info.version + fetch = () => ({ staging: fetchNpmTarball(H, name, info.version), ref: info.version, resolvedSha: info.integrity }) + } else { + const { url } = resolveSource(row.repo, true) + latest = latestTagFor(url) + if (!latest) { + console.log(' no tag ' + row.id + ' (' + row.repo + '); nothing to compare against') + continue + } + fetch = () => resolveRefAndClone(H, row.repo, { ...opts, ref: latest, allowAnySource: true }) + } + + if (latest === row.ref) { + console.log(' up to date ' + row.id + ' ' + row.ref) + continue + } + pending += 1 + if (opts.check) { + console.log(' update avail ' + row.id + ' ' + row.ref + ' -> ' + latest) + continue + } + + const cloned = await fetch() + staging = cloned.staging + const manifest = readManifest(staging) + if (manifest.name !== row.id) { + const bad = staging + staging = null + fs.rmSync(bad, { recursive: true, force: true }) + throw new CliError(EXIT.FAIL, row.id + ': updated manifest name "' + manifest.name + '" no longer matches the registered id') + } + const dest = cloneDir(H, row.id) + console.log(' backed up ' + dest + ' -> ' + moveToBackups(H, dest, row.id)) + fs.renameSync(staging, dest) + staging = null + row.ref = cloned.ref + row.resolvedSha = cloned.resolvedSha + updated += 1 + console.log(' updated ' + row.id + ' -> ' + row.ref + ' @ ' + shortSha(row.resolvedSha)) + } + + if (updated) { + writeRegistry(H, reg) + syncAll(H, false) + } + if (opts.check) { + console.log('update --check: ' + pending + ' pending') + return pending ? EXIT.FAIL : EXIT.OK + } + console.log('update: ' + updated + ' updated') + return EXIT.OK + } finally { + if (staging) fs.rmSync(staging, { recursive: true, force: true }) + releaseLock() + } +} + // ---------------------------------------------------------------- capability resolution (§13) // A manifest that fails full validation (e.g. schema too new) may still be legible enough to see @@ -1827,9 +2057,11 @@ const HELP = [ 'Usage', ' img2 install [--from ] [--home ] [--yes] [--migrate-legacy]', ' img2 add [--ref ] [--force] [--allow-any-source]', + ' img2 add npm:[@] [--force] [--allow-any-source]', ' img2 add --link [--force]', ' img2 remove ', ' img2 list', + ' img2 update [] [--check] [--yes] [--allow-any-source]', ' img2 doctor [--json]', ' img2 sync [--check]', ' img2 capabilities [--from-kind ] [--to-kind ] [--plugin ] [--json]', @@ -1842,8 +2074,8 @@ const HELP = [ ' --ref pin a plugin to a tag or branch (default: newest semver tag)', ' --link register a local plugin checkout via symlink (no clone)', ' --force replace an existing registered plugin', - ' --allow-any-source accept a source outside the ' + DEFAULT_ORG + '/* org', - ' --check sync: verify generated artifacts without writing', + ' --allow-any-source accept a source outside the ' + DEFAULT_ORG + '/* org (or @' + DEFAULT_ORG + ' npm scope)', + ' --check sync: verify generated artifacts without writing; update: report pending updates without fetching', ' --from-kind capabilities: the edge\'s source kind', ' --to-kind capabilities: the edge\'s destination kind', ' --plugin capabilities: disambiguate to one named provider', @@ -1912,6 +2144,7 @@ const COMMANDS = { add: (opts, args) => cmdAdd(opts, args[0]), remove: (opts, args) => cmdRemove(opts, args[0]), list: (opts) => cmdList(opts), + update: (opts, args) => cmdUpdate(opts, args), doctor: (opts) => cmdDoctor(opts), sync: (opts) => cmdSync(opts), capabilities: (opts) => cmdCapabilities(opts), diff --git a/docs/PLUGIN_CONTRACT.md b/docs/PLUGIN_CONTRACT.md index 3337b45..60b1ecf 100644 --- a/docs/PLUGIN_CONTRACT.md +++ b/docs/PLUGIN_CONTRACT.md @@ -116,9 +116,20 @@ adopted the day a provider actually needs it, per the project's own CUT rule. - `id` = manifest `name` = host link suffix. `img2 add` on an existing id exits non-zero naming the row; replacement requires `--force`. -- `ref` defaults to the repo's newest reachable tag; a moving branch requires explicit - `--ref ` and is recorded as such. `resolvedSha` is always recorded so a user's - setup is reproducible. +- `repo` is one of: `org/repo` shorthand, a git URL, `npm:` (an npm-distributed + plugin), or `link:` (a `--link`'d local checkout). The four keys stay the same + across all of them — only what `ref`/`resolvedSha` mean changes. +- For a git `repo`, `ref` defaults to the repo's newest reachable tag; a moving branch + requires explicit `--ref ` and is recorded as such. `resolvedSha` is the + resolved commit SHA. +- For an `npm:` `repo`, `ref` is the resolved version (an explicit `npm:@` + spec pins it; otherwise the newest published version) and `resolvedSha` is npm's own + `dist.integrity` for that version (a `sha512-…` string, not a git SHA) — this harness's + only other notion of "pinned content hash" for a package it fetched instead of cloning. +- `resolvedSha` is always recorded, whatever its form, so a user's setup is reproducible. +- `img2 update [] [--check]` re-resolves each row's source the same way `add` would + (newest git tag, or newest npm version) and re-fetches it in place if it differs from + `ref`; a `link:` row has nothing to fetch and is left untouched. ### Reserved: `overrides` @@ -146,10 +157,11 @@ four rules; it defines no mechanism. ## 7. Trust boundary -- `img2 add ` accepts `org/repo`, a URL, or `--link ` (symlink a local - checkout for development; no clone). -- Default allowed source is the `img2threejs/*` org. Anything else requires - `--allow-any-source` and prints what it is about to clone and link. +- `img2 add ` accepts `org/repo`, a URL, `npm:[@]`, or + `--link ` (symlink a local checkout for development; no clone). +- Default allowed source is the `img2threejs/*` org (git) or the `@img2threejs` scope + (npm — this project's own plugins are published as `@img2threejs/plugin-*`). Anything + else requires `--allow-any-source` and prints what it is about to clone and link. - The harness NEVER runs plugin tests or imports plugin code during add/doctor/sync. - Threat model: a plugin is arbitrary code the agent will later execute in the user's workspace. `add` therefore pins (`resolvedSha`), attributes (registry row), and links diff --git a/package.json b/package.json index 624135d..67d4451 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,10 @@ "files": [ "bin", "img2_core", - "docs" + "docs", + "!**/__pycache__/**", + "!**/*.pyc", + "!**/.omc/**" ], "scripts": { "test": "node --test tests/*.test.mjs && python3 -m unittest discover -s tests/python -p 'test_*.py'" diff --git a/tests/cli.e2e.test.mjs b/tests/cli.e2e.test.mjs index 384d41b..dfc1134 100644 --- a/tests/cli.e2e.test.mjs +++ b/tests/cli.e2e.test.mjs @@ -75,6 +75,113 @@ function pluginWithStep(root, name, command) { return makePluginRepo(root, name, { steps: [{ id: 'run', title: 'Run', command }] }) } +// ---------------------------------------------------------------- fake npm registry +// +// There is no real npm registry to point at in a test, and `npm add`'s job is exactly the two +// subprocess calls a fake `npm` on PATH can intercept (`view`, `pack`) -- the rest (readManifest, +// staging, the registry row) is identical to the git path and already covered by it. `tar` is left +// real: extraction (untrusted content -> disk) is the harness's own code, not npm's. + +// A package's file content -- the same shape `makePluginRepo` writes, minus the git repo (an +// npm-distributed plugin has no .git at all, which is exactly the case the doctor gitignore-check +// fix guards). +function npmPluginFiles(name, version, { capabilities, tool = 'print("ok")\n' } = {}) { + const manifest = { + schema: 1, + name, + version, + description: 'Test npm plugin ' + name + '.', + capabilities: capabilities || [{ from: 'image', to: 'threejs-code' }], + requires: { harness: '>=0.1.0', coreApi: 1 }, + } + return { + 'plugin.json': JSON.stringify(manifest, null, 2) + '\n', + 'SKILL.md': '# ' + name + '\n', + '.gitignore': '_img2_local.py\n', + 'tools/noop.py': tool, + } +} + +// Lays a package version's files under a "package/" directory -- the same prefix a real npm +// tarball uses, and the one the harness's `tar --strip-components=1` expects to strip. +function fakeNpmPackage(root, name, version, integrity, files) { + const pkgDir = path.join(root, 'fake-npm-pkgs', name.replace('/', '__'), version, 'package') + fs.mkdirSync(pkgDir, { recursive: true }) + for (const [rel, content] of Object.entries(files)) { + const dest = path.join(pkgDir, rel) + fs.mkdirSync(path.dirname(dest), { recursive: true }) + fs.writeFileSync(dest, content) + } + return { name, version, integrity, dir: path.dirname(pkgDir) } +} + +function fakeNpmScript(manifestPath) { + return [ + '#!/usr/bin/env node', + 'const fs = require("fs")', + 'const path = require("path")', + 'const { execFileSync } = require("child_process")', + 'const MANIFEST = ' + JSON.stringify(manifestPath), + 'const packages = JSON.parse(fs.readFileSync(MANIFEST, "utf8"))', + 'const args = process.argv.slice(2)', + 'const sub = args[0]', + 'function findPkg(spec) {', + ' let name = spec, version = null', + ' const at = spec.startsWith("@") ? spec.indexOf("@", 1) : spec.indexOf("@")', + ' if (at !== -1) { name = spec.slice(0, at); version = spec.slice(at + 1) }', + ' const candidates = packages.filter((p) => p.name === name)', + ' if (!candidates.length) { console.error("fake npm: no such package " + name); process.exit(1) }', + ' const pkg = version ? candidates.find((p) => p.version === version) : candidates[candidates.length - 1]', + ' if (!pkg) { console.error("fake npm: no such version " + spec); process.exit(1) }', + ' return pkg', + '}', + '// A real npm often warns on stderr for reasons unrelated to success (e.g. a TLS override); the', + '// harness must tolerate stderr noise on a successful call, so the fake always emits one.', + 'process.stderr.write("npm warn (fake) using a test registry\\n")', + 'if (sub === "view") {', + ' const pkg = findPkg(args[1])', + ' if (args.includes("--json")) console.log(JSON.stringify({ version: pkg.version, dist: { integrity: pkg.integrity } }))', + ' else console.log(pkg.version)', + ' process.exit(0)', + '}', + 'if (sub === "pack") {', + ' const pkg = findPkg(args[1])', + ' const dest = args[args.indexOf("--pack-destination") + 1]', + ' fs.mkdirSync(dest, { recursive: true })', + ' const tarballName = pkg.name.replace("@", "").replace("/", "-") + "-" + pkg.version + ".tgz"', + ' execFileSync("tar", ["-czf", path.join(dest, tarballName), "-C", pkg.dir, "package"])', + ' console.log(tarballName)', + ' process.exit(0)', + '}', + 'console.error("fake npm: unsupported subcommand " + sub)', + 'process.exit(1)', + ].join('\n') + '\n' +} + +// Starts a fake npm registry backed by a JSON manifest file the fake `npm` executable re-reads on +// every invocation -- so a test can register a newer version mid-run (simulating a new release) +// without restarting anything. Returns the PATH-prependable bin dir and the manifest path. +function startFakeNpm(root, packages) { + const manifestPath = path.join(root, 'fake-npm-manifest-' + Math.random().toString(36).slice(2) + '.json') + fs.writeFileSync(manifestPath, JSON.stringify(packages.map(({ name, version, integrity, dir }) => ({ name, version, integrity, dir })))) + const binDir = path.join(root, 'fake-npm-bin-' + Math.random().toString(36).slice(2)) + fs.mkdirSync(binDir, { recursive: true }) + const script = path.join(binDir, 'npm') + fs.writeFileSync(script, fakeNpmScript(manifestPath)) + fs.chmodSync(script, 0o755) + return { binDir, manifestPath } +} + +function addFakeNpmVersion(manifestPath, entry) { + const packages = JSON.parse(fs.readFileSync(manifestPath, 'utf8')) + packages.push(entry) + fs.writeFileSync(manifestPath, JSON.stringify(packages)) +} + +function withFakeNpm(sb, binDir) { + return { ...sb.env, PATH: binDir + path.delimiter + sb.env.PATH } +} + function addCapPlugin(sb, name, capabilities, extra = {}) { const plugin = makePluginRepo(sb.root, name, { manifest: { capabilities }, ...extra }) const r = run(['add', 'file://' + plugin, '--allow-any-source', '--yes'], sb.env, sb.root) @@ -1583,3 +1690,171 @@ test('an unknown actor is refused rather than defaulted', (t) => { assert.equal(env.status, 'data-fault') assert.match(env.problems[0].reason, /"actor" must be one of program, agent, human/) }) + +// ---------------------------------------------------------------- npm: source + +test('add npm: a default-org scoped package is fetched, pinning version and dist.integrity', (t) => { + const sb = installed(t) + const pkg = fakeNpmPackage( + sb.root, + '@img2threejs/plugin-fake', + '0.1.0', + 'sha512-abc123def456==', + npmPluginFiles('plugin-fake', '0.1.0'), + ) + const { binDir } = startFakeNpm(sb.root, [pkg]) + const env = withFakeNpm(sb, binDir) + + const r = run(['add', 'npm:@img2threejs/plugin-fake', '--yes'], env, sb.root) + assert.equal(r.status, 0, r.stderr + r.stdout) + + const row = JSON.parse(fs.readFileSync(path.join(sb.H, 'plugins.json'), 'utf8')).plugins[0] + assert.equal(row.id, 'plugin-fake') + assert.equal(row.repo, 'npm:@img2threejs/plugin-fake') + assert.equal(row.ref, '0.1.0') + assert.equal(row.resolvedSha, 'sha512-abc123def456==') + + const dest = path.join(sb.H, 'plugins', 'plugin-fake') + assert.ok(fs.existsSync(path.join(dest, 'plugin.json'))) + assert.ok(!fs.existsSync(path.join(dest, '.git')), 'npm fetch must not leave a git checkout behind') + const link = path.join(sb.HOME, '.claude', 'skills', 'img2-plugin-fake') + assert.equal(fs.realpathSync(link), fs.realpathSync(dest)) + + const doctor = run(['doctor'], env, sb.root) + assert.equal(doctor.status, 0, 'a non-git plugin dir must not be held to the .gitignore rule: ' + doctor.stdout + doctor.stderr) + + const list = run(['list'], env, sb.root) + // shortSha shows the 7 chars AFTER "sha512-" (the prefix itself would be uninformative), not the + // literal prefix -- "sha512-abc123def456==" -> "abc123d". + assert.match(list.stdout, /plugin-fake\s+0\.1\.0\s+0\.1\.0\s+abc123d/, list.stdout) +}) + +test('add npm: an explicit @version pins that release', (t) => { + const sb = installed(t) + const older = fakeNpmPackage(sb.root, '@img2threejs/plugin-pin', '0.1.0', 'sha512-old==', npmPluginFiles('plugin-pin', '0.1.0')) + const newer = fakeNpmPackage(sb.root, '@img2threejs/plugin-pin', '0.2.0', 'sha512-new==', npmPluginFiles('plugin-pin', '0.2.0')) + const { binDir } = startFakeNpm(sb.root, [older, newer]) + const env = withFakeNpm(sb, binDir) + + const r = run(['add', 'npm:@img2threejs/plugin-pin@0.1.0', '--yes'], env, sb.root) + assert.equal(r.status, 0, r.stderr + r.stdout) + const row = JSON.parse(fs.readFileSync(path.join(sb.H, 'plugins.json'), 'utf8')).plugins[0] + assert.equal(row.ref, '0.1.0') + assert.equal(row.resolvedSha, 'sha512-old==') +}) + +test('add npm: an unscoped or non-default-scope package needs --allow-any-source, printing the same confirmation as a git source', (t) => { + const sb = installed(t) + const pkg = fakeNpmPackage(sb.root, 'some-pkg', '1.0.0', 'sha512-xyz==', npmPluginFiles('some-pkg', '1.0.0')) + const { binDir } = startFakeNpm(sb.root, [pkg]) + const env = withFakeNpm(sb, binDir) + + let r = run(['add', 'npm:some-pkg'], env, sb.root) + assert.equal(r.status, 2, 'expected refusal, got: ' + r.stderr + r.stdout) + assert.match(r.stderr, /allow-any-source/) + assert.deepEqual(JSON.parse(fs.readFileSync(path.join(sb.H, 'plugins.json'), 'utf8')).plugins, []) + + r = run(['add', 'npm:some-pkg', '--allow-any-source'], env, sb.root) + assert.equal(r.status, 3, 'expected NEEDS_INPUT: ' + r.stderr + r.stdout) + + r = run(['add', 'npm:some-pkg', '--allow-any-source', '--yes'], env, sb.root) + assert.equal(r.status, 0, r.stderr + r.stdout) + assert.match(r.stdout, /about to clone and link a non-img2threejs source: npm:some-pkg/) + assert.equal(JSON.parse(fs.readFileSync(path.join(sb.H, 'plugins.json'), 'utf8')).plugins.length, 1) +}) + +// The fake registry's `npm` always writes a warning to stderr (mirroring a real npm run under +// NODE_TLS_REJECT_UNAUTHORIZED=0) on every call, including the successful ones this add makes -- +// exercising that a non-empty stderr from a subprocess that exits 0 is not itself a failure. +test('add npm: a non-empty stderr from npm on an otherwise successful call is not a failure', (t) => { + const sb = installed(t) + const pkg = fakeNpmPackage(sb.root, '@img2threejs/plugin-noisy', '0.1.0', 'sha512-noisy==', npmPluginFiles('plugin-noisy', '0.1.0')) + const { binDir } = startFakeNpm(sb.root, [pkg]) + const env = withFakeNpm(sb, binDir) + + const r = run(['add', 'npm:@img2threejs/plugin-noisy', '--yes'], env, sb.root) + assert.equal(r.status, 0, r.stderr + r.stdout) +}) + +test('add npm: npm missing from PATH is a clear failure, not a crash', (t) => { + const sb = installed(t) + const r = run(['add', 'npm:@img2threejs/plugin-x', '--yes'], { ...sb.env, PATH: '' }, sb.root) + assert.equal(r.status, 1, r.stderr + r.stdout) + assert.match(r.stderr, /npm is not on PATH/) + assert.deepEqual(JSON.parse(fs.readFileSync(path.join(sb.H, 'plugins.json'), 'utf8')).plugins, []) +}) + +test('update: an npm row fetches a newer version, backs up the old clone, and updates ref/resolvedSha', (t) => { + const sb = installed(t) + const v1 = fakeNpmPackage(sb.root, '@img2threejs/plugin-up', '0.1.0', 'sha512-v1==', npmPluginFiles('plugin-up', '0.1.0')) + const { binDir, manifestPath } = startFakeNpm(sb.root, [v1]) + const env = withFakeNpm(sb, binDir) + + let r = run(['add', 'npm:@img2threejs/plugin-up', '--yes'], env, sb.root) + assert.equal(r.status, 0, r.stderr + r.stdout) + + let check = run(['update', '--check'], env, sb.root) + assert.equal(check.status, 0, 'nothing pending yet: ' + check.stdout + check.stderr) + assert.match(check.stdout, /up to date/) + + const v2 = fakeNpmPackage(sb.root, '@img2threejs/plugin-up', '0.2.0', 'sha512-v2==', npmPluginFiles('plugin-up', '0.2.0')) + addFakeNpmVersion(manifestPath, v2) + + check = run(['update', '--check'], env, sb.root) + assert.equal(check.status, 1, 'a pending update must be non-zero: ' + check.stdout) + assert.match(check.stdout, /update avail\s+plugin-up 0\.1\.0 -> 0\.2\.0/) + // --check must not have touched anything + assert.equal(JSON.parse(fs.readFileSync(path.join(sb.H, 'plugins.json'), 'utf8')).plugins[0].ref, '0.1.0') + + r = run(['update'], env, sb.root) + assert.equal(r.status, 0, r.stderr + r.stdout) + assert.match(r.stdout, /updated\s+plugin-up -> 0\.2\.0/) + + const row = JSON.parse(fs.readFileSync(path.join(sb.H, 'plugins.json'), 'utf8')).plugins[0] + assert.equal(row.ref, '0.2.0') + assert.equal(row.resolvedSha, 'sha512-v2==') + assert.ok(fs.readdirSync(path.join(sb.H, 'backups')).some((b) => b.startsWith('plugin-up-'))) + const manifest = JSON.parse(fs.readFileSync(path.join(sb.H, 'plugins', 'plugin-up', 'plugin.json'), 'utf8')) + assert.equal(manifest.version, '0.2.0') + + const doctor = run(['doctor'], env, sb.root) + assert.equal(doctor.status, 0, doctor.stdout + doctor.stderr) + + const again = run(['update'], env, sb.root) + assert.equal(again.status, 0, again.stderr + again.stdout) + assert.match(again.stdout, /update: 0 updated/) +}) + +test('update: a --link row is reported as local and left untouched', (t) => { + const sb = installed(t) + const local = makePluginRepo(sb.root, 'dev-plugin') + const r = run(['add', '--link', local], sb.env, sb.root) + assert.equal(r.status, 0, r.stderr + r.stdout) + + const update = run(['update'], sb.env, sb.root) + assert.equal(update.status, 0, update.stdout + update.stderr) + assert.match(update.stdout, /local\s+dev-plugin \(link:/) + assert.equal(fs.realpathSync(path.join(sb.H, 'plugins', 'dev-plugin')), fs.realpathSync(local)) +}) + +test('update: a git row updates to the newest reachable tag', (t) => { + const sb = installed(t) + const plugin = makePluginRepo(sb.root, 'git-up', { tag: 'v0.1.0' }) + let r = run(['add', 'file://' + plugin, '--allow-any-source', '--yes'], sb.env, sb.root) + assert.equal(r.status, 0, r.stderr + r.stdout) + + const doc = JSON.parse(fs.readFileSync(path.join(plugin, 'plugin.json'), 'utf8')) + doc.version = '0.2.0' + fs.writeFileSync(path.join(plugin, 'plugin.json'), JSON.stringify(doc, null, 2) + '\n') + gitq(['add', '-A'], plugin) + gitq(['-c', 'user.name=img2-test', '-c', 'user.email=test@img2.invalid', 'commit', '-q', '-m', 'bump'], plugin) + gitq(['tag', 'v0.2.0'], plugin) + + r = run(['update', '--allow-any-source', '--yes'], sb.env, sb.root) + assert.equal(r.status, 0, r.stderr + r.stdout) + assert.match(r.stdout, /updated\s+git-up -> v0\.2\.0/) + + const row = JSON.parse(fs.readFileSync(path.join(sb.H, 'plugins.json'), 'utf8')).plugins[0] + assert.equal(row.ref, 'v0.2.0') + assert.match(row.resolvedSha, /^[0-9a-f]{40}$/) +}) diff --git a/tests/helpers.test.mjs b/tests/helpers.test.mjs index 4e7a5de..9dfb505 100644 --- a/tests/helpers.test.mjs +++ b/tests/helpers.test.mjs @@ -13,9 +13,11 @@ import { emptyRegistry, findRow, mergeAdditionalDirectories, + parseNpmSpec, parseSemver, rangeSatisfied, removeRow, + resolveNpmSource, resolveSource, shlexSplit, stepArgv, @@ -71,6 +73,23 @@ test('resolveSource enforces the default org', () => { assert.equal(resolveSource('file:///tmp/x', true).url, 'file:///tmp/x') }) +test('parseNpmSpec splits name and version, treating a scope\'s own "@" as part of the name', () => { + assert.deepEqual(parseNpmSpec('npm:@img2threejs/plugin-cs2'), { name: '@img2threejs/plugin-cs2', version: null }) + assert.deepEqual(parseNpmSpec('npm:@img2threejs/plugin-cs2@0.1.2'), { name: '@img2threejs/plugin-cs2', version: '0.1.2' }) + assert.deepEqual(parseNpmSpec('npm:some-pkg'), { name: 'some-pkg', version: null }) + assert.deepEqual(parseNpmSpec('npm:some-pkg@1.0.0'), { name: 'some-pkg', version: '1.0.0' }) + throwsCli(() => parseNpmSpec('npm:'), EXIT.REFUSED, /needs a package name/) + throwsCli(() => parseNpmSpec('npm:some-pkg@'), EXIT.REFUSED, /empty version/) +}) + +test('resolveNpmSource enforces the default @img2threejs scope', () => { + assert.deepEqual(resolveNpmSource('@img2threejs/plugin-cs2', false), { defaultOrg: true }) + assert.equal(resolveNpmSource('@evil/plugin-x', true).defaultOrg, false) + throwsCli(() => resolveNpmSource('@evil/plugin-x', false), EXIT.REFUSED, /allow-any-source/) + throwsCli(() => resolveNpmSource('unscoped-pkg', false), EXIT.REFUSED, /allow-any-source/) + assert.equal(resolveNpmSource('unscoped-pkg', true).defaultOrg, false) +}) + test('mergeAdditionalDirectories is idempotent and preserves unknown keys', () => { const first = mergeAdditionalDirectories({ theme: 'dark', permissions: { allow: ['Bash'] } }, '/x/.img2') assert.equal(first.changed, true)