From d1989b32826b669dfd3c053a2cf1bbe6b1e8b1e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Mon, 9 Nov 2026 08:34:21 +0000 Subject: [PATCH 01/18] ci(release): post to Slack when a released host needs an upload by hand --- .github/workflows/release.yml | 20 +++++- .github/workflows/upload-reminder.yml | 100 ++++++++++++++++++++++++++ CONTRIBUTING.md | 11 ++- platforms/copilot/vars.yaml | 4 ++ scripts/new-host.mjs | 4 ++ scripts/validate.mjs | 12 ++++ 6 files changed, 149 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/upload-reminder.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8827416..e1d97a0 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -170,7 +170,8 @@ jobs: gh release edit "$TAG" --repo "$GITHUB_REPOSITORY" --title "$TAG" --notes-file - # Steps that publish a host somewhere else go here, selected by component, - # for example: if: matrix.component == 'openai'. Mirror repos are the mirror job below. + # for example: if: matrix.component == 'openai'. Mirror repos are the mirror + # job below; uploads done by hand get a Slack reminder from upload-reminder. - name: Write the summary run: | @@ -198,3 +199,20 @@ jobs: host: ${{ matrix.component }} tag: ${{ matrix.tag }} secrets: inherit + + # Each released host with upload: in its vars.yaml gets a reminder in + # #proj-integrations to upload it by hand; for the others this is a no-op. + upload-reminder: + name: Upload reminder ${{ matrix.component }} + needs: [detect, release] + permissions: + contents: read + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.detect.outputs.packages) }} + uses: ./.github/workflows/upload-reminder.yml + with: + host: ${{ matrix.component }} + tag: ${{ matrix.tag }} + secrets: inherit diff --git a/.github/workflows/upload-reminder.yml b/.github/workflows/upload-reminder.yml new file mode 100644 index 0000000..7764ef0 --- /dev/null +++ b/.github/workflows/upload-reminder.yml @@ -0,0 +1,100 @@ +name: Upload reminder + +# Posts to #proj-integrations that a release has to be uploaded by hand, for a +# host with upload: in platforms//vars.yaml (for example a marketplace +# that reviews each version). release.yml calls it for each released host. Run +# it by hand to send a reminder again; a manual run is a dry run unless you untick it. + +on: + workflow_call: + inputs: + host: + type: string + required: true + tag: + description: "The release, -v" + type: string + required: true + workflow_dispatch: + inputs: + host: + description: "Host, for example openai" + type: string + required: true + tag: + description: "The release, for example openai-v0.1.9" + type: string + required: true + dry_run: + description: "Show the message without posting it" + type: boolean + default: true + +permissions: {} + +jobs: + remind: + name: Upload reminder ${{ inputs.host }} + runs-on: ubuntu-latest + # The release environment deploys only from main and holds the webhook. + environment: release + permissions: + contents: read + env: + HOST: ${{ inputs.host }} + TAG: ${{ inputs.tag }} + # workflow_call has no dry_run input, so a release always posts. + DRY_RUN: ${{ inputs.dry_run == true }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + ref: refs/tags/${{ inputs.tag }} + persist-credentials: false + - uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0 + with: + node-version: "22" + cache: npm + - run: npm ci + + - name: Post the reminder + env: + # A Slack incoming webhook for #proj-integrations. + WEBHOOK: ${{ secrets.SLACK_WEBHOOK_INTEGRATIONS }} + run: | + set -euo pipefail + upload=$(node --input-type=module -e ' + import { vars } from "./scripts/lib.mjs"; + const v = vars(process.env.HOST); + if (`${process.env.HOST}-v${v.version}` !== process.env.TAG) { + console.error(`::error::${process.env.TAG} is not a ${process.env.HOST} release: at that tag, ${process.env.HOST} is at ${v.version}.`); + process.exit(1); + } + console.log(JSON.stringify(v.upload ?? false));') + if [ "$upload" = false ]; then + echo "::notice::$HOST has upload: false in platforms/$HOST/vars.yaml, so there's nothing to upload by hand." + exit 0 + fi + + repo="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" + payload=$(jq -n --argjson upload "$upload" --arg host "$HOST" --arg tag "$TAG" --arg repo "$repo" '{ + text: (":package: *\($tag)* is released and needs an upload to *\($upload.to)*.\n" + + "Files: <\($repo)/tree/\($tag)/dist/\($host)|dist/\($host) at \($tag)>" + + " · Upload: <\($upload.url)|\($upload.to)>" + + " · Release: <\($repo)/releases/tag/\($tag)|\($tag)>") + }') + { + echo "## $HOST upload reminder" + jq -r .text <<<"$payload" + } >>"$GITHUB_STEP_SUMMARY" + + if [ "$DRY_RUN" = true ]; then + echo "Dry run: #proj-integrations would get this message." + jq . <<<"$payload" + exit 0 + fi + if [ -z "$WEBHOOK" ]; then + echo "::warning::No SLACK_WEBHOOK_INTEGRATIONS secret in the release environment, so the reminder wasn't posted to Slack. It's in the job summary: upload $TAG to $(jq -r .to <<<"$upload")." + exit 0 + fi + curl -fsS -X POST -H 'Content-type: application/json' --data "$payload" "$WEBHOOK" >/dev/null + echo "::notice::Posted the $TAG upload reminder to #proj-integrations." diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d66be52..0505edd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,7 +55,7 @@ Run `npm run new-host`. It asks for the host id (`platforms//`, `dist//` It also adds the host to `release-please-config.json` and `.release-please-manifest.json` at `0.1.0`. Answers can also be piped in, one per line. -What only a person can decide is marked `TODO(new-host)`, and `npm run validate` fails until it's done: the install and sign-in steps, where the agent can write files, and the plugin files. Each host reads its own layout (Claude: `.claude-plugin/plugin.json` and `.mcp.json`; Cursor: `.cursor-plugin/plugin.json`), so move the templates to those paths, add the fields the host needs, update the manifest path in `release-please-config.json`, and add a marketplace catalog at the top of the repo if the host installs from a subfolder. Use an existing `platforms//` as the example. Then run `npm run render` and commit `platforms//`, `dist//`, the release-please files and `README.md` together. The render adds the host to the "Supported hosts" list in `README.md`, so don't add it by hand. +What only a person can decide is marked `TODO(new-host)`, and `npm run validate` fails until it's done: the install and sign-in steps, where the agent can write files, whether releases are uploaded by hand (`upload:`), and the plugin files. Each host reads its own layout (Claude: `.claude-plugin/plugin.json` and `.mcp.json`; Cursor: `.cursor-plugin/plugin.json`), so move the templates to those paths, add the fields the host needs, update the manifest path in `release-please-config.json`, and add a marketplace catalog at the top of the repo if the host installs from a subfolder. Use an existing `platforms//` as the example. Then run `npm run render` and commit `platforms//`, `dist//`, the release-please files and `README.md` together. The render adds the host to the "Supported hosts" list in `README.md`, so don't add it by hand. ## Commits and releases @@ -71,3 +71,12 @@ Some installs name a repo of their own instead of a folder here: `copilot plugin - It pushes with the deploy key in the `MIRROR_KEY_` secret of the `release` environment. The key, the mirror's branch protection and its settings are in [iac-bootstrap](https://github.com/ZenRows/iac-bootstrap). - To see what the mirror would get: `npm run mirror -- []`, then `git diff` in the clone. To publish by hand (for example after fixing a failed run), run the Mirror workflow from Actions with the host and the release tag; it's a dry run unless you untick it. With no tag it publishes `main`'s `dist//`, names the commit in the README banner and tags nothing. - Manual runs, dry runs included, start from `main`: the job uses the `release` environment, which only deploys from `main`. + +### Uploads by hand + +Some hosts take each release only as an upload that someone does by hand, for example a marketplace that reviews every version. Every host's `vars.yaml` says which: + +- `upload: false`: installs read this repo or the host's mirror, so nothing is uploaded. +- `upload:` with `to:` (where, for example `Cursor Marketplace`) and `url:` (an `https://` link to the page you upload on): each release of the host posts a reminder to #proj-integrations naming the release, its files in `dist//` at the tag, and that link. + +`npm run validate` fails if a host has no `upload:`. `.github/workflows/upload-reminder.yml` posts through a Slack incoming webhook in the `SLACK_WEBHOOK_INTEGRATIONS` secret of the `release` environment. Without the secret, the run warns and leaves the message in its job summary. To send a reminder again, run the Upload reminder workflow from Actions, on `main`, with the host and the release tag; it's a dry run unless you untick it. diff --git a/platforms/copilot/vars.yaml b/platforms/copilot/vars.yaml index 4ef2ba9..ad61669 100644 --- a/platforms/copilot/vars.yaml +++ b/platforms/copilot/vars.yaml @@ -9,3 +9,7 @@ skill_prefix: "" # Published to this repo too on each release, for installs that name it # (copilot plugin marketplace add ZenRows/copilot-zenrows). See CONTRIBUTING.md. mirror: ZenRows/copilot-zenrows +# Where each release is uploaded by hand, if anywhere: to: , url: . +# The release workflow posts it to #proj-integrations. false: installs read +# this repo or its mirror, so nothing is uploaded. See CONTRIBUTING.md. +upload: false diff --git a/scripts/new-host.mjs b/scripts/new-host.mjs index 3608378..748ab4f 100644 --- a/scripts/new-host.mjs +++ b/scripts/new-host.mjs @@ -65,6 +65,10 @@ files.set( "# scrape-webpage: zenrows-scrape", "# To leave a shared skill out on this host, list it here, for example:", "# skip_skills: [using-zenrows]", + "# Where each release is uploaded by hand, if anywhere: to: , url: .", + "# The release workflow posts it to #proj-integrations. false: installs read", + "# this repo or its mirror, so nothing is uploaded. See CONTRIBUTING.md.", + 'upload: "TODO(new-host): false, or to: and url:"', "", ].join("\n"), ); diff --git a/scripts/validate.mjs b/scripts/validate.mjs index 97aeec3..056e718 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -188,6 +188,18 @@ for (const host of hosts()) { } } + // Every host says whether its releases are uploaded somewhere by hand, so the + // release workflow knows whether to post a reminder. + const upload = values.upload; + if (upload === undefined) { + errors.push(`${host}: vars.yaml has no upload: (false, or to: and url:; see CONTRIBUTING.md)`); + } else if ( + upload !== false && + !(typeof upload?.to === "string" && upload.to.trim() && typeof upload.url === "string" && upload.url.startsWith("https://")) + ) { + errors.push(`${host}: vars.yaml upload: must be false, or to: and url: `); + } + // A whole-file override stops receiving fixes to the shared file it copied. const lock = readLock(host); for (const [shared, hash] of Object.entries(lock)) { From 888082094be4aacd64df92c74ba36a2bd963ee3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Mon, 9 Nov 2026 08:48:07 +0000 Subject: [PATCH 02/18] ci(release): build the reminder's tag check without shell-like expansions --- .github/workflows/upload-reminder.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/upload-reminder.yml b/.github/workflows/upload-reminder.yml index 7764ef0..5ae4767 100644 --- a/.github/workflows/upload-reminder.yml +++ b/.github/workflows/upload-reminder.yml @@ -64,9 +64,10 @@ jobs: set -euo pipefail upload=$(node --input-type=module -e ' import { vars } from "./scripts/lib.mjs"; - const v = vars(process.env.HOST); - if (`${process.env.HOST}-v${v.version}` !== process.env.TAG) { - console.error(`::error::${process.env.TAG} is not a ${process.env.HOST} release: at that tag, ${process.env.HOST} is at ${v.version}.`); + const { HOST, TAG } = process.env; + const v = vars(HOST); + if (HOST + "-v" + v.version !== TAG) { + console.error("::error::" + TAG + " is not a " + HOST + " release: at that tag, " + HOST + " is at " + v.version + "."); process.exit(1); } console.log(JSON.stringify(v.upload ?? false));') From 01c0ef70ccfcc65748162f347de0f52e40f942c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Mon, 9 Nov 2026 13:57:04 +0000 Subject: [PATCH 03/18] ci(release): tag @ZenGrow in the upload reminder and fail without the webhook --- .github/workflows/upload-reminder.yml | 12 ++++++++---- CONTRIBUTING.md | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/.github/workflows/upload-reminder.yml b/.github/workflows/upload-reminder.yml index 5ae4767..4b4acbe 100644 --- a/.github/workflows/upload-reminder.yml +++ b/.github/workflows/upload-reminder.yml @@ -60,6 +60,8 @@ jobs: env: # A Slack incoming webhook for #proj-integrations. WEBHOOK: ${{ secrets.SLACK_WEBHOOK_INTEGRATIONS }} + # The @ZenGrow user group, which owns the integrations. + MENTION: "" run: | set -euo pipefail upload=$(node --input-type=module -e ' @@ -77,8 +79,8 @@ jobs: fi repo="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" - payload=$(jq -n --argjson upload "$upload" --arg host "$HOST" --arg tag "$TAG" --arg repo "$repo" '{ - text: (":package: *\($tag)* is released and needs an upload to *\($upload.to)*.\n" + payload=$(jq -n --argjson upload "$upload" --arg mention "$MENTION" --arg host "$HOST" --arg tag "$TAG" --arg repo "$repo" '{ + text: ("\($mention) :package: *\($tag)* is released and needs an upload to *\($upload.to)*.\n" + "Files: <\($repo)/tree/\($tag)/dist/\($host)|dist/\($host) at \($tag)>" + " · Upload: <\($upload.url)|\($upload.to)>" + " · Release: <\($repo)/releases/tag/\($tag)|\($tag)>") @@ -93,9 +95,11 @@ jobs: jq . <<<"$payload" exit 0 fi + # Fail rather than warn: the release is already out, and a red job is + # harder to miss than a reminder nobody received. if [ -z "$WEBHOOK" ]; then - echo "::warning::No SLACK_WEBHOOK_INTEGRATIONS secret in the release environment, so the reminder wasn't posted to Slack. It's in the job summary: upload $TAG to $(jq -r .to <<<"$upload")." - exit 0 + echo "::error::No SLACK_WEBHOOK_INTEGRATIONS secret in the release environment, so the reminder wasn't posted to Slack. It's in the job summary: upload $TAG to $(jq -r .to <<<"$upload")." + exit 1 fi curl -fsS -X POST -H 'Content-type: application/json' --data "$payload" "$WEBHOOK" >/dev/null echo "::notice::Posted the $TAG upload reminder to #proj-integrations." diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0505edd..dd9faeb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -79,4 +79,4 @@ Some hosts take each release only as an upload that someone does by hand, for ex - `upload: false`: installs read this repo or the host's mirror, so nothing is uploaded. - `upload:` with `to:` (where, for example `Cursor Marketplace`) and `url:` (an `https://` link to the page you upload on): each release of the host posts a reminder to #proj-integrations naming the release, its files in `dist//` at the tag, and that link. -`npm run validate` fails if a host has no `upload:`. `.github/workflows/upload-reminder.yml` posts through a Slack incoming webhook in the `SLACK_WEBHOOK_INTEGRATIONS` secret of the `release` environment. Without the secret, the run warns and leaves the message in its job summary. To send a reminder again, run the Upload reminder workflow from Actions, on `main`, with the host and the release tag; it's a dry run unless you untick it. +`npm run validate` fails if a host has no `upload:`. `.github/workflows/upload-reminder.yml` posts through a Slack incoming webhook in the `SLACK_WEBHOOK_INTEGRATIONS` secret of the `release` environment and tags @ZenGrow. Without the secret, the job fails, so a missed reminder shows as a red run; the message is still in its job summary. To send a reminder again, run the Upload reminder workflow from Actions, on `main`, with the host and the release tag; it's a dry run unless you untick it. From c7269bd5493532b48508a03dd6b1d0f97f54a534 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Mon, 9 Nov 2026 18:41:23 +0000 Subject: [PATCH 04/18] build: let a host keep its own validate and package scripts platforms//scripts/ is now reserved like skills/: render never copies it into dist/. npm run validate runs platforms//scripts/validate.mjs on dist/ when it exists, and npm run package -- runs its package.sh. --- CONTRIBUTING.md | 4 +++- package.json | 1 + scripts/lib.mjs | 5 +++-- scripts/package.mjs | 18 ++++++++++++++++++ scripts/validate.mjs | 17 +++++++++++++++++ 5 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 scripts/package.mjs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d66be52..ce0173b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -11,10 +11,12 @@ platforms// vars.yaml values for {{var}} skills//.md the host's text for a {{> slot}} in that skill overrides.lock whole-file overrides and the shared version each was copied from (only once one exists) + scripts/ the host's own tooling, never shipped: validate.mjs (run by npm run validate on + dist/) and package.sh (run by npm run package -- ), when the host needs them everything else the host's manifests, README, …, copied into dist// ({{version}} comes from release-please) dist// GENERATED: the plugin the host installs .github/plugin/marketplace.json Copilot's marketplace catalog (see below) -scripts/ render, validate, override, new-host, mirror +scripts/ render, validate, override, new-host, package, mirror ``` ### Marketplace catalogs diff --git a/package.json b/package.json index 2ec4b1b..4cfa237 100644 --- a/package.json +++ b/package.json @@ -12,6 +12,7 @@ "override": "node scripts/override.mjs create", "override:ack": "node scripts/override.mjs ack", "new-host": "node scripts/new-host.mjs", + "package": "node scripts/package.mjs", "mirror": "node scripts/mirror.mjs" }, "devDependencies": { diff --git a/scripts/lib.mjs b/scripts/lib.mjs index f607988..c8b03cf 100644 --- a/scripts/lib.mjs +++ b/scripts/lib.mjs @@ -14,9 +14,10 @@ export const distDir = join(root, "dist"); // Files copied into every host's plugin as they are. const sharedFiles = { LICENSE: "LICENSE" }; const sharedDirs = { "shared/assets": "assets" }; -// In platforms//, these are inputs to the render, not files of the plugin. +// In platforms//, these are inputs to the render or the host's own tooling +// (scripts/validate.mjs, scripts/package.sh), not files of the plugin. // Everything else there (manifests, README, …) is copied into dist//. -const reserved = ["vars.yaml", "overrides.lock", "skills/"]; +const reserved = ["vars.yaml", "overrides.lock", "skills/", "scripts/"]; // {{vars}} are filled in files with these extensions; others are copied as bytes. const textFile = /\.(json|md|mdc|ya?ml|txt|toml)$/; diff --git a/scripts/package.mjs b/scripts/package.mjs new file mode 100644 index 0000000..274068f --- /dev/null +++ b/scripts/package.mjs @@ -0,0 +1,18 @@ +// npm run package -- : builds what a host uploads by hand (for example +// OpenAI's ZIP) by running platforms//scripts/package.sh. +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { hosts, platformsDir } from "./lib.mjs"; + +const host = process.argv[2]; +if (!host || !hosts().includes(host)) { + console.error(`Usage: npm run package -- , one of: ${hosts().join(", ")}`); + process.exit(1); +} +const script = join(platformsDir, host, "scripts", "package.sh"); +if (!existsSync(script)) { + console.log(`${host} needs no packaging: it installs from dist/${host}/ (no platforms/${host}/scripts/package.sh).`); + process.exit(0); +} +process.exit(spawnSync("bash", [script], { stdio: "inherit" }).status ?? 1); diff --git a/scripts/validate.mjs b/scripts/validate.mjs index 97aeec3..aa11a4c 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -1,6 +1,7 @@ // Checks what each host ships: skill frontmatter, skill references, manifests // (against the JSON Schema they declare), the marketplace catalogs at the top of // the repo, and that no whole-file override was copied from an older shared file. +import { spawnSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { basename, dirname, join } from "node:path"; import Ajv2020 from "ajv/dist/2020.js"; @@ -217,6 +218,22 @@ for (const host of hosts()) { if (text.includes("TODO(new-host)")) errors.push(`${host}: platforms/${host}/${file} still has a TODO(new-host)`); } + // platforms//scripts/ is the host's tooling and never ships, so only the + // two files the repo runs belong there. + for (const file of listFiles(join(platformsDir, host, "scripts"))) { + if (!["validate.mjs", "package.sh"].includes(file)) { + errors.push(`${host}: platforms/${host}/scripts/${file} isn't shipped (scripts/ holds only validate.mjs and package.sh); move plugin files elsewhere`); + } + } + + // Checks only one host needs, such as OpenAI's submission limits, live in + // platforms//scripts/validate.mjs and run on the committed dist/. + const extra = join(platformsDir, host, "scripts", "validate.mjs"); + if (existsSync(extra)) { + const run = spawnSync(process.execPath, [extra, join(root, "dist", host)], { encoding: "utf8" }); + if (run.status !== 0) errors.push(`${host}: platforms/${host}/scripts/validate.mjs failed:\n${(run.stderr || run.stdout).trim()}`); + } + // How much skill text the host owns instead of sharing: its slot files and // whole-file overrides. Information only; a growing number is worth a look. const owned = listFiles(join(platformsDir, host, "skills")) From 020778b3f851bd86254473ff787d45cb0ee73fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Mon, 9 Nov 2026 18:46:40 +0000 Subject: [PATCH 05/18] feat(render): shared text snippets and code hosts shared/text/.md holds one fact in plain text. A skill inlines it with {{< name}}; code uses {{text:name}}, a quoted string literal. using-zenrows and extract-structured-data now take AUTH004, AUTH014, AUTH006 and REQS007's page_not_recognised from there; every plugin renders byte-identically. Validate fails on a snippet with markdown, more than one line, or no user. kind: code in vars.yaml makes a host whose output isn't a plugin (code for another project, or a marketplace listing): it ships only its own files, with {{...}} filled in .ts/.js too, no skills, LICENSE, assets or plugin.json, keeps its README out of dist, needs no release-please entry yet, and may keep a scripts/check.sh. --- CONTRIBUTING.md | 20 ++++++++--- scripts/lib.mjs | 40 ++++++++++++++++++---- scripts/render.mjs | 7 ++-- scripts/validate.mjs | 30 +++++++++++++--- shared/text/auth004.md | 1 + shared/text/auth006.md | 1 + shared/text/auth014.md | 1 + shared/text/reqs007-page-not-recognised.md | 1 + skills/extract-structured-data/SKILL.md | 2 +- skills/using-zenrows/SKILL.md | 4 +-- 10 files changed, 86 insertions(+), 21 deletions(-) create mode 100644 shared/text/auth004.md create mode 100644 shared/text/auth006.md create mode 100644 shared/text/auth014.md create mode 100644 shared/text/reqs007-page-not-recognised.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ce0173b..75d7c9f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,20 +1,22 @@ # Contributing -Each host's plugin in `dist//` is rendered from shared skills plus that host's own files. Edit the sources, never `dist/`. +Each host's plugin in `dist//` is rendered from shared skills plus that host's own files. Code hosts (a piece for another project, a marketplace listing) are rendered the same way from their own source and the shared text. Edit the sources, never `dist/`. ## Layout ``` skills//SKILL.md shared skills; {{var}} and {{> slot}} mark what differs per host -shared/assets/ files every host ships as they are (logo) +shared/assets/ files every plugin host ships as they are (logo) +shared/text/.md one fact in plain text, used by skills ({{< name}}) and code hosts ({{text:name}}) platforms// vars.yaml values for {{var}} skills//.md the host's text for a {{> slot}} in that skill overrides.lock whole-file overrides and the shared version each was copied from (only once one exists) scripts/ the host's own tooling, never shipped: validate.mjs (run by npm run validate on - dist/) and package.sh (run by npm run package -- ), when the host needs them + dist/), package.sh (run by npm run package -- ) and check.sh (a code + host's build and test, run by its CI workflow), when the host needs them everything else the host's manifests, README, …, copied into dist// ({{version}} comes from release-please) -dist// GENERATED: the plugin the host installs +dist// GENERATED: the plugin the host installs, or a code host's code .github/plugin/marketplace.json Copilot's marketplace catalog (see below) scripts/ render, validate, override, new-host, package, mirror ``` @@ -40,7 +42,15 @@ Requires Node 22. Run `npm ci` once. - **A shared skill a host shouldn't ship:** list it under `skip_skills:` in that host's `vars.yaml` (Cursor uses a rules file instead of `using-zenrows`). A `{{skill:…}}` reference to a skipped skill fails the render. - **A reference to another skill:** write `{{skill:batch}}`, never the plain name. It renders as that skill's name on the host: `skill_prefix` plus the shared name, unless `skill_names` in `vars.yaml` renames it (OpenClaw ships `scrape-webpage` as `zenrows-scrape`). The skill's own `name:` in its frontmatter uses it too. -Unknown variables, unknown skill names, missing slots and unused slot files fail the render, and `npm run validate` fails on a reference to a skill the host doesn't ship, and on a skill name written plainly instead of as `{{skill:…}}`. +Unknown variables, unknown skill names, missing slots, unused slot files and unknown snippets fail the render, and `npm run validate` fails on a reference to a skill the host doesn't ship, and on a skill name written plainly instead of as `{{skill:…}}`. + +## Text shared with code hosts + +A fact that a skill and a code host both state (what `AUTH004` means, why Extract refused a page) lives once in `shared/text/.md`, as one line of plain text. A skill inlines it with `{{< name}}`; code uses `{{text:name}}`, which renders as a quoted string, and adds its own words around it (an agent is told to use a skill, a user to check their connection). Snippets stay plain text so they read right in both, and `npm run validate` fails on markdown in one, or on one nothing uses. + +## Code hosts + +`kind: code` in `vars.yaml` makes a host whose output isn't a plugin but files for somewhere else: code for another project (a piece as it sits in that project's repo), or a listing a marketplace reads. A code host ships only its own files, with `{{…}}` filled in source files too: no shared skills, `LICENSE` or assets, and no `plugin.json`. Its `README.md` documents it here and isn't shipped. It needs no release-please entry until it's released from this repo. If it's code, its `scripts/check.sh` can build and test `dist//` the way the target project does, run by its own workflow. ## Manifest schemas diff --git a/scripts/lib.mjs b/scripts/lib.mjs index c8b03cf..c34ef00 100644 --- a/scripts/lib.mjs +++ b/scripts/lib.mjs @@ -10,6 +10,7 @@ export const root = fileURLToPath(new URL("..", import.meta.url)); export const skillsDir = join(root, "skills"); export const platformsDir = join(root, "platforms"); export const distDir = join(root, "dist"); +export const sharedTextDir = join(root, "shared", "text"); // Files copied into every host's plugin as they are. const sharedFiles = { LICENSE: "LICENSE" }; @@ -19,7 +20,7 @@ const sharedDirs = { "shared/assets": "assets" }; // Everything else there (manifests, README, …) is copied into dist//. const reserved = ["vars.yaml", "overrides.lock", "skills/", "scripts/"]; // {{vars}} are filled in files with these extensions; others are copied as bytes. -const textFile = /\.(json|md|mdc|ya?ml|txt|toml)$/; +export const textFile = /\.(json|md|mdc|ya?ml|txt|toml|[cm]?[jt]s)$/; // Errors here are for whoever edited a skill or platform file: print the // message, not a stack trace. @@ -54,8 +55,10 @@ function versions() { export function vars(host) { const v = parseYaml(readFileSync(join(platformsDir, host, "vars.yaml"), "utf8")) ?? {}; const version = versions()[`dist/${host}`]; - if (!version) throw new Error(`.release-please-manifest.json has no "dist/${host}" entry`); - return { ...v, version }; + // A code host keeps its version in its own package.json until it's released + // from here, so it needs no release-please entry yet. + if (!version && !isCodeHost(v)) throw new Error(`.release-please-manifest.json has no "dist/${host}" entry`); + return version ? { ...v, version } : v; } // overrides.lock: ": " per line; # starts a comment. @@ -69,6 +72,8 @@ export function readLock(host) { // the host's own skills (platforms//skills//SKILL.md with no shared // skill of that name). Returns [{ name, own }]. export function hostSkills(host, values = vars(host)) { + // A code host (kind: code, such as the Activepieces piece) ships no skills. + if (isCodeHost(values)) return []; const shared = skillNames(); const skip = values.skip_skills ?? []; for (const name of skip) { @@ -84,14 +89,34 @@ export function hostSkills(host, values = vars(host)) { ].sort((a, b) => a.name.localeCompare(b.name)); } +// kind: code in vars.yaml marks a host whose output is code (a package or a +// piece for another project), not a plugin: it ships only its own files, with +// {{…}} filled in its source, and no shared skills, LICENSE or assets. +export function isCodeHost(values) { + const kind = values.kind ?? "plugin"; + if (!["plugin", "code"].includes(kind)) throw new Error(`kind: ${kind} in vars.yaml; use plugin or code`); + return kind === "code"; +} + // A skill's name on a host: skill_prefix + the shared name, unless vars.yaml // renames it in skill_names (OpenClaw ships scrape-webpage as zenrows-scrape). export function hostSkillName(values, name) { return values.skill_names?.[name] ?? `${values.skill_prefix ?? ""}${name}`; } -// {{name}} is a variable from vars.yaml; {{skill:name}} is that skill's name on the host. +// shared/text/.md: one fact in plain text, used by skills and by code +// hosts alike, so a wording fix reaches both. +export function sharedText(name, where) { + const file = join(sharedTextDir, `${name}.md`); + if (!existsSync(file)) throw new Error(`${where}: there's no shared/text/${name}.md`); + return readFileSync(file, "utf8").replace(/\n$/, ""); +} + +// {{name}} is a variable from vars.yaml; {{skill:name}} is that skill's name on +// the host; {{< name}} inlines shared/text/.md, and {{text:name}} inserts it +// as a quoted string literal (JSON), for code. function fill(text, values, where) { + text = text.replace(/\{\{<\s*([a-z0-9-]+)\s*\}\}/g, (_, name) => sharedText(name, where)); return text.replace(/\{\{\s*([a-z_]+)(?::([a-z0-9-]+))?\s*\}\}/g, (token, name, arg) => { if (name === "skill") { if (!values.skills.includes(arg)) { @@ -100,6 +125,7 @@ function fill(text, values, where) { } return hostSkillName(values, arg); } + if (name === "text" && arg !== undefined) return JSON.stringify(sharedText(arg, where)); if (arg !== undefined || !(name in values) || typeof values[name] === "object") { throw new Error(`${where}: unknown variable ${token}`); } @@ -137,12 +163,14 @@ export function renderHost(host) { if (!usedSlots.has(path)) throw new Error(`${path}: no skill this host ships has this slot`); } - for (const [from, to] of Object.entries(sharedFiles)) out.set(to, readFileSync(join(root, from))); - for (const [from, to] of Object.entries(sharedDirs)) { + for (const [from, to] of isCodeHost(values) ? [] : Object.entries(sharedFiles)) out.set(to, readFileSync(join(root, from))); + for (const [from, to] of isCodeHost(values) ? [] : Object.entries(sharedDirs)) { for (const file of listFiles(join(root, from))) out.set(`${to}/${file}`, readFileSync(join(root, from, file))); } for (const file of listFiles(platform)) { if (reserved.some((r) => file === r || file.startsWith(r))) continue; + // A code host's README documents the integration here; it isn't part of the code. + if (isCodeHost(values) && file === "README.md") continue; out.set(file, renderFile(join(platform, file), values)); } return out; diff --git a/scripts/render.mjs b/scripts/render.mjs index 940d551..eff864e 100644 --- a/scripts/render.mjs +++ b/scripts/render.mjs @@ -4,7 +4,7 @@ // render. import { mkdirSync, readFileSync, rmSync, writeFileSync, existsSync } from "node:fs"; import { dirname, join } from "node:path"; -import { distDir, hosts, listFiles, renderHost, root, vars } from "./lib.mjs"; +import { distDir, hosts, isCodeHost, listFiles, renderHost, root, vars } from "./lib.mjs"; const check = process.argv.includes("--check"); const only = process.argv.slice(2).filter((a) => !a.startsWith("--")); @@ -26,7 +26,10 @@ function hostsList() { const rows = hosts() .map((host) => [vars(host).host_full ?? host, host]) .sort(([a], [b]) => a.localeCompare(b)) - .map(([name, host]) => `| ${name} | [dist/${host}/README.md](dist/${host}/README.md) |`); + .map(([name, host]) => { + const guide = isCodeHost(vars(host)) ? `platforms/${host}/README.md` : `dist/${host}/README.md`; + return `| ${name} | [${guide}](${guide}) |`; + }); const table = ["| Host | Setup guide |", "|---|---|", ...rows].join("\n"); const head = readme.slice(0, readme.indexOf("\n", start) + 1); return Buffer.from(`${head}${table}\n${readme.slice(end)}`); diff --git a/scripts/validate.mjs b/scripts/validate.mjs index aa11a4c..e68c3b0 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -6,7 +6,7 @@ import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { basename, dirname, join } from "node:path"; import Ajv2020 from "ajv/dist/2020.js"; import { parse as parseYaml } from "yaml"; -import { catalogs, hostSkillName, hostSkills, hosts, listFiles, platformsDir, readLock, renderHost, root, sha256, skillNames, vars } from "./lib.mjs"; +import { catalogs, hostSkillName, hostSkills, hosts, isCodeHost, listFiles, platformsDir, readLock, renderHost, root, sha256, sharedTextDir, skillNames, textFile, vars } from "./lib.mjs"; const errors = []; const report = []; @@ -158,7 +158,7 @@ for (const host of hosts()) { if (!json[key]) errors.push(`${host}: ${file} has no "${key}"`); } } - if (!manifests.length) errors.push(`${host}: no plugin.json in platforms/${host}/`); + if (!manifests.length && !isCodeHost(values)) errors.push(`${host}: no plugin.json in platforms/${host}/`); // Marketplace catalogs at the top of the repo must point at this host's // dist/ folder, with this host's version. @@ -219,10 +219,11 @@ for (const host of hosts()) { } // platforms//scripts/ is the host's tooling and never ships, so only the - // two files the repo runs belong there. + // files the repo runs belong there: validate.mjs, package.sh, and check.sh (a + // code host's build and test, run by its CI workflow). for (const file of listFiles(join(platformsDir, host, "scripts"))) { - if (!["validate.mjs", "package.sh"].includes(file)) { - errors.push(`${host}: platforms/${host}/scripts/${file} isn't shipped (scripts/ holds only validate.mjs and package.sh); move plugin files elsewhere`); + if (!["validate.mjs", "package.sh", "check.sh"].includes(file)) { + errors.push(`${host}: platforms/${host}/scripts/${file} isn't shipped (scripts/ holds only validate.mjs, package.sh and check.sh); move plugin files elsewhere`); } } @@ -242,6 +243,25 @@ for (const host of hosts()) { report.push(`${host}: ${Object.keys(lock).length} whole-file overrides, ${owned} skill lines of its own (slots and overrides)`); } +// shared/text/ snippets are plain text, so they read the same in a skill and in +// a code string, and each must be used somewhere ({{< name}} or {{text:name}}). +const snippetSources = [ + ...skillNames().map((n) => join(root, "skills", n, "SKILL.md")), + ...listFiles(platformsDir).map((f) => join(platformsDir, f)), +]; +const used = new Set(); +for (const file of snippetSources) { + if (!textFile.test(file)) continue; + for (const [, , name] of readFileSync(file, "utf8").matchAll(/\{\{\s*(<\s*|text:)([a-z0-9-]+)\s*\}\}/g)) used.add(name); +} +for (const file of listFiles(sharedTextDir)) { + const name = file.replace(/\.md$/, ""); + const text = readFileSync(join(sharedTextDir, file), "utf8"); + if (!used.has(name)) errors.push(`shared/text/${file} isn't used by any skill or host`); + if (/[`*_[\]#]/.test(text)) errors.push(`shared/text/${file} has markdown; keep snippets plain text so code can use them`); + if (text.trim().includes("\n")) errors.push(`shared/text/${file} has more than one line; keep one fact per snippet`); +} + console.log(report.join("\n")); if (errors.length) { console.error(`\n${errors.map((e) => `❌ ${e}`).join("\n")}`); diff --git a/shared/text/auth004.md b/shared/text/auth004.md new file mode 100644 index 0000000..09426a7 --- /dev/null +++ b/shared/text/auth004.md @@ -0,0 +1 @@ +the account's credit allowance is spent diff --git a/shared/text/auth006.md b/shared/text/auth006.md new file mode 100644 index 0000000..b5f06c3 --- /dev/null +++ b/shared/text/auth006.md @@ -0,0 +1 @@ +too many requests are running at once diff --git a/shared/text/auth014.md b/shared/text/auth014.md new file mode 100644 index 0000000..c8511e2 --- /dev/null +++ b/shared/text/auth014.md @@ -0,0 +1 @@ +this API key reached its credit cap diff --git a/shared/text/reqs007-page-not-recognised.md b/shared/text/reqs007-page-not-recognised.md new file mode 100644 index 0000000..dbc749c --- /dev/null +++ b/shared/text/reqs007-page-not-recognised.md @@ -0,0 +1 @@ +the domain is prepared, but this page has nothing Extract extracts (a home or search page, for example) diff --git a/skills/extract-structured-data/SKILL.md b/skills/extract-structured-data/SKILL.md index 3f25523..cdb1c4c 100644 --- a/skills/extract-structured-data/SKILL.md +++ b/skills/extract-structured-data/SKILL.md @@ -24,7 +24,7 @@ Use the dedicated `extract` tool, not `scrape`. It returns structured JSON direc - **`mode` selects the extraction strategy** (`auto`, `autoparse`, `css`). It is not the adaptive request setting, which is the separate boolean `mode_auto` and is on by default. `scrape` has no `mode` parameter at all. - **`extract` with `mode='auto'` works on any domain Zenrows has prepared, with any API key.** There is no per-account gate. When it can't extract a page it returns `REQS007` (or the older `AUTH010`) with a `reason`, and nothing is charged: - `domain_not_prepared`: call `prepare_domain` with the domain and 1 to 5 example URLs, one per kind of page the user wants (for example one product page and one listing page). It costs no credits and takes a few minutes. Poll `prepared_domain_status` about every 30 seconds until `status` is `ready`, then call `extract` again. Each new domain uses one slot of the account's monthly allowance, which `prepared_domain_status` shows. If the user can't wait, use `mode='css'` meanwhile. - - `page_not_recognised`: the domain is prepared, but this page has nothing Extract extracts (a home or search page, for example). Try a product, article or listing page on the same site. If this page should work, retry it once after a few minutes, not in a loop; otherwise use `mode='css'`. Don't call `prepare_domain`: it doesn't change this answer. + - `page_not_recognised`: {{< reqs007-page-not-recognised}}. Try a product, article or listing page on the same site. If this page should work, retry it once after a few minutes, not in a loop; otherwise use `mode='css'`. Don't call `prepare_domain`: it doesn't change this answer. - `domain_not_enabled`: preparing doesn't help. Use `mode='css'`, or tell the user to contact Zenrows support to have the site enabled. - **`fallback_autoparse` is deprecated and has no effect.** Extract no longer falls back to autoparse. Don't set it; if you do, the result only carries a note saying it was ignored. diff --git a/skills/using-zenrows/SKILL.md b/skills/using-zenrows/SKILL.md index f8e0a31..06f723f 100644 --- a/skills/using-zenrows/SKILL.md +++ b/skills/using-zenrows/SKILL.md @@ -78,8 +78,8 @@ Reads the connected account's plan and usage. It changes nothing. - Do not build manual retry loops around a request. Zenrows already escalates to heavier settings when a page needs it. - `401` or `AUTH003`: Zenrows is not connected, or the sign-in is no longer valid (for example the API key was rotated). Use the `{{skill:getting-started}}` skill to reconnect. Do not ask for a key. -- `AUTH004` (the account's credit allowance is spent) or `AUTH014` (this API key reached its credit cap): a plan limit, not a sign-in problem. Reconnecting doesn't clear it. Tell the user; they can check their usage and plan in the Zenrows dashboard, or raise the key's cap. -- `AUTH006`: too many requests are running at once. Wait for running requests to finish, then retry. +- `AUTH004` ({{< auth004}}) or `AUTH014` ({{< auth014}}): a plan limit, not a sign-in problem. Reconnecting doesn't clear it. Tell the user; they can check their usage and plan in the Zenrows dashboard, or raise the key's cap. +- `AUTH006`: {{< auth006}}. Wait for running requests to finish, then retry. - `413`: the response is too large. Narrow it with `css_extractor`. - `REQS007` (or the older `AUTH010`) from `extract`: read its `reason`. `domain_not_prepared`: prepare the domain with `prepare_domain` and wait for `prepared_domain_status` to say `ready`. `page_not_recognised`: try a product, article or listing page, or `mode='css'`; don't prepare the domain again. `domain_not_enabled`: use `mode='css'`. Not charged in any case. Details in the `{{skill:extract-structured-data}}` skill. - Look up any other error code at https://docs.zenrows.com. From 75b61be6523c81b13c81f723f061f96b75089347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Tue, 10 Nov 2026 08:53:18 +0000 Subject: [PATCH 06/18] refactor(render): keep only what code hosts and shared text need Per-host scripts (validate.mjs, package.sh, npm run package) are only used by the OpenAI plugin, and check.sh and the AUTH/REQS007 snippets only by the Activepieces piece: they stay with those hosts. What's left is what any code host needs: kind: code, and shared/text with {{< name}} and {{text:name}}. Comments and CONTRIBUTING describe any code host, a piece or a listing. --- CONTRIBUTING.md | 13 +++++-------- package.json | 1 - scripts/lib.mjs | 18 +++++++++--------- scripts/package.mjs | 18 ------------------ scripts/validate.mjs | 18 ------------------ shared/text/auth004.md | 1 - shared/text/auth006.md | 1 - shared/text/auth014.md | 1 - shared/text/reqs007-page-not-recognised.md | 1 - skills/extract-structured-data/SKILL.md | 2 +- skills/using-zenrows/SKILL.md | 4 ++-- 11 files changed, 17 insertions(+), 61 deletions(-) delete mode 100644 scripts/package.mjs delete mode 100644 shared/text/auth004.md delete mode 100644 shared/text/auth006.md delete mode 100644 shared/text/auth014.md delete mode 100644 shared/text/reqs007-page-not-recognised.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 75d7c9f..f6f1b5e 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -12,13 +12,10 @@ platforms// vars.yaml values for {{var}} skills//.md the host's text for a {{> slot}} in that skill overrides.lock whole-file overrides and the shared version each was copied from (only once one exists) - scripts/ the host's own tooling, never shipped: validate.mjs (run by npm run validate on - dist/), package.sh (run by npm run package -- ) and check.sh (a code - host's build and test, run by its CI workflow), when the host needs them everything else the host's manifests, README, …, copied into dist// ({{version}} comes from release-please) -dist// GENERATED: the plugin the host installs, or a code host's code +dist// GENERATED: the plugin the host installs, or a code host's files .github/plugin/marketplace.json Copilot's marketplace catalog (see below) -scripts/ render, validate, override, new-host, package, mirror +scripts/ render, validate, override, new-host, mirror ``` ### Marketplace catalogs @@ -44,13 +41,13 @@ Requires Node 22. Run `npm ci` once. Unknown variables, unknown skill names, missing slots, unused slot files and unknown snippets fail the render, and `npm run validate` fails on a reference to a skill the host doesn't ship, and on a skill name written plainly instead of as `{{skill:…}}`. -## Text shared with code hosts +## Shared text -A fact that a skill and a code host both state (what `AUTH004` means, why Extract refused a page) lives once in `shared/text/.md`, as one line of plain text. A skill inlines it with `{{< name}}`; code uses `{{text:name}}`, which renders as a quoted string, and adds its own words around it (an agent is told to use a skill, a user to check their connection). Snippets stay plain text so they read right in both, and `npm run validate` fails on markdown in one, or on one nothing uses. +A fact that more than one place states (what an error code means, what a request costs) lives once in `shared/text/.md`, as one line of plain text. A skill or a text file inlines it with `{{< name}}`; code uses `{{text:name}}`, which renders as a quoted string, and adds its own words around it. Snippets stay plain text so they read right in both, and `npm run validate` fails on markdown in one, or on one nothing uses. ## Code hosts -`kind: code` in `vars.yaml` makes a host whose output isn't a plugin but files for somewhere else: code for another project (a piece as it sits in that project's repo), or a listing a marketplace reads. A code host ships only its own files, with `{{…}}` filled in source files too: no shared skills, `LICENSE` or assets, and no `plugin.json`. Its `README.md` documents it here and isn't shipped. It needs no release-please entry until it's released from this repo. If it's code, its `scripts/check.sh` can build and test `dist//` the way the target project does, run by its own workflow. +`kind: code` in `vars.yaml` makes a host whose output isn't a plugin but files for somewhere else: code for another project (a piece as it sits in that project's repo), or a listing a marketplace reads. A code host ships only its own files, with `{{…}}` filled in source files too: no shared skills, `LICENSE` or assets, and no `plugin.json`. Its `README.md` documents it here and isn't shipped. It needs no release-please entry until it's released from this repo. ## Manifest schemas diff --git a/package.json b/package.json index 4cfa237..2ec4b1b 100644 --- a/package.json +++ b/package.json @@ -12,7 +12,6 @@ "override": "node scripts/override.mjs create", "override:ack": "node scripts/override.mjs ack", "new-host": "node scripts/new-host.mjs", - "package": "node scripts/package.mjs", "mirror": "node scripts/mirror.mjs" }, "devDependencies": { diff --git a/scripts/lib.mjs b/scripts/lib.mjs index c34ef00..3505bb4 100644 --- a/scripts/lib.mjs +++ b/scripts/lib.mjs @@ -15,10 +15,9 @@ export const sharedTextDir = join(root, "shared", "text"); // Files copied into every host's plugin as they are. const sharedFiles = { LICENSE: "LICENSE" }; const sharedDirs = { "shared/assets": "assets" }; -// In platforms//, these are inputs to the render or the host's own tooling -// (scripts/validate.mjs, scripts/package.sh), not files of the plugin. +// In platforms//, these are inputs to the render, not files of the plugin. // Everything else there (manifests, README, …) is copied into dist//. -const reserved = ["vars.yaml", "overrides.lock", "skills/", "scripts/"]; +const reserved = ["vars.yaml", "overrides.lock", "skills/"]; // {{vars}} are filled in files with these extensions; others are copied as bytes. export const textFile = /\.(json|md|mdc|ya?ml|txt|toml|[cm]?[jt]s)$/; @@ -55,8 +54,8 @@ function versions() { export function vars(host) { const v = parseYaml(readFileSync(join(platformsDir, host, "vars.yaml"), "utf8")) ?? {}; const version = versions()[`dist/${host}`]; - // A code host keeps its version in its own package.json until it's released - // from here, so it needs no release-please entry yet. + // A code host may not be released from here (yet), so it needs no + // release-please entry; it then has no {{version}}. if (!version && !isCodeHost(v)) throw new Error(`.release-please-manifest.json has no "dist/${host}" entry`); return version ? { ...v, version } : v; } @@ -72,7 +71,7 @@ export function readLock(host) { // the host's own skills (platforms//skills//SKILL.md with no shared // skill of that name). Returns [{ name, own }]. export function hostSkills(host, values = vars(host)) { - // A code host (kind: code, such as the Activepieces piece) ships no skills. + // A code host (kind: code) ships no skills. if (isCodeHost(values)) return []; const shared = skillNames(); const skip = values.skip_skills ?? []; @@ -89,9 +88,10 @@ export function hostSkills(host, values = vars(host)) { ].sort((a, b) => a.name.localeCompare(b.name)); } -// kind: code in vars.yaml marks a host whose output is code (a package or a -// piece for another project), not a plugin: it ships only its own files, with -// {{…}} filled in its source, and no shared skills, LICENSE or assets. +// kind: code in vars.yaml marks a host whose output isn't a plugin but files for +// somewhere else (code for another project, a listing a marketplace reads): it +// ships only its own files, with {{…}} filled in, and no shared skills, LICENSE +// or assets. export function isCodeHost(values) { const kind = values.kind ?? "plugin"; if (!["plugin", "code"].includes(kind)) throw new Error(`kind: ${kind} in vars.yaml; use plugin or code`); diff --git a/scripts/package.mjs b/scripts/package.mjs deleted file mode 100644 index 274068f..0000000 --- a/scripts/package.mjs +++ /dev/null @@ -1,18 +0,0 @@ -// npm run package -- : builds what a host uploads by hand (for example -// OpenAI's ZIP) by running platforms//scripts/package.sh. -import { spawnSync } from "node:child_process"; -import { existsSync } from "node:fs"; -import { join } from "node:path"; -import { hosts, platformsDir } from "./lib.mjs"; - -const host = process.argv[2]; -if (!host || !hosts().includes(host)) { - console.error(`Usage: npm run package -- , one of: ${hosts().join(", ")}`); - process.exit(1); -} -const script = join(platformsDir, host, "scripts", "package.sh"); -if (!existsSync(script)) { - console.log(`${host} needs no packaging: it installs from dist/${host}/ (no platforms/${host}/scripts/package.sh).`); - process.exit(0); -} -process.exit(spawnSync("bash", [script], { stdio: "inherit" }).status ?? 1); diff --git a/scripts/validate.mjs b/scripts/validate.mjs index e68c3b0..826d6d6 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -1,7 +1,6 @@ // Checks what each host ships: skill frontmatter, skill references, manifests // (against the JSON Schema they declare), the marketplace catalogs at the top of // the repo, and that no whole-file override was copied from an older shared file. -import { spawnSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { basename, dirname, join } from "node:path"; import Ajv2020 from "ajv/dist/2020.js"; @@ -218,23 +217,6 @@ for (const host of hosts()) { if (text.includes("TODO(new-host)")) errors.push(`${host}: platforms/${host}/${file} still has a TODO(new-host)`); } - // platforms//scripts/ is the host's tooling and never ships, so only the - // files the repo runs belong there: validate.mjs, package.sh, and check.sh (a - // code host's build and test, run by its CI workflow). - for (const file of listFiles(join(platformsDir, host, "scripts"))) { - if (!["validate.mjs", "package.sh", "check.sh"].includes(file)) { - errors.push(`${host}: platforms/${host}/scripts/${file} isn't shipped (scripts/ holds only validate.mjs, package.sh and check.sh); move plugin files elsewhere`); - } - } - - // Checks only one host needs, such as OpenAI's submission limits, live in - // platforms//scripts/validate.mjs and run on the committed dist/. - const extra = join(platformsDir, host, "scripts", "validate.mjs"); - if (existsSync(extra)) { - const run = spawnSync(process.execPath, [extra, join(root, "dist", host)], { encoding: "utf8" }); - if (run.status !== 0) errors.push(`${host}: platforms/${host}/scripts/validate.mjs failed:\n${(run.stderr || run.stdout).trim()}`); - } - // How much skill text the host owns instead of sharing: its slot files and // whole-file overrides. Information only; a growing number is worth a look. const owned = listFiles(join(platformsDir, host, "skills")) diff --git a/shared/text/auth004.md b/shared/text/auth004.md deleted file mode 100644 index 09426a7..0000000 --- a/shared/text/auth004.md +++ /dev/null @@ -1 +0,0 @@ -the account's credit allowance is spent diff --git a/shared/text/auth006.md b/shared/text/auth006.md deleted file mode 100644 index b5f06c3..0000000 --- a/shared/text/auth006.md +++ /dev/null @@ -1 +0,0 @@ -too many requests are running at once diff --git a/shared/text/auth014.md b/shared/text/auth014.md deleted file mode 100644 index c8511e2..0000000 --- a/shared/text/auth014.md +++ /dev/null @@ -1 +0,0 @@ -this API key reached its credit cap diff --git a/shared/text/reqs007-page-not-recognised.md b/shared/text/reqs007-page-not-recognised.md deleted file mode 100644 index dbc749c..0000000 --- a/shared/text/reqs007-page-not-recognised.md +++ /dev/null @@ -1 +0,0 @@ -the domain is prepared, but this page has nothing Extract extracts (a home or search page, for example) diff --git a/skills/extract-structured-data/SKILL.md b/skills/extract-structured-data/SKILL.md index cdb1c4c..3f25523 100644 --- a/skills/extract-structured-data/SKILL.md +++ b/skills/extract-structured-data/SKILL.md @@ -24,7 +24,7 @@ Use the dedicated `extract` tool, not `scrape`. It returns structured JSON direc - **`mode` selects the extraction strategy** (`auto`, `autoparse`, `css`). It is not the adaptive request setting, which is the separate boolean `mode_auto` and is on by default. `scrape` has no `mode` parameter at all. - **`extract` with `mode='auto'` works on any domain Zenrows has prepared, with any API key.** There is no per-account gate. When it can't extract a page it returns `REQS007` (or the older `AUTH010`) with a `reason`, and nothing is charged: - `domain_not_prepared`: call `prepare_domain` with the domain and 1 to 5 example URLs, one per kind of page the user wants (for example one product page and one listing page). It costs no credits and takes a few minutes. Poll `prepared_domain_status` about every 30 seconds until `status` is `ready`, then call `extract` again. Each new domain uses one slot of the account's monthly allowance, which `prepared_domain_status` shows. If the user can't wait, use `mode='css'` meanwhile. - - `page_not_recognised`: {{< reqs007-page-not-recognised}}. Try a product, article or listing page on the same site. If this page should work, retry it once after a few minutes, not in a loop; otherwise use `mode='css'`. Don't call `prepare_domain`: it doesn't change this answer. + - `page_not_recognised`: the domain is prepared, but this page has nothing Extract extracts (a home or search page, for example). Try a product, article or listing page on the same site. If this page should work, retry it once after a few minutes, not in a loop; otherwise use `mode='css'`. Don't call `prepare_domain`: it doesn't change this answer. - `domain_not_enabled`: preparing doesn't help. Use `mode='css'`, or tell the user to contact Zenrows support to have the site enabled. - **`fallback_autoparse` is deprecated and has no effect.** Extract no longer falls back to autoparse. Don't set it; if you do, the result only carries a note saying it was ignored. diff --git a/skills/using-zenrows/SKILL.md b/skills/using-zenrows/SKILL.md index 06f723f..f8e0a31 100644 --- a/skills/using-zenrows/SKILL.md +++ b/skills/using-zenrows/SKILL.md @@ -78,8 +78,8 @@ Reads the connected account's plan and usage. It changes nothing. - Do not build manual retry loops around a request. Zenrows already escalates to heavier settings when a page needs it. - `401` or `AUTH003`: Zenrows is not connected, or the sign-in is no longer valid (for example the API key was rotated). Use the `{{skill:getting-started}}` skill to reconnect. Do not ask for a key. -- `AUTH004` ({{< auth004}}) or `AUTH014` ({{< auth014}}): a plan limit, not a sign-in problem. Reconnecting doesn't clear it. Tell the user; they can check their usage and plan in the Zenrows dashboard, or raise the key's cap. -- `AUTH006`: {{< auth006}}. Wait for running requests to finish, then retry. +- `AUTH004` (the account's credit allowance is spent) or `AUTH014` (this API key reached its credit cap): a plan limit, not a sign-in problem. Reconnecting doesn't clear it. Tell the user; they can check their usage and plan in the Zenrows dashboard, or raise the key's cap. +- `AUTH006`: too many requests are running at once. Wait for running requests to finish, then retry. - `413`: the response is too large. Narrow it with `css_extractor`. - `REQS007` (or the older `AUTH010`) from `extract`: read its `reason`. `domain_not_prepared`: prepare the domain with `prepare_domain` and wait for `prepared_domain_status` to say `ready`. `page_not_recognised`: try a product, article or listing page, or `mode='css'`; don't prepare the domain again. `domain_not_enabled`: use `mode='css'`. Not charged in any case. Details in the `{{skill:extract-structured-data}}` skill. - Look up any other error code at https://docs.zenrows.com. From 21d3148e5cc96951c12ca415f8557f4eee49e591 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Tue, 10 Nov 2026 16:55:32 +0000 Subject: [PATCH 07/18] fix(render): catch unfilled tokens and misplaced files in code hosts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fail the render on any {{…}} left in a rendered text file, and on a skills/ folder in a code host. Validate requires a code host README without {{…}}, matches {{< name}} the way the render does, and only treats backticks, *, links and headings as markdown in a snippet. --- scripts/lib.mjs | 24 +++++++++++++++++++++--- scripts/validate.mjs | 20 ++++++++++++++++++-- 2 files changed, 39 insertions(+), 5 deletions(-) diff --git a/scripts/lib.mjs b/scripts/lib.mjs index 3505bb4..00fbc45 100644 --- a/scripts/lib.mjs +++ b/scripts/lib.mjs @@ -71,8 +71,13 @@ export function readLock(host) { // the host's own skills (platforms//skills//SKILL.md with no shared // skill of that name). Returns [{ name, own }]. export function hostSkills(host, values = vars(host)) { - // A code host (kind: code) ships no skills. - if (isCodeHost(values)) return []; + // A code host (kind: code) ships no skills, so a skills/ folder there would be ignored. + if (isCodeHost(values)) { + if (listFiles(join(platformsDir, host, "skills")).length) { + throw new Error(`platforms/${host}/skills/: a code host (kind: code) ships no skills; move these files out of skills/`); + } + return []; + } const shared = skillNames(); const skip = values.skip_skills ?? []; for (const name of skip) { @@ -120,7 +125,11 @@ function fill(text, values, where) { return text.replace(/\{\{\s*([a-z_]+)(?::([a-z0-9-]+))?\s*\}\}/g, (token, name, arg) => { if (name === "skill") { if (!values.skills.includes(arg)) { - const why = skillNames().includes(arg) ? "this host skips it (skip_skills)" : "there's no such skill"; + const why = !skillNames().includes(arg) + ? "there's no such skill" + : isCodeHost(values) + ? "a code host (kind: code) ships no skills" + : "this host skips it (skip_skills)"; throw new Error(`${where}: ${token} names a skill this host doesn't ship: ${why}`); } return hostSkillName(values, arg); @@ -173,6 +182,15 @@ export function renderHost(host) { if (isCodeHost(values) && file === "README.md") continue; out.set(file, renderFile(join(platform, file), values)); } + + // A {{…}} the render didn't fill (a mistyped token, or a file type outside + // textFile) would otherwise ship as it is. Binary files (not valid UTF-8) are skipped. + for (const [path, buf] of out) { + const text = buf.toString("utf8"); + if (Buffer.from(text, "utf8").equals(buf) && text.includes("{{")) { + throw new Error(`${host}: ${path} still has an unrendered {{…}}`); + } + } return out; } diff --git a/scripts/validate.mjs b/scripts/validate.mjs index 826d6d6..c94e78f 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -217,6 +217,20 @@ for (const host of hosts()) { if (text.includes("TODO(new-host)")) errors.push(`${host}: platforms/${host}/${file} still has a TODO(new-host)`); } + // A code host's README stays here (it isn't shipped) and the repo's hosts + // list links to it. + if (isCodeHost(values)) { + const readme = join(platformsDir, host, "README.md"); + if (!existsSync(readme)) { + errors.push(`${host}: a code host needs platforms/${host}/README.md; the hosts list in README.md links to it`); + } else if (readFileSync(readme, "utf8").includes("{{")) { + // It isn't rendered, so a {{…}} would show as it is on GitHub. + errors.push(`${host}: platforms/${host}/README.md isn't rendered, so it can't use {{…}}; write the text out`); + } + report.push(`${host}: code host, ships only its own files`); + continue; + } + // How much skill text the host owns instead of sharing: its slot files and // whole-file overrides. Information only; a growing number is worth a look. const owned = listFiles(join(platformsDir, host, "skills")) @@ -234,13 +248,15 @@ const snippetSources = [ const used = new Set(); for (const file of snippetSources) { if (!textFile.test(file)) continue; - for (const [, , name] of readFileSync(file, "utf8").matchAll(/\{\{\s*(<\s*|text:)([a-z0-9-]+)\s*\}\}/g)) used.add(name); + for (const [, , name] of readFileSync(file, "utf8").matchAll(/\{\{(<\s*|\s*text:)([a-z0-9-]+)\s*\}\}/g)) used.add(name); } for (const file of listFiles(sharedTextDir)) { const name = file.replace(/\.md$/, ""); const text = readFileSync(join(sharedTextDir, file), "utf8"); if (!used.has(name)) errors.push(`shared/text/${file} isn't used by any skill or host`); - if (/[`*_[\]#]/.test(text)) errors.push(`shared/text/${file} has markdown; keep snippets plain text so code can use them`); + // Backticks, *, links and headings would show up as is in a code string. + // _ and [ ] don't count: parameter names (js_render) and [{...}] use them. + if (/[`*]/.test(text) || /\]\(/.test(text) || /^#/.test(text.trim())) errors.push(`shared/text/${file} has markdown; keep snippets plain text so code can use them`); if (text.trim().includes("\n")) errors.push(`shared/text/${file} has more than one line; keep one fact per snippet`); } From 16b1aec35cd7284ab84f901da2121bfe435fd708 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Tue, 10 Nov 2026 16:55:32 +0000 Subject: [PATCH 08/18] test(render): check the mistakes render and validate must catch --- .github/workflows/ci.yml | 2 + package.json | 1 + scripts/test.mjs | 85 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 88 insertions(+) create mode 100644 scripts/test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 47bb1d9..01a0d82 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -33,6 +33,8 @@ jobs: - run: npm run render -- --check # Frontmatter, skill references, manifests against their JSON Schemas, and overrides.lock. - run: npm run validate + # The mistakes render and validate must catch (scripts/test.mjs). + - run: npm test workflows: name: Lint workflows diff --git a/package.json b/package.json index 2ec4b1b..11d8c95 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,7 @@ "prepare": "git rev-parse --git-dir >/dev/null 2>&1 && git config core.hooksPath .githooks || true", "render": "node scripts/render.mjs", "validate": "node scripts/validate.mjs", + "test": "node --test scripts/test.mjs", "override": "node scripts/override.mjs create", "override:ack": "node scripts/override.mjs ack", "new-host": "node scripts/new-host.mjs", diff --git a/scripts/test.mjs b/scripts/test.mjs new file mode 100644 index 0000000..a196d1a --- /dev/null +++ b/scripts/test.mjs @@ -0,0 +1,85 @@ +// npm test: the mistakes the render and validate must catch. Each case copies +// the repo to a temporary folder, makes one change (a throwaway code host, a +// snippet, …), runs npm run render and npm run validate there, and checks the +// first failure, or that both pass. Add a case whenever a new rule is added. +import { spawnSync } from "node:child_process"; +import { cpSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { root } from "./lib.mjs"; + +const copied = ["scripts", "skills", "shared", "platforms", "dist", "README.md", "LICENSE", "package.json", ".release-please-manifest.json", "release-please-config.json", ".github"]; + +// A code host that passes as it is; cases add or remove files from it. +const piece = { + "platforms/piece/vars.yaml": "host_full: Piece\nkind: code\n", + "platforms/piece/README.md": "# Piece\n", + "platforms/piece/src/index.ts": "export const host = \"{{host_full}}\";\n", +}; + +// Runs render then validate in a copy of the repo with `files` written over it +// (null deletes). Returns { ok, out }: out is the failing step's output. +function run(files) { + const dir = mkdtempSync(join(tmpdir(), "zenrows-integrations-test-")); + try { + for (const path of copied) cpSync(join(root, path), join(dir, path), { recursive: true }); + symlinkSync(join(root, "node_modules"), join(dir, "node_modules")); + mkdirSync(join(root, ".cache"), { recursive: true }); + symlinkSync(join(root, ".cache"), join(dir, ".cache")); + for (const [path, content] of Object.entries(files)) { + if (content === null) rmSync(join(dir, path), { recursive: true, force: true }); + else { + mkdirSync(dirname(join(dir, path)), { recursive: true }); + writeFileSync(join(dir, path), content); + } + } + for (const script of ["render", "validate"]) { + const r = spawnSync("node", [join("scripts", `${script}.mjs`)], { cwd: dir, encoding: "utf8" }); + if (r.status !== 0) return { ok: false, out: `${r.stdout}${r.stderr}` }; + } + return { ok: true, out: "" }; + } finally { + rmSync(dir, { recursive: true, force: true }); + } +} + +const passes = (files) => () => { + const r = run(files); + assert.ok(r.ok, r.out); +}; +const fails = (files, message) => () => { + const r = run(files); + assert.ok(!r.ok, "expected a failure, but render and validate passed"); + assert.ok(r.out.includes(message), `expected "${message}" in:\n${r.out}`); +}; + +test("the repo as it is passes", passes({})); +test("a code host passes", passes(piece)); + +test("a {{…}} in a file type the render doesn't fill fails", () => { + fails({ ...piece, "platforms/piece/src/view.tsx": "export const a = \"{{host_full}}\";\n" }, "src/view.tsx still has an unrendered {{…}}")(); + fails({ ...piece, "platforms/piece/setup.py": "name = \"{{host_full}}\"\n" }, "setup.py still has an unrendered {{…}}")(); +}); +test("a mistyped snippet token fails", fails({ + ...piece, + "shared/text/cost.md": "A request with js_render costs 5 credits.\n", + "platforms/piece/src/index.ts": "export const cost = {{ < cost}};\n", +}, "src/index.ts still has an unrendered {{…}}")); +test("an unknown variable fails", fails({ ...piece, "platforms/piece/src/index.ts": "export const a = \"{{hots_full}}\";\n" }, "unknown variable {{hots_full}}")); + +test("a code host with no README fails", fails({ ...piece, "platforms/piece/README.md": null }, "a code host needs platforms/piece/README.md")); +test("a code host README with a {{…}} fails", fails({ ...piece, "platforms/piece/README.md": "# {{host_full}}\n" }, "README.md isn't rendered")); +test("a code host with a skills/ folder fails", fails({ ...piece, "platforms/piece/skills/scrape/SKILL.md": "x\n" }, "a code host (kind: code) ships no skills")); +test("a skill reference in a code host fails", fails({ ...piece, "platforms/piece/src/index.ts": "export const s = \"{{skill:batch}}\";\n" }, "a code host (kind: code) ships no skills")); +test("an unknown kind fails", fails({ ...piece, "platforms/piece/vars.yaml": "host_full: Piece\nkind: Code\n" }, "kind: Code in vars.yaml")); + +// Snippets: plain text, one line, used somewhere. +const snippet = (text) => ({ ...piece, "shared/text/cost.md": text, "platforms/piece/src/index.ts": "export const cost = {{text:cost}};\n" }); +test("a snippet with parameter names and brackets passes", passes(snippet("A request with js_render returns [{...}] and costs 5 credits.\n"))); +test("a snippet with backticks fails", fails(snippet("A request with `js_render` costs 5 credits.\n"), "has markdown")); +test("a snippet with a link fails", fails(snippet("See [pricing](https://www.zenrows.com/pricing).\n"), "has markdown")); +test("a snippet on two lines fails", fails(snippet("One fact.\nAnother fact.\n"), "has more than one line")); +test("an unused snippet fails", fails({ ...piece, "shared/text/cost.md": "A request costs 5 credits.\n" }, "isn't used by any skill or host")); +test("a missing snippet fails", fails({ ...piece, "platforms/piece/src/index.ts": "export const cost = {{text:nope}};\n" }, "there's no shared/text/nope.md")); From 29bafffa8c0670f8e7f03ca12da648708e18479e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Tue, 10 Nov 2026 16:55:32 +0000 Subject: [PATCH 09/18] docs(contributing): describe the code host rules and npm test --- CONTRIBUTING.md | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f6f1b5e..c66833a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -15,7 +15,7 @@ platforms// everything else the host's manifests, README, …, copied into dist// ({{version}} comes from release-please) dist// GENERATED: the plugin the host installs, or a code host's files .github/plugin/marketplace.json Copilot's marketplace catalog (see below) -scripts/ render, validate, override, new-host, mirror +scripts/ render, validate, override, new-host, mirror, test ``` ### Marketplace catalogs @@ -39,15 +39,19 @@ Requires Node 22. Run `npm ci` once. - **A shared skill a host shouldn't ship:** list it under `skip_skills:` in that host's `vars.yaml` (Cursor uses a rules file instead of `using-zenrows`). A `{{skill:…}}` reference to a skipped skill fails the render. - **A reference to another skill:** write `{{skill:batch}}`, never the plain name. It renders as that skill's name on the host: `skill_prefix` plus the shared name, unless `skill_names` in `vars.yaml` renames it (OpenClaw ships `scrape-webpage` as `zenrows-scrape`). The skill's own `name:` in its frontmatter uses it too. -Unknown variables, unknown skill names, missing slots, unused slot files and unknown snippets fail the render, and `npm run validate` fails on a reference to a skill the host doesn't ship, and on a skill name written plainly instead of as `{{skill:…}}`. +Unknown variables, unknown skill names, missing slots, unused slot files, unknown snippets and any `{{…}}` left in a rendered file fail the render, and `npm run validate` fails on a reference to a skill the host doesn't ship, and on a skill name written plainly instead of as `{{skill:…}}`. ## Shared text -A fact that more than one place states (what an error code means, what a request costs) lives once in `shared/text/.md`, as one line of plain text. A skill or a text file inlines it with `{{< name}}`; code uses `{{text:name}}`, which renders as a quoted string, and adds its own words around it. Snippets stay plain text so they read right in both, and `npm run validate` fails on markdown in one, or on one nothing uses. +A fact that more than one place states (what an error code means, what a request costs) lives once in `shared/text/.md`, as one line of plain text. A skill or a text file inlines it with `{{< name}}`; code uses `{{text:name}}`, which renders as a quoted string, and adds its own words around it. Snippets stay plain text so they read right in both, and `npm run validate` fails on markdown in one (backticks, `*`, a link or a heading), or on one nothing uses. ## Code hosts -`kind: code` in `vars.yaml` makes a host whose output isn't a plugin but files for somewhere else: code for another project (a piece as it sits in that project's repo), or a listing a marketplace reads. A code host ships only its own files, with `{{…}}` filled in source files too: no shared skills, `LICENSE` or assets, and no `plugin.json`. Its `README.md` documents it here and isn't shipped. It needs no release-please entry until it's released from this repo. +`kind: code` in `vars.yaml` makes a host whose output isn't a plugin but files for somewhere else: code for another project (a piece as it sits in that project's repo), or a listing a marketplace reads. A code host ships only its own files, with `{{…}}` filled in source files too: no shared skills, `LICENSE` or assets, and no `plugin.json`. Its `README.md` documents it here and isn't shipped or rendered, so it writes its text out instead of using `{{…}}`; `npm run validate` requires one, and the render fails if the host has a `skills/` folder. It needs no release-please entry until it's released from this repo. + +## Tests + +`npm test` (CI runs it too) checks that render and validate catch the mistakes they're meant to: each case in `scripts/test.mjs` makes one change in a temporary copy of the repo (a throwaway code host, a snippet, …) and expects a given error, or a pass. When you add a rule, add a case that breaks it. ## Manifest schemas From 1117a8b7a8fb792c685626f27808479b8f7ae1d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Tue, 10 Nov 2026 17:21:59 +0000 Subject: [PATCH 10/18] =?UTF-8?q?fix(validate):=20allow=20documented=20{{?= =?UTF-8?q?=E2=80=A6}}=20in=20a=20code=20host=20README,=20and=20test=20wit?= =?UTF-8?q?h=20every=20tracked=20file?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- CONTRIBUTING.md | 2 +- scripts/test.mjs | 11 ++++++++--- scripts/validate.mjs | 5 +++-- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c66833a..7a10f0f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,7 +47,7 @@ A fact that more than one place states (what an error code means, what a request ## Code hosts -`kind: code` in `vars.yaml` makes a host whose output isn't a plugin but files for somewhere else: code for another project (a piece as it sits in that project's repo), or a listing a marketplace reads. A code host ships only its own files, with `{{…}}` filled in source files too: no shared skills, `LICENSE` or assets, and no `plugin.json`. Its `README.md` documents it here and isn't shipped or rendered, so it writes its text out instead of using `{{…}}`; `npm run validate` requires one, and the render fails if the host has a `skills/` folder. It needs no release-please entry until it's released from this repo. +`kind: code` in `vars.yaml` makes a host whose output isn't a plugin but files for somewhere else: code for another project (a piece as it sits in that project's repo), or a listing a marketplace reads. A code host ships only its own files, with `{{…}}` filled in source files too: no shared skills, `LICENSE` or assets, and no `plugin.json`. Its `README.md` documents it here and isn't shipped or rendered, so it writes its text out instead of using `{{…}}` (outside code spans, which document the syntax); `npm run validate` requires one, and the render fails if the host has a `skills/` folder. It needs no release-please entry until it's released from this repo. ## Tests diff --git a/scripts/test.mjs b/scripts/test.mjs index a196d1a..f97fe47 100644 --- a/scripts/test.mjs +++ b/scripts/test.mjs @@ -2,7 +2,7 @@ // the repo to a temporary folder, makes one change (a throwaway code host, a // snippet, …), runs npm run render and npm run validate there, and checks the // first failure, or that both pass. Add a case whenever a new rule is added. -import { spawnSync } from "node:child_process"; +import { execFileSync, spawnSync } from "node:child_process"; import { cpSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; @@ -10,7 +10,8 @@ import { test } from "node:test"; import assert from "node:assert/strict"; import { root } from "./lib.mjs"; -const copied = ["scripts", "skills", "shared", "platforms", "dist", "README.md", "LICENSE", "package.json", ".release-please-manifest.json", "release-please-config.json", ".github"]; +// Every tracked file, so catalogs at the top of the repo (.claude-plugin/, …) come too. +const copied = execFileSync("git", ["ls-files"], { cwd: root, encoding: "utf8" }).split("\n").filter(Boolean); // A code host that passes as it is; cases add or remove files from it. const piece = { @@ -24,7 +25,10 @@ const piece = { function run(files) { const dir = mkdtempSync(join(tmpdir(), "zenrows-integrations-test-")); try { - for (const path of copied) cpSync(join(root, path), join(dir, path), { recursive: true }); + for (const path of copied) { + mkdirSync(dirname(join(dir, path)), { recursive: true }); + cpSync(join(root, path), join(dir, path)); + } symlinkSync(join(root, "node_modules"), join(dir, "node_modules")); mkdirSync(join(root, ".cache"), { recursive: true }); symlinkSync(join(root, ".cache"), join(dir, ".cache")); @@ -71,6 +75,7 @@ test("an unknown variable fails", fails({ ...piece, "platforms/piece/src/index.t test("a code host with no README fails", fails({ ...piece, "platforms/piece/README.md": null }, "a code host needs platforms/piece/README.md")); test("a code host README with a {{…}} fails", fails({ ...piece, "platforms/piece/README.md": "# {{host_full}}\n" }, "README.md isn't rendered")); +test("a code host README that documents {{…}} in code passes", passes({ ...piece, "platforms/piece/README.md": "# Piece\n\nFacts come in through `{{text:name}}`.\n\n```\n{{< name}}\n```\n" })); test("a code host with a skills/ folder fails", fails({ ...piece, "platforms/piece/skills/scrape/SKILL.md": "x\n" }, "a code host (kind: code) ships no skills")); test("a skill reference in a code host fails", fails({ ...piece, "platforms/piece/src/index.ts": "export const s = \"{{skill:batch}}\";\n" }, "a code host (kind: code) ships no skills")); test("an unknown kind fails", fails({ ...piece, "platforms/piece/vars.yaml": "host_full: Piece\nkind: Code\n" }, "kind: Code in vars.yaml")); diff --git a/scripts/validate.mjs b/scripts/validate.mjs index c94e78f..8fcb348 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -223,8 +223,9 @@ for (const host of hosts()) { const readme = join(platformsDir, host, "README.md"); if (!existsSync(readme)) { errors.push(`${host}: a code host needs platforms/${host}/README.md; the hosts list in README.md links to it`); - } else if (readFileSync(readme, "utf8").includes("{{")) { - // It isn't rendered, so a {{…}} would show as it is on GitHub. + } else if (readFileSync(readme, "utf8").replace(/```[\s\S]*?```|`[^`\n]*`/g, "").includes("{{")) { + // It isn't rendered, so a {{…}} would show as it is on GitHub. One in + // `code` documents the syntax, so code spans and blocks are left out. errors.push(`${host}: platforms/${host}/README.md isn't rendered, so it can't use {{…}}; write the text out`); } report.push(`${host}: code host, ships only its own files`); From 9cb7a046de33bbec84712505b107529a8e4b654d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Tue, 10 Nov 2026 18:35:06 +0000 Subject: [PATCH 11/18] fix(render): flag only this tool's leftover tokens, and check snippets and vars.yaml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Other tools' {{…}} (JSX, Make, n8n) ships as written. Snippets are trimmed and can't contain {{…}}, and a malformed vars.yaml or kind names the file. --- scripts/lib.mjs | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/scripts/lib.mjs b/scripts/lib.mjs index 00fbc45..98dab3f 100644 --- a/scripts/lib.mjs +++ b/scripts/lib.mjs @@ -18,6 +18,11 @@ const sharedDirs = { "shared/assets": "assets" }; // In platforms//, these are inputs to the render, not files of the plugin. // Everything else there (manifests, README, …) is copied into dist//. const reserved = ["vars.yaml", "overrides.lock", "skills/"]; +// What's left of one of this tool's tokens ({{name}}, {{skill:x}}, {{< x}}, +// {{> x}}, {{text:x}}) after the render, for example in a file type it doesn't +// fill or with a stray space. Other {{…}} (n8n or Make expressions, JSX's +// style={{…}}) isn't ours and ships as written. +export const leftoverToken = /\{\{\s*(?:[<>]\??\s*[a-z0-9_-]+|[a-z_]+(?::[a-z0-9-]+)?)\s*\}\}/; // {{vars}} are filled in files with these extensions; others are copied as bytes. export const textFile = /\.(json|md|mdc|ya?ml|txt|toml|[cm]?[jt]s)$/; @@ -53,6 +58,10 @@ function versions() { export function vars(host) { const v = parseYaml(readFileSync(join(platformsDir, host, "vars.yaml"), "utf8")) ?? {}; + if (typeof v !== "object" || Array.isArray(v)) throw new Error(`platforms/${host}/vars.yaml must be a list of name: value lines`); + if ("kind" in v && v.kind !== "plugin" && v.kind !== "code") { + throw new Error(`platforms/${host}/vars.yaml: kind is ${JSON.stringify(v.kind)}; use plugin or code`); + } const version = versions()[`dist/${host}`]; // A code host may not be released from here (yet), so it needs no // release-please entry; it then has no {{version}}. @@ -98,9 +107,8 @@ export function hostSkills(host, values = vars(host)) { // ships only its own files, with {{…}} filled in, and no shared skills, LICENSE // or assets. export function isCodeHost(values) { - const kind = values.kind ?? "plugin"; - if (!["plugin", "code"].includes(kind)) throw new Error(`kind: ${kind} in vars.yaml; use plugin or code`); - return kind === "code"; + // vars() has already rejected any other kind, naming the file. + return values.kind === "code"; } // A skill's name on a host: skill_prefix + the shared name, unless vars.yaml @@ -114,7 +122,12 @@ export function hostSkillName(values, name) { export function sharedText(name, where) { const file = join(sharedTextDir, `${name}.md`); if (!existsSync(file)) throw new Error(`${where}: there's no shared/text/${name}.md`); - return readFileSync(file, "utf8").replace(/\n$/, ""); + // Trimmed, so a trailing blank line or a CRLF ending doesn't end up in a string. + const text = readFileSync(file, "utf8").trim(); + if (text.includes("{{")) { + throw new Error(`shared/text/${name}.md uses {{…}}; a snippet is a plain fact, and the text around it adds what differs per host`); + } + return text; } // {{name}} is a variable from vars.yaml; {{skill:name}} is that skill's name on @@ -183,12 +196,13 @@ export function renderHost(host) { out.set(file, renderFile(join(platform, file), values)); } - // A {{…}} the render didn't fill (a mistyped token, or a file type outside + // A token the render didn't fill (a mistyped one, or in a file type outside // textFile) would otherwise ship as it is. Binary files (not valid UTF-8) are skipped. for (const [path, buf] of out) { const text = buf.toString("utf8"); - if (Buffer.from(text, "utf8").equals(buf) && text.includes("{{")) { - throw new Error(`${host}: ${path} still has an unrendered {{…}}`); + const token = Buffer.from(text, "utf8").equals(buf) && text.match(leftoverToken); + if (token) { + throw new Error(`${host}: ${path} still has an unrendered ${token[0]}`); } } return out; From c098863b6bba945f6c3eceea248549642bf51a9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Tue, 10 Nov 2026 18:35:06 +0000 Subject: [PATCH 12/18] fix(validate): don't count a snippet named only in a code host README or vars.yaml as used --- scripts/validate.mjs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/validate.mjs b/scripts/validate.mjs index 8fcb348..8aa12b4 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -220,13 +220,8 @@ for (const host of hosts()) { // A code host's README stays here (it isn't shipped) and the repo's hosts // list links to it. if (isCodeHost(values)) { - const readme = join(platformsDir, host, "README.md"); - if (!existsSync(readme)) { + if (!existsSync(join(platformsDir, host, "README.md"))) { errors.push(`${host}: a code host needs platforms/${host}/README.md; the hosts list in README.md links to it`); - } else if (readFileSync(readme, "utf8").replace(/```[\s\S]*?```|`[^`\n]*`/g, "").includes("{{")) { - // It isn't rendered, so a {{…}} would show as it is on GitHub. One in - // `code` documents the syntax, so code spans and blocks are left out. - errors.push(`${host}: platforms/${host}/README.md isn't rendered, so it can't use {{…}}; write the text out`); } report.push(`${host}: code host, ships only its own files`); continue; @@ -242,9 +237,14 @@ for (const host of hosts()) { // shared/text/ snippets are plain text, so they read the same in a skill and in // a code string, and each must be used somewhere ({{< name}} or {{text:name}}). +// A code host's README isn't rendered, and vars.yaml and overrides.lock are +// inputs, so a snippet named only there isn't used. +const codeHostReadmes = hosts().filter((h) => isCodeHost(vars(h))).map((h) => `${h}/README.md`); const snippetSources = [ ...skillNames().map((n) => join(root, "skills", n, "SKILL.md")), - ...listFiles(platformsDir).map((f) => join(platformsDir, f)), + ...listFiles(platformsDir) + .filter((f) => !codeHostReadmes.includes(f) && !/\/(vars\.yaml|overrides\.lock)$/.test(f)) + .map((f) => join(platformsDir, f)), ]; const used = new Set(); for (const file of snippetSources) { From 6def597b6271da532d704fe55121bc919ea691de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Tue, 10 Nov 2026 18:35:06 +0000 Subject: [PATCH 13/18] test(render): cover the new rules and copy tracked files with -z --- scripts/test.mjs | 56 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 45 insertions(+), 11 deletions(-) diff --git a/scripts/test.mjs b/scripts/test.mjs index f97fe47..736befc 100644 --- a/scripts/test.mjs +++ b/scripts/test.mjs @@ -3,15 +3,18 @@ // snippet, …), runs npm run render and npm run validate there, and checks the // first failure, or that both pass. Add a case whenever a new rule is added. import { execFileSync, spawnSync } from "node:child_process"; -import { cpSync, mkdirSync, mkdtempSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; +import { cpSync, existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { tmpdir } from "node:os"; import { dirname, join } from "node:path"; import { test } from "node:test"; import assert from "node:assert/strict"; import { root } from "./lib.mjs"; -// Every tracked file, so catalogs at the top of the repo (.claude-plugin/, …) come too. -const copied = execFileSync("git", ["ls-files"], { cwd: root, encoding: "utf8" }).split("\n").filter(Boolean); +// Every tracked file (-z keeps non-ASCII names as they are), so catalogs at the +// top of the repo (.claude-plugin/, …) come too. One deleted locally is left out. +const copied = execFileSync("git", ["ls-files", "-z"], { cwd: root, encoding: "utf8" }) + .split("\0") + .filter((path) => path && existsSync(join(root, path))); // A code host that passes as it is; cases add or remove files from it. const piece = { @@ -21,7 +24,8 @@ const piece = { }; // Runs render then validate in a copy of the repo with `files` written over it -// (null deletes). Returns { ok, out }: out is the failing step's output. +// (null deletes). Returns { ok, out, dist }: out is the failing step's output, +// dist(path) reads a rendered file. function run(files) { const dir = mkdtempSync(join(tmpdir(), "zenrows-integrations-test-")); try { @@ -43,7 +47,11 @@ function run(files) { const r = spawnSync("node", [join("scripts", `${script}.mjs`)], { cwd: dir, encoding: "utf8" }); if (r.status !== 0) return { ok: false, out: `${r.stdout}${r.stderr}` }; } - return { ok: true, out: "" }; + const rendered = new Map(Object.keys(files).filter((p) => p.startsWith("platforms/")).map((p) => { + const out = p.replace(/^platforms\//, "dist/"); + return [out, existsSync(join(dir, out)) ? readFileSync(join(dir, out), "utf8") : undefined]; + })); + return { ok: true, out: "", dist: (path) => rendered.get(path) }; } finally { rmSync(dir, { recursive: true, force: true }); } @@ -63,28 +71,54 @@ test("the repo as it is passes", passes({})); test("a code host passes", passes(piece)); test("a {{…}} in a file type the render doesn't fill fails", () => { - fails({ ...piece, "platforms/piece/src/view.tsx": "export const a = \"{{host_full}}\";\n" }, "src/view.tsx still has an unrendered {{…}}")(); - fails({ ...piece, "platforms/piece/setup.py": "name = \"{{host_full}}\"\n" }, "setup.py still has an unrendered {{…}}")(); + fails({ ...piece, "platforms/piece/src/view.tsx": "export const a = \"{{host_full}}\";\n" }, "src/view.tsx still has an unrendered {{host_full}}")(); + fails({ ...piece, "platforms/piece/setup.py": "name = \"{{host_full}}\"\n" }, "setup.py still has an unrendered {{host_full}}")(); }); test("a mistyped snippet token fails", fails({ ...piece, "shared/text/cost.md": "A request with js_render costs 5 credits.\n", "platforms/piece/src/index.ts": "export const cost = {{ < cost}};\n", -}, "src/index.ts still has an unrendered {{…}}")); +}, "src/index.ts still has an unrendered {{ < cost}}")); test("an unknown variable fails", fails({ ...piece, "platforms/piece/src/index.ts": "export const a = \"{{hots_full}}\";\n" }, "unknown variable {{hots_full}}")); test("a code host with no README fails", fails({ ...piece, "platforms/piece/README.md": null }, "a code host needs platforms/piece/README.md")); -test("a code host README with a {{…}} fails", fails({ ...piece, "platforms/piece/README.md": "# {{host_full}}\n" }, "README.md isn't rendered")); -test("a code host README that documents {{…}} in code passes", passes({ ...piece, "platforms/piece/README.md": "# Piece\n\nFacts come in through `{{text:name}}`.\n\n```\n{{< name}}\n```\n" })); test("a code host with a skills/ folder fails", fails({ ...piece, "platforms/piece/skills/scrape/SKILL.md": "x\n" }, "a code host (kind: code) ships no skills")); test("a skill reference in a code host fails", fails({ ...piece, "platforms/piece/src/index.ts": "export const s = \"{{skill:batch}}\";\n" }, "a code host (kind: code) ships no skills")); -test("an unknown kind fails", fails({ ...piece, "platforms/piece/vars.yaml": "host_full: Piece\nkind: Code\n" }, "kind: Code in vars.yaml")); +test("an unknown kind fails", () => { + fails({ ...piece, "platforms/piece/vars.yaml": "host_full: Piece\nkind: Code\n" }, 'platforms/piece/vars.yaml: kind is "Code"')(); + fails({ ...piece, "platforms/piece/vars.yaml": "host_full: Piece\nkind: [code]\n" }, 'platforms/piece/vars.yaml: kind is ["code"]')(); + fails({ ...piece, "platforms/piece/vars.yaml": "host_full: Piece\nkind:\n" }, "platforms/piece/vars.yaml: kind is null")(); +}); +test("a vars.yaml that isn't name: value lines fails", fails({ ...piece, "platforms/piece/vars.yaml": "- host_full: Piece\n" }, "platforms/piece/vars.yaml must be a list of name: value lines")); + +// {{…}} that isn't one of this tool's tokens ships as written. +test("other tools' {{…}} ship as written", () => { + const files = { + ...piece, + "platforms/piece/src/view.tsx": "export const v =

;\n", + "platforms/piece/communication.json": "{ \"url\": \"{{parameters.url}}\", \"n8n\": \"{{ $json.url }}\" }\n", + }; + const r = run(files); + assert.ok(r.ok, r.out); + assert.equal(r.dist("dist/piece/src/view.tsx"), files["platforms/piece/src/view.tsx"]); + assert.equal(r.dist("dist/piece/communication.json"), files["platforms/piece/communication.json"]); +}); // Snippets: plain text, one line, used somewhere. const snippet = (text) => ({ ...piece, "shared/text/cost.md": text, "platforms/piece/src/index.ts": "export const cost = {{text:cost}};\n" }); test("a snippet with parameter names and brackets passes", passes(snippet("A request with js_render returns [{...}] and costs 5 credits.\n"))); test("a snippet with backticks fails", fails(snippet("A request with `js_render` costs 5 credits.\n"), "has markdown")); test("a snippet with a link fails", fails(snippet("See [pricing](https://www.zenrows.com/pricing).\n"), "has markdown")); +test("a snippet with a blank last line or CRLF renders clean", () => { + for (const text of ["A request costs 5 credits.\n\n", "A request costs 5 credits.\r\n"]) { + const r = run(snippet(text)); + assert.ok(r.ok, r.out); + assert.equal(r.dist("dist/piece/src/index.ts"), "export const cost = \"A request costs 5 credits.\";\n"); + } +}); +test("a snippet with a {{…}} fails", fails(snippet("Ask {{host}} to sign in.\n"), "uses {{…}}; a snippet is a plain fact")); +test("a snippet named only in a vars.yaml comment fails", fails({ ...piece, "shared/text/cost.md": "A request costs 5 credits.\n", "platforms/piece/vars.yaml": "host_full: Piece\nkind: code\n# shared text goes in as {{< cost}}\n" }, "isn't used by any skill or host")); test("a snippet on two lines fails", fails(snippet("One fact.\nAnother fact.\n"), "has more than one line")); +test("a snippet named only in a code host README fails", fails({ ...piece, "shared/text/cost.md": "A request costs 5 credits.\n", "platforms/piece/README.md": "# Piece\n\nUses `{{< cost}}`.\n" }, "isn't used by any skill or host")); test("an unused snippet fails", fails({ ...piece, "shared/text/cost.md": "A request costs 5 credits.\n" }, "isn't used by any skill or host")); test("a missing snippet fails", fails({ ...piece, "platforms/piece/src/index.ts": "export const cost = {{text:nope}};\n" }, "there's no shared/text/nope.md")); From 751868cc667f8770eaa74fa37110a01549c8ee8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Tue, 10 Nov 2026 18:35:06 +0000 Subject: [PATCH 14/18] docs(contributing): describe which tokens fail the render and the snippet rules --- CONTRIBUTING.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 7a10f0f..a18f613 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -39,15 +39,15 @@ Requires Node 22. Run `npm ci` once. - **A shared skill a host shouldn't ship:** list it under `skip_skills:` in that host's `vars.yaml` (Cursor uses a rules file instead of `using-zenrows`). A `{{skill:…}}` reference to a skipped skill fails the render. - **A reference to another skill:** write `{{skill:batch}}`, never the plain name. It renders as that skill's name on the host: `skill_prefix` plus the shared name, unless `skill_names` in `vars.yaml` renames it (OpenClaw ships `scrape-webpage` as `zenrows-scrape`). The skill's own `name:` in its frontmatter uses it too. -Unknown variables, unknown skill names, missing slots, unused slot files, unknown snippets and any `{{…}}` left in a rendered file fail the render, and `npm run validate` fails on a reference to a skill the host doesn't ship, and on a skill name written plainly instead of as `{{skill:…}}`. +Unknown variables, unknown skill names, missing slots, unused slot files, unknown snippets and any of these tokens left in a rendered file (a typo, or a file type the render doesn't fill) fail the render. Other `{{…}}`, such as an n8n or Make expression or JSX's `style={{…}}`, isn't one of these tokens and ships as written, and `npm run validate` fails on a reference to a skill the host doesn't ship, and on a skill name written plainly instead of as `{{skill:…}}`. ## Shared text -A fact that more than one place states (what an error code means, what a request costs) lives once in `shared/text/.md`, as one line of plain text. A skill or a text file inlines it with `{{< name}}`; code uses `{{text:name}}`, which renders as a quoted string, and adds its own words around it. Snippets stay plain text so they read right in both, and `npm run validate` fails on markdown in one (backticks, `*`, a link or a heading), or on one nothing uses. +A fact that more than one place states (what an error code means, what a request costs) lives once in `shared/text/.md`, as one line of plain text with no `{{…}}`. A skill or a text file inlines it with `{{< name}}`; code uses `{{text:name}}`, which renders as a quoted string, and adds its own words around it. Snippets stay plain text so they read right in both, and `npm run validate` fails on markdown in one (backticks, `*`, a link or a heading), or on one nothing uses. ## Code hosts -`kind: code` in `vars.yaml` makes a host whose output isn't a plugin but files for somewhere else: code for another project (a piece as it sits in that project's repo), or a listing a marketplace reads. A code host ships only its own files, with `{{…}}` filled in source files too: no shared skills, `LICENSE` or assets, and no `plugin.json`. Its `README.md` documents it here and isn't shipped or rendered, so it writes its text out instead of using `{{…}}` (outside code spans, which document the syntax); `npm run validate` requires one, and the render fails if the host has a `skills/` folder. It needs no release-please entry until it's released from this repo. +`kind: code` in `vars.yaml` makes a host whose output isn't a plugin but files for somewhere else: code for another project (a piece as it sits in that project's repo), or a listing a marketplace reads. A code host ships only its own files, with `{{…}}` filled in source files too: no shared skills, `LICENSE` or assets, and no `plugin.json`. Its `README.md` documents it here and isn't shipped or rendered; `npm run validate` requires one, and the render fails if the host has a `skills/` folder. It needs no release-please entry until it's released from this repo. ## Tests From c4557312af168734a273d7c176f5d6d4da4f6408 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Wed, 11 Nov 2026 08:49:16 +0000 Subject: [PATCH 15/18] fix(validate): ask for upload: only on released hosts, and reject links Slack can't show --- CONTRIBUTING.md | 2 +- scripts/test.mjs | 11 +++++++++++ scripts/validate.mjs | 17 ++++++++++++----- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3673c6c..bc3b5b5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -92,4 +92,4 @@ Some hosts take each release only as an upload that someone does by hand, for ex - `upload: false`: installs read this repo or the host's mirror, so nothing is uploaded. - `upload:` with `to:` (where, for example `Cursor Marketplace`) and `url:` (an `https://` link to the page you upload on): each release of the host posts a reminder to #proj-integrations naming the release, its files in `dist//` at the tag, and that link. -`npm run validate` fails if a host has no `upload:`. `.github/workflows/upload-reminder.yml` posts through a Slack incoming webhook in the `SLACK_WEBHOOK_INTEGRATIONS` secret of the `release` environment and tags @ZenGrow. Without the secret, the job fails, so a missed reminder shows as a red run; the message is still in its job summary. To send a reminder again, run the Upload reminder workflow from Actions, on `main`, with the host and the release tag; it's a dry run unless you untick it. +`npm run validate` fails if a released host has no `upload:`. A host with no release-please entry (a code host not released from here) never posts a reminder, so it leaves `upload:` out, or writes `false`. `.github/workflows/upload-reminder.yml` posts through a Slack incoming webhook in the `SLACK_WEBHOOK_INTEGRATIONS` secret of the `release` environment and tags @ZenGrow. Without the secret, the job fails, so a missed reminder shows as a red run; the message is still in its job summary. To send a reminder again, run the Upload reminder workflow from Actions, on `main`, with the host and the release tag; it's a dry run unless you untick it. diff --git a/scripts/test.mjs b/scripts/test.mjs index 736befc..154907c 100644 --- a/scripts/test.mjs +++ b/scripts/test.mjs @@ -84,6 +84,17 @@ test("an unknown variable fails", fails({ ...piece, "platforms/piece/src/index.t test("a code host with no README fails", fails({ ...piece, "platforms/piece/README.md": null }, "a code host needs platforms/piece/README.md")); test("a code host with a skills/ folder fails", fails({ ...piece, "platforms/piece/skills/scrape/SKILL.md": "x\n" }, "a code host (kind: code) ships no skills")); test("a skill reference in a code host fails", fails({ ...piece, "platforms/piece/src/index.ts": "export const s = \"{{skill:batch}}\";\n" }, "a code host (kind: code) ships no skills")); +test("an unreleased code host needs no upload:", passes(piece)); +test("an unreleased code host with an upload: to and url fails", fails({ ...piece, "platforms/piece/vars.yaml": "host_full: Piece\nkind: code\nupload:\n to: npm\n url: https://www.npmjs.com/\n" }, "has no release, so an upload reminder can never fire")); +test("a released host without upload: fails", fails({ + "platforms/copilot/vars.yaml": readFileSync(join(root, "platforms/copilot/vars.yaml"), "utf8").replace(/^upload:.*\n/m, ""), +}, "copilot: vars.yaml has no upload:")); +test("an upload: link Slack can't show fails", () => { + const vars = readFileSync(join(root, "platforms/copilot/vars.yaml"), "utf8").replace(/^upload:.*\n/m, ""); + for (const upload of ['upload:\n to: X\n url: "https://a b"\n', 'upload:\n to: X\n url: "https://a|b"\n', 'upload:\n to: "A | B"\n url: https://example.com\n']) { + fails({ "platforms/copilot/vars.yaml": vars + upload }, "copilot: vars.yaml upload: must be false")(); + } +}); test("an unknown kind fails", () => { fails({ ...piece, "platforms/piece/vars.yaml": "host_full: Piece\nkind: Code\n" }, 'platforms/piece/vars.yaml: kind is "Code"')(); fails({ ...piece, "platforms/piece/vars.yaml": "host_full: Piece\nkind: [code]\n" }, 'platforms/piece/vars.yaml: kind is ["code"]')(); diff --git a/scripts/validate.mjs b/scripts/validate.mjs index 0a553fc..325a206 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -188,16 +188,23 @@ for (const host of hosts()) { } } - // Every host says whether its releases are uploaded somewhere by hand, so the - // release workflow knows whether to post a reminder. + // Every released host says whether its releases are uploaded somewhere by + // hand, so the release workflow knows whether to post a reminder. A host with + // no release-please entry (a code host not released from here) never gets a + // tag, so a reminder could never fire for it. const upload = values.upload; - if (upload === undefined) { + if (values.version === undefined) { + if (upload !== undefined && upload !== false) { + errors.push(`${host}: vars.yaml sets upload:, but ${host} has no release, so an upload reminder can never fire`); + } + } else if (upload === undefined) { errors.push(`${host}: vars.yaml has no upload: (false, or to: and url:; see CONTRIBUTING.md)`); } else if ( upload !== false && - !(typeof upload?.to === "string" && upload.to.trim() && typeof upload.url === "string" && upload.url.startsWith("https://")) + // The reminder links them as Slack's , which a space, <, > or | would cut short. + !(typeof upload?.to === "string" && upload.to.trim() && !/[<>|]/.test(upload.to) && typeof upload.url === "string" && /^https:\/\/[^\s<>|]+$/.test(upload.url)) ) { - errors.push(`${host}: vars.yaml upload: must be false, or to: and url: `); + errors.push(`${host}: vars.yaml upload: must be false, or to: and url: , without spaces in the link or <, > or | in either`); } // A whole-file override stops receiving fixes to the shared file it copied. From a94ab8cd8d386989168d64beea709075cb83b1d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Wed, 11 Nov 2026 08:49:16 +0000 Subject: [PATCH 16/18] ci(release): declare dry_run on the reusable upload reminder, as mirror.yml does --- .github/workflows/upload-reminder.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/upload-reminder.yml b/.github/workflows/upload-reminder.yml index 4b4acbe..4fde74b 100644 --- a/.github/workflows/upload-reminder.yml +++ b/.github/workflows/upload-reminder.yml @@ -15,6 +15,9 @@ on: description: "The release, -v" type: string required: true + dry_run: + type: boolean + default: false workflow_dispatch: inputs: host: @@ -43,8 +46,7 @@ jobs: env: HOST: ${{ inputs.host }} TAG: ${{ inputs.tag }} - # workflow_call has no dry_run input, so a release always posts. - DRY_RUN: ${{ inputs.dry_run == true }} + DRY_RUN: ${{ inputs.dry_run }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: From ec5224dc1e01678c8443d2f4720f4989de3a1160 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Wed, 11 Nov 2026 08:58:25 +0000 Subject: [PATCH 17/18] ci(release): word the upload reminder's notices like mirror.yml's --- .github/workflows/upload-reminder.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/upload-reminder.yml b/.github/workflows/upload-reminder.yml index 4fde74b..bbfdaba 100644 --- a/.github/workflows/upload-reminder.yml +++ b/.github/workflows/upload-reminder.yml @@ -71,12 +71,13 @@ jobs: const { HOST, TAG } = process.env; const v = vars(HOST); if (HOST + "-v" + v.version !== TAG) { - console.error("::error::" + TAG + " is not a " + HOST + " release: at that tag, " + HOST + " is at " + v.version + "."); + // \x27 is an apostrophe: a literal one would end this shell-quoted script. + console.error("::error::" + TAG + " isn\x27t a " + HOST + " release: at that tag, " + HOST + " is at " + v.version + "."); process.exit(1); } console.log(JSON.stringify(v.upload ?? false));') if [ "$upload" = false ]; then - echo "::notice::$HOST has upload: false in platforms/$HOST/vars.yaml, so there's nothing to upload by hand." + echo "::notice::$HOST has no upload to do by hand (upload: in platforms/$HOST/vars.yaml is false or missing)." exit 0 fi From cc40395e92643cc5bade3d98814ea6ea99981673 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Wed, 11 Nov 2026 09:46:40 +0000 Subject: [PATCH 18/18] fix(render): catch tokens with a stray space or capital letter --- scripts/lib.mjs | 7 ++++--- scripts/test.mjs | 18 +++++++++++++----- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/scripts/lib.mjs b/scripts/lib.mjs index 98dab3f..9d66649 100644 --- a/scripts/lib.mjs +++ b/scripts/lib.mjs @@ -20,9 +20,10 @@ const sharedDirs = { "shared/assets": "assets" }; const reserved = ["vars.yaml", "overrides.lock", "skills/"]; // What's left of one of this tool's tokens ({{name}}, {{skill:x}}, {{< x}}, // {{> x}}, {{text:x}}) after the render, for example in a file type it doesn't -// fill or with a stray space. Other {{…}} (n8n or Make expressions, JSX's -// style={{…}}) isn't ours and ships as written. -export const leftoverToken = /\{\{\s*(?:[<>]\??\s*[a-z0-9_-]+|[a-z_]+(?::[a-z0-9-]+)?)\s*\}\}/; +// fill, or with a stray space or capital letter the fill regexes don't take. +// Other {{…}} (n8n or Make expressions, JSX's style={{…}}) isn't ours and ships +// as written. +export const leftoverToken = /\{\{\s*(?:[<>]\??\s*[a-z0-9_-]+|[a-z_]+(?:\s*:\s*[a-z0-9-]+)?)\s*\}\}/i; // {{vars}} are filled in files with these extensions; others are copied as bytes. export const textFile = /\.(json|md|mdc|ya?ml|txt|toml|[cm]?[jt]s)$/; diff --git a/scripts/test.mjs b/scripts/test.mjs index 736befc..11a3a79 100644 --- a/scripts/test.mjs +++ b/scripts/test.mjs @@ -74,11 +74,19 @@ test("a {{…}} in a file type the render doesn't fill fails", () => { fails({ ...piece, "platforms/piece/src/view.tsx": "export const a = \"{{host_full}}\";\n" }, "src/view.tsx still has an unrendered {{host_full}}")(); fails({ ...piece, "platforms/piece/setup.py": "name = \"{{host_full}}\"\n" }, "setup.py still has an unrendered {{host_full}}")(); }); -test("a mistyped snippet token fails", fails({ - ...piece, - "shared/text/cost.md": "A request with js_render costs 5 credits.\n", - "platforms/piece/src/index.ts": "export const cost = {{ < cost}};\n", -}, "src/index.ts still has an unrendered {{ < cost}}")); +test("a mistyped snippet token fails", () => { + for (const token of ["{{ < cost}}", "{{text: cost}}", "{{< COST}}", "{{Text:cost}}"]) { + fails({ + ...piece, + "shared/text/cost.md": "A request with js_render costs 5 credits.\n", + "platforms/piece/src/index.ts": `export const cost = ${token};\n`, + }, `src/index.ts still has an unrendered ${token}`)(); + } +}); +test("a mistyped skill or variable token fails", () => { + fails({ ...piece, "platforms/piece/src/index.ts": "export const s = \"{{skill: batch}}\";\n" }, "still has an unrendered {{skill: batch}}")(); + fails({ ...piece, "platforms/piece/src/index.ts": "export const a = \"{{Host_full}}\";\n" }, "still has an unrendered {{Host_full}}")(); +}); test("an unknown variable fails", fails({ ...piece, "platforms/piece/src/index.ts": "export const a = \"{{hots_full}}\";\n" }, "unknown variable {{hots_full}}")); test("a code host with no README fails", fails({ ...piece, "platforms/piece/README.md": null }, "a code host needs platforms/piece/README.md"));