From e82033e5f074597980f6793e3d2ec2fd55db9b69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Fri, 13 Nov 2026 11:59:29 +0000 Subject: [PATCH 1/5] feat(skills): say what Zenrows does once, in shared/text/ One sentence in shared/text/zenrows-description.md, used by Copilot's plugin.json and README and the using-zenrows skill. validate now fails when a host or skill writes a snippet's text out instead of using its token. --- dist/copilot/README.md | 2 +- dist/copilot/plugin.json | 2 +- dist/copilot/skills/using-zenrows/SKILL.md | 2 +- platforms/copilot/README.md | 2 +- platforms/copilot/plugin.json | 2 +- scripts/test.mjs | 1 + scripts/validate.mjs | 8 +++++++- shared/text/zenrows-description.md | 1 + skills/using-zenrows/SKILL.md | 2 +- 9 files changed, 15 insertions(+), 7 deletions(-) create mode 100644 shared/text/zenrows-description.md diff --git a/dist/copilot/README.md b/dist/copilot/README.md index 774dddc..bcb5db2 100644 --- a/dist/copilot/README.md +++ b/dist/copilot/README.md @@ -1,6 +1,6 @@ # Zenrows for GitHub Copilot -Live web data for GitHub Copilot. This plugin connects Copilot to the Zenrows MCP server, so Copilot can fetch the current content of any public page, pull structured fields from it, run batch jobs over many URLs, and drive a hosted browser for pages that need clicks, forms or pagination. It also adds skills that tell Copilot which Zenrows tool fits each task and how to keep requests efficient. +Live web data for GitHub Copilot. This plugin connects Copilot to the Zenrows MCP server, so it can fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows. It also adds skills that tell Copilot which Zenrows tool fits each task and how to keep requests efficient. ## What you get diff --git a/dist/copilot/plugin.json b/dist/copilot/plugin.json index 959deee..545c8b3 100644 --- a/dist/copilot/plugin.json +++ b/dist/copilot/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "zenrows", "version": "0.1.1", - "description": "Live web data for GitHub Copilot: fetch pages, extract structured data, run batch jobs and drive hosted browser sessions through the Zenrows MCP server.", + "description": "Live web data for GitHub Copilot through the Zenrows MCP server: fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows.", "author": { "name": "Zenrows", "email": "integrations@zenrows.com", diff --git a/dist/copilot/skills/using-zenrows/SKILL.md b/dist/copilot/skills/using-zenrows/SKILL.md index 02c60f6..94d93ba 100644 --- a/dist/copilot/skills/using-zenrows/SKILL.md +++ b/dist/copilot/skills/using-zenrows/SKILL.md @@ -5,7 +5,7 @@ description: Overview of the Zenrows tools for live web data. Use when a task ne # Using Zenrows -Zenrows gives Copilot live web data through the Zenrows MCP server: page fetching, structured extraction, batch jobs over many URLs, crawls from a start URL, and hosted browser sessions. Zenrows is a paid service: every request uses credits on the user's Zenrows plan. +Zenrows gives Copilot live web data through the Zenrows MCP server: Copilot can fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows. Zenrows is a paid service: every request uses credits on the user's Zenrows plan. The Zenrows MCP server signs in with OAuth. If the Zenrows tools are missing or return `AUTH003`, use the `getting-started` skill. Never ask the user to paste an API key into the conversation. diff --git a/platforms/copilot/README.md b/platforms/copilot/README.md index 774dddc..eddebb2 100644 --- a/platforms/copilot/README.md +++ b/platforms/copilot/README.md @@ -1,6 +1,6 @@ # Zenrows for GitHub Copilot -Live web data for GitHub Copilot. This plugin connects Copilot to the Zenrows MCP server, so Copilot can fetch the current content of any public page, pull structured fields from it, run batch jobs over many URLs, and drive a hosted browser for pages that need clicks, forms or pagination. It also adds skills that tell Copilot which Zenrows tool fits each task and how to keep requests efficient. +Live web data for {{host_full}}. This plugin connects {{host}} to the Zenrows MCP server, so it can {{< zenrows-description}}. It also adds skills that tell Copilot which Zenrows tool fits each task and how to keep requests efficient. ## What you get diff --git a/platforms/copilot/plugin.json b/platforms/copilot/plugin.json index 91d526f..fa9a959 100644 --- a/platforms/copilot/plugin.json +++ b/platforms/copilot/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "zenrows", "version": "{{version}}", - "description": "Live web data for GitHub Copilot: fetch pages, extract structured data, run batch jobs and drive hosted browser sessions through the Zenrows MCP server.", + "description": "Live web data for {{host_full}} through the Zenrows MCP server: {{< zenrows-description}}.", "author": { "name": "Zenrows", "email": "integrations@zenrows.com", diff --git a/scripts/test.mjs b/scripts/test.mjs index 3ea8b3a..93141dc 100644 --- a/scripts/test.mjs +++ b/scripts/test.mjs @@ -149,5 +149,6 @@ test("a snippet with a {{…}} fails", fails(snippet("Ask {{host}} to sign in.\n test("a snippet named only in a vars.yaml comment fails", fails({ ...piece, "shared/text/cost.md": "A request costs 5 credits.\n", "platforms/piece/vars.yaml": "host_full: Piece\nkind: code\n# shared text goes in as {{< cost}}\n" }, "isn't used by any skill or host")); test("a snippet on two lines fails", fails(snippet("One fact.\nAnother fact.\n"), "has more than one line")); test("a snippet named only in a code host README fails", fails({ ...piece, "shared/text/cost.md": "A request costs 5 credits.\n", "platforms/piece/README.md": "# Piece\n\nUses `{{< cost}}`.\n" }, "isn't used by any skill or host")); +test("a snippet written out instead of its token fails", fails({ ...piece, "shared/text/cost.md": "A request costs 5 credits.\n", "platforms/piece/src/index.ts": "export const a = {{text:cost}};\n// A request costs 5 credits.\n" }, "platforms/piece/src/index.ts writes out shared/text/cost.md")); test("an unused snippet fails", fails({ ...piece, "shared/text/cost.md": "A request costs 5 credits.\n" }, "isn't used by any skill or host")); test("a missing snippet fails", fails({ ...piece, "platforms/piece/src/index.ts": "export const cost = {{text:nope}};\n" }, "there's no shared/text/nope.md")); diff --git a/scripts/validate.mjs b/scripts/validate.mjs index 325a206..3f365fc 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -2,7 +2,7 @@ // (against the JSON Schema they declare), the marketplace catalogs at the top of // the repo, and that no whole-file override was copied from an older shared file. import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; -import { basename, dirname, join } from "node:path"; +import { basename, dirname, join, relative } from "node:path"; import Ajv2020 from "ajv/dist/2020.js"; import { parse as parseYaml } from "yaml"; import { catalogs, hostSkillName, hostSkills, hosts, isCodeHost, listFiles, platformsDir, readLock, renderHost, root, sha256, sharedTextDir, skillNames, textFile, vars } from "./lib.mjs"; @@ -278,6 +278,12 @@ for (const file of listFiles(sharedTextDir)) { // _ and [ ] don't count: parameter names (js_render) and [{...}] use them. if (/[`*]/.test(text) || /\]\(/.test(text) || /^#/.test(text.trim())) errors.push(`shared/text/${file} has markdown; keep snippets plain text so code can use them`); if (text.trim().includes("\n")) errors.push(`shared/text/${file} has more than one line; keep one fact per snippet`); + // A copy of the fact written out would miss the next change to it. + for (const source of snippetSources) { + if (readFileSync(source, "utf8").includes(text.trim())) { + errors.push(`${relative(root, source)} writes out shared/text/${file}; use {{< ${name}}} (or {{text:${name}}} in code) instead`); + } + } } console.log(report.join("\n")); diff --git a/shared/text/zenrows-description.md b/shared/text/zenrows-description.md new file mode 100644 index 0000000..bdb45e5 --- /dev/null +++ b/shared/text/zenrows-description.md @@ -0,0 +1 @@ +fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows diff --git a/skills/using-zenrows/SKILL.md b/skills/using-zenrows/SKILL.md index f8e0a31..d2b2c86 100644 --- a/skills/using-zenrows/SKILL.md +++ b/skills/using-zenrows/SKILL.md @@ -5,7 +5,7 @@ description: Overview of the Zenrows tools for live web data. Use when a task ne # Using Zenrows -Zenrows gives {{host}} live web data through the Zenrows MCP server: page fetching, structured extraction, batch jobs over many URLs, crawls from a start URL, and hosted browser sessions. Zenrows is a paid service: every request uses credits on the user's Zenrows plan. +Zenrows gives {{host}} live web data through the Zenrows MCP server: {{host}} can {{< zenrows-description}}. Zenrows is a paid service: every request uses credits on the user's Zenrows plan. The {{connection}} signs in with OAuth. If the Zenrows tools are missing or return `AUTH003`, use the `{{skill:getting-started}}` skill. Never ask the user to paste an API key into the conversation. From cf79b822e3eba168c83aa4ff9a82326033043277 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Fri, 13 Nov 2026 17:07:37 +0000 Subject: [PATCH 2/5] feat(render): render every plugin's README from one template shared/README.md is every plugin host's README; a host writes only readme/setup.md and, if it needs more, readme/more.md. The description snippet now carries its own lead-in, so manifests, the README and the using-zenrows skill use just the token. {{>? slot}} makes a slot optional. new-host scaffolds the setup slot instead of a whole README. --- CONTRIBUTING.md | 6 ++- dist/copilot/README.md | 4 +- dist/copilot/plugin.json | 2 +- dist/copilot/skills/using-zenrows/SKILL.md | 2 +- platforms/copilot/README.md | 55 ---------------------- platforms/copilot/plugin.json | 2 +- platforms/copilot/readme/setup.md | 15 ++++++ scripts/lib.mjs | 41 +++++++++++++--- scripts/new-host.mjs | 36 +++----------- scripts/test.mjs | 21 +++++++-- scripts/validate.mjs | 1 + shared/README.md | 43 +++++++++++++++++ shared/text/zenrows-description.md | 2 +- skills/using-zenrows/SKILL.md | 2 +- 14 files changed, 128 insertions(+), 104 deletions(-) delete mode 100644 platforms/copilot/README.md create mode 100644 platforms/copilot/readme/setup.md create mode 100644 shared/README.md diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bc3b5b5..0e0bb28 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,13 +6,15 @@ Each host's plugin in `dist//` is rendered from shared skills plus that ho ``` skills//SKILL.md shared skills; {{var}} and {{> slot}} mark what differs per host +shared/README.md every plugin's README; {{> setup}} and {{>? more}} take the host's own sections shared/assets/ files every plugin host ships as they are (logo) shared/text/.md one fact in plain text, used by skills ({{< name}}) and code hosts ({{text:name}}) platforms// vars.yaml values for {{var}} skills//.md the host's text for a {{> slot}} in that skill + readme/setup.md the host's Setup section of its README (readme/more.md: optional extra sections) overrides.lock whole-file overrides and the shared version each was copied from (only once one exists) - everything else the host's manifests, README, …, copied into dist// ({{version}} comes from release-please) + everything else the host's manifests, …, copied into dist// ({{version}} comes from release-please) dist// GENERATED: the plugin the host installs, or a code host's files .github/plugin/marketplace.json Copilot's marketplace catalog (see below) scripts/ render, validate, override, new-host, mirror, test @@ -34,6 +36,8 @@ Requires Node 22. Run `npm ci` once. - **A word or phrase:** use a variable, for example `{{host}}`, and set it in `platforms//vars.yaml`. - **A paragraph or a few lines:** use a slot, for example `{{> connect}}`, and write the host's text in `platforms//skills//connect.md`. Everything outside the slot stays shared, so later fixes still reach the host. +- **An optional paragraph:** `{{>? name}}` is a slot a host may leave out; without the file, its line goes. The README uses it for `readme/more.md`. +- **The README:** every plugin's README is `shared/README.md`. A host writes only `readme/setup.md` (how to install and sign in) and, if it needs more, `readme/more.md`; its own `README.md` fails the render. Code hosts keep their own README. - **The whole file, as a last resort:** `npm run override -- ` copies the shared skill into `platforms//` and records which version it was copied from in `overrides.lock`. When the shared skill changes later, `npm run validate` fails until someone updates the copy, or confirms the change doesn't apply, and runs `npm run override:ack -- `. - **A skill only one host has:** put it in `platforms//skills//SKILL.md`, with no shared skill of that name. It ships only on that host and needs no `overrides.lock` entry. - **A shared skill a host shouldn't ship:** list it under `skip_skills:` in that host's `vars.yaml` (Cursor uses a rules file instead of `using-zenrows`). A `{{skill:…}}` reference to a skipped skill fails the render. diff --git a/dist/copilot/README.md b/dist/copilot/README.md index bcb5db2..5e4428f 100644 --- a/dist/copilot/README.md +++ b/dist/copilot/README.md @@ -1,11 +1,11 @@ # Zenrows for GitHub Copilot -Live web data for GitHub Copilot. This plugin connects Copilot to the Zenrows MCP server, so it can fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows. It also adds skills that tell Copilot which Zenrows tool fits each task and how to keep requests efficient. +Live web data: fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows. This plugin connects Copilot to the Zenrows MCP server and adds skills that tell Copilot which Zenrows tool fits each task and how to keep requests efficient. ## What you get - The Zenrows MCP server (`https://mcp.zenrows.com/mcp`). You sign in to your Zenrows account with OAuth; there is no API key to copy. -- Nine skills: +- Skills: | Skill | Use it for | |---|---| diff --git a/dist/copilot/plugin.json b/dist/copilot/plugin.json index 545c8b3..f9a2ee4 100644 --- a/dist/copilot/plugin.json +++ b/dist/copilot/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "zenrows", "version": "0.1.1", - "description": "Live web data for GitHub Copilot through the Zenrows MCP server: fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows.", + "description": "Live web data: fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows.", "author": { "name": "Zenrows", "email": "integrations@zenrows.com", diff --git a/dist/copilot/skills/using-zenrows/SKILL.md b/dist/copilot/skills/using-zenrows/SKILL.md index 94d93ba..0355e36 100644 --- a/dist/copilot/skills/using-zenrows/SKILL.md +++ b/dist/copilot/skills/using-zenrows/SKILL.md @@ -5,7 +5,7 @@ description: Overview of the Zenrows tools for live web data. Use when a task ne # Using Zenrows -Zenrows gives Copilot live web data through the Zenrows MCP server: Copilot can fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows. Zenrows is a paid service: every request uses credits on the user's Zenrows plan. +Live web data: fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows. GitHub Copilot reaches it through the Zenrows MCP server. Zenrows is a paid service: every request uses credits on the user's Zenrows plan. The Zenrows MCP server signs in with OAuth. If the Zenrows tools are missing or return `AUTH003`, use the `getting-started` skill. Never ask the user to paste an API key into the conversation. diff --git a/platforms/copilot/README.md b/platforms/copilot/README.md deleted file mode 100644 index eddebb2..0000000 --- a/platforms/copilot/README.md +++ /dev/null @@ -1,55 +0,0 @@ -# Zenrows for GitHub Copilot - -Live web data for {{host_full}}. This plugin connects {{host}} to the Zenrows MCP server, so it can {{< zenrows-description}}. It also adds skills that tell Copilot which Zenrows tool fits each task and how to keep requests efficient. - -## What you get - -- The Zenrows MCP server (`https://mcp.zenrows.com/mcp`). You sign in to your Zenrows account with OAuth; there is no API key to copy. -- Nine skills: - -| Skill | Use it for | -|---|---| -| `using-zenrows` | Overview: tools, defaults, errors and plan limits | -| `getting-started` | First-run checks and troubleshooting | -| `scrape-webpage` | Reading one page | -| `extract-structured-data` | Named fields or JSON from one page | -| `batch` | A list of five or more URLs as one job | -| `crawl` | Many pages under one site section | -| `map` | Listing a site's URLs without fetching them | -| `browser-automation` | Clicks, forms, pagination and multi-step flows | -| `capture-api-calls` | Finding the API calls a page makes | - -## Requirements - -A Zenrows account. Requests use credits on your Zenrows plan. You can [sign up here](https://app.zenrows.com/register). - -## Setup - -Install the plugin. In Copilot CLI, add this repository as a marketplace and install from it: - -``` -copilot plugin marketplace add ZenRows/zenrows-integrations -copilot plugin install zenrows@zenrows -``` - -In VS Code, run **Chat: Plugins** from the Command Palette and install from `ZenRows/zenrows-integrations`. For Copilot cloud agent, add the plugin to `enabledPlugins` in the repository's `.github/copilot/settings.json`. - -Then sign in to Zenrows: - -- **Copilot CLI:** run `/mcp auth zenrows` and sign in in the browser. -- **VS Code:** run **MCP: List Servers**, select `zenrows`, start it and allow it to authenticate. - -Then ask Copilot something like: "Use Zenrows to get the product names and prices from https://www.scrapingcourse.com/ecommerce/". - -## Privacy - -The plugin sends the URLs and options you ask Copilot to fetch to Zenrows, which processes them under the [Zenrows privacy policy](https://www.zenrows.com/legal/privacy). - -## Support - -- Documentation: [docs.zenrows.com](https://docs.zenrows.com) -- Contact: [integrations@zenrows.com](mailto:integrations@zenrows.com) - -## License - -MIT diff --git a/platforms/copilot/plugin.json b/platforms/copilot/plugin.json index fa9a959..4f6289d 100644 --- a/platforms/copilot/plugin.json +++ b/platforms/copilot/plugin.json @@ -2,7 +2,7 @@ "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "zenrows", "version": "{{version}}", - "description": "Live web data for {{host_full}} through the Zenrows MCP server: {{< zenrows-description}}.", + "description": "{{< zenrows-description}}.", "author": { "name": "Zenrows", "email": "integrations@zenrows.com", diff --git a/platforms/copilot/readme/setup.md b/platforms/copilot/readme/setup.md new file mode 100644 index 0000000..cbf186b --- /dev/null +++ b/platforms/copilot/readme/setup.md @@ -0,0 +1,15 @@ +Install the plugin. In Copilot CLI, add this repository as a marketplace and install from it: + +``` +copilot plugin marketplace add ZenRows/zenrows-integrations +copilot plugin install zenrows@zenrows +``` + +In VS Code, run **Chat: Plugins** from the Command Palette and install from `ZenRows/zenrows-integrations`. For Copilot cloud agent, add the plugin to `enabledPlugins` in the repository's `.github/copilot/settings.json`. + +Then sign in to Zenrows: + +- **Copilot CLI:** run `/mcp auth zenrows` and sign in in the browser. +- **VS Code:** run **MCP: List Servers**, select `zenrows`, start it and allow it to authenticate. + +Then ask Copilot something like: "Use Zenrows to get the product names and prices from https://www.scrapingcourse.com/ecommerce/". diff --git a/scripts/lib.mjs b/scripts/lib.mjs index a88f921..286a12a 100644 --- a/scripts/lib.mjs +++ b/scripts/lib.mjs @@ -17,7 +17,10 @@ const sharedFiles = { LICENSE: "LICENSE" }; const sharedDirs = { "shared/assets": "assets" }; // In platforms//, these are inputs to the render, not files of the plugin. // Everything else there (manifests, README, …) is copied into dist//. -const reserved = ["vars.yaml", "overrides.lock", "skills/"]; +const reserved = ["vars.yaml", "overrides.lock", "skills/", "readme/"]; +// A plugin host's README.md is rendered from this template, with the host's +// slots from platforms//readme/. +const readmeTemplate = join(root, "shared", "README.md"); // What's left of one of this tool's tokens ({{name}}, {{skill:x}}, {{text:x}}, // {{< x}}, {{> x}}) after the render, for example in a file type it doesn't // fill, or with a stray space, capital letter, _ or . the fill regexes don't @@ -158,6 +161,23 @@ function fill(text, values, where) { } // Renders one host. Returns Map, Buffer>. +// Puts a host's slot files into text: {{> slot}} needs /.md; {{>? slot}} +// is optional, and without the file its line (and one blank line after it) goes. +// Records each slot file used in usedSlots. +function fillSlots(text, dir, where, usedSlots) { + return text + .replace(/^[ \t]*\{\{>\?\s*([a-z0-9_-]+)\s*\}\}[ \t]*\n(\n)?/gm, (line, slot) => (existsSync(join(dir, `${slot}.md`)) ? line : "")) + .replace(/\{\{>(\?)?\s*([a-z0-9_-]+)\s*\}\}/g, (_, optional, slot) => { + const file = join(dir, `${slot}.md`); + if (!existsSync(file)) { + if (optional) return ""; + throw new Error(`${where}: slot {{> ${slot}}} has no ${relative(root, file)}`); + } + usedSlots.add(relative(root, file)); + return readFileSync(file, "utf8").replace(/\n$/, ""); + }); +} + export function renderHost(host) { const values = vars(host); const skills = hostSkills(host, values); @@ -171,12 +191,7 @@ export function renderHost(host) { const own = join(platform, "skills", name, "SKILL.md"); const source = existsSync(own) ? own : join(skillsDir, name, "SKILL.md"); const where = relative(root, source); - let text = readFileSync(source, "utf8").replace(/\{\{>\s*([a-z0-9_-]+)\s*\}\}/g, (_, slot) => { - const file = join(platform, "skills", name, `${slot}.md`); - if (!existsSync(file)) throw new Error(`${where}: slot {{> ${slot}}} has no ${relative(root, file)}`); - usedSlots.add(relative(root, file)); - return readFileSync(file, "utf8").replace(/\n$/, ""); - }); + let text = fillSlots(readFileSync(source, "utf8"), join(platform, "skills", name), where, usedSlots); text = fill(text, values, where); out.set(`skills/${hostSkillName(values, name)}/SKILL.md`, Buffer.from(text)); } @@ -187,6 +202,18 @@ export function renderHost(host) { if (!usedSlots.has(path)) throw new Error(`${path}: no skill this host ships has this slot`); } + if (!isCodeHost(values)) { + if (existsSync(join(platform, "README.md"))) { + throw new Error(`platforms/${host}/README.md: a plugin's README comes from shared/README.md; put the host's own text in platforms/${host}/readme/setup.md (and readme/more.md)`); + } + const where = relative(root, readmeTemplate); + out.set("README.md", Buffer.from(fill(fillSlots(readFileSync(readmeTemplate, "utf8"), join(platform, "readme"), where, usedSlots), values, where))); + for (const file of listFiles(join(platform, "readme"))) { + const path = relative(root, join(platform, "readme", file)); + if (!usedSlots.has(path)) throw new Error(`${path}: shared/README.md has no slot for it`); + } + } + for (const [from, to] of isCodeHost(values) ? [] : Object.entries(sharedFiles)) out.set(to, readFileSync(join(root, from))); for (const [from, to] of isCodeHost(values) ? [] : Object.entries(sharedDirs)) { for (const file of listFiles(join(root, from))) out.set(`${to}/${file}`, readFileSync(join(root, from, file))); diff --git a/scripts/new-host.mjs b/scripts/new-host.mjs index 748ab4f..8ad4704 100644 --- a/scripts/new-host.mjs +++ b/scripts/new-host.mjs @@ -1,6 +1,6 @@ // npm run new-host: asks for the host's id and full name, and creates // platforms// from templates (vars, a generic manifest and MCP config, the -// README and the getting-started sign-in slot) plus the host's release-please +// README setup slot and the getting-started sign-in slot) plus the host's release-please // package. What only a person can decide is marked TODO(new-host), and // npm run validate fails until it's done. import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; @@ -74,13 +74,13 @@ files.set( ); // A generic manifest and MCP config. Each host reads its own layout, so the -// README's TODO asks for them to be moved and completed. +// setup slot's TODO asks for them to be moved and completed. files.set( "plugin.json", json({ name: "zenrows", version: "{{version}}", - description: `Live web data for ${name}: fetch pages, extract structured data, run batch jobs and drive hosted browser sessions through the Zenrows MCP server.`, + description: "{{< zenrows-description}}.", author: { name: "Zenrows", email: "integrations@zenrows.com", url: "https://www.zenrows.com" }, homepage: "https://docs.zenrows.com", repository: "https://github.com/ZenRows/zenrows-integrations", @@ -98,36 +98,14 @@ files.set( ` - TODO(new-host): how to connect and sign in to the \`zenrows\` server in ${name}, one bullet per app (CLI, IDE, …).\n`, ); +// The README comes from shared/README.md; the host writes only its Setup section. files.set( - "README.md", - `# Zenrows for ${name} - -TODO(new-host): before this host ships, check its plugin files. platforms/${id}/plugin.json and mcp.json are generic templates: move them to the paths ${name} reads (for example .claude-plugin/plugin.json), add the fields it needs, update the manifest path in release-please-config.json, and add a marketplace catalog at the top of the repo if ${name} installs from a subfolder. Then delete this paragraph. - -Live web data for ${name}. This plugin connects ${name} to the Zenrows MCP server, so it can fetch the current content of any public page, pull structured fields from it, run batch jobs over many URLs, and drive a hosted browser for pages that need clicks, forms or pagination. It also adds skills that tell ${name} which Zenrows tool fits each task and how to keep requests efficient. - -## Requirements - -A Zenrows account. Requests use credits on your Zenrows plan. You can [sign up here](https://app.zenrows.com/register). - -## Setup + "readme/setup.md", + `TODO(new-host): before this host ships, check its plugin files. platforms/${id}/plugin.json and mcp.json are generic templates: move them to the paths ${name} reads (for example .claude-plugin/plugin.json), add the fields it needs, update the manifest path in release-please-config.json, and add a marketplace catalog at the top of the repo if ${name} installs from a subfolder. Then delete this paragraph. TODO(new-host): how to install the plugin in ${name}, and how to sign in to Zenrows. Then ask ${name} something like: "Use Zenrows to get the product names and prices from https://www.scrapingcourse.com/ecommerce/". - -## Privacy - -The plugin sends the URLs and options you ask ${name} to fetch to Zenrows, which processes them under the [Zenrows privacy policy](https://www.zenrows.com/legal/privacy). - -## Support - -- Documentation: [docs.zenrows.com](https://docs.zenrows.com) -- Contact: [integrations@zenrows.com](mailto:integrations@zenrows.com) - -## License - -MIT `, ); @@ -163,7 +141,7 @@ writeFileSync(manifestPath, json(versions)); console.log(` Created platforms/${id}/ from templates. Next: 1. Fill in every TODO(new-host): grep -rn "TODO(new-host)" platforms/${id} - (npm run validate fails until they're gone). The README's first TODO covers + (npm run validate fails until they're gone). readme/setup.md's first TODO covers moving plugin.json and mcp.json to the layout ${name} reads. 2. npm run render, then npm run validate, and commit platforms/${id}, dist/${id} and the release-please files together.`); diff --git a/scripts/test.mjs b/scripts/test.mjs index 93141dc..61e8dd3 100644 --- a/scripts/test.mjs +++ b/scripts/test.mjs @@ -26,7 +26,7 @@ const piece = { // Runs render then validate in a copy of the repo with `files` written over it // (null deletes). Returns { ok, out, dist }: out is the failing step's output, // dist(path) reads a rendered file. -function run(files) { +function run(files, reads = []) { const dir = mkdtempSync(join(tmpdir(), "zenrows-integrations-test-")); try { for (const path of copied) { @@ -47,10 +47,8 @@ function run(files) { const r = spawnSync("node", [join("scripts", `${script}.mjs`)], { cwd: dir, encoding: "utf8" }); if (r.status !== 0) return { ok: false, out: `${r.stdout}${r.stderr}` }; } - const rendered = new Map(Object.keys(files).filter((p) => p.startsWith("platforms/")).map((p) => { - const out = p.replace(/^platforms\//, "dist/"); - return [out, existsSync(join(dir, out)) ? readFileSync(join(dir, out), "utf8") : undefined]; - })); + const outs = [...Object.keys(files).filter((p) => p.startsWith("platforms/")).map((p) => p.replace(/^platforms\//, "dist/")), ...reads]; + const rendered = new Map(outs.map((out) => [out, existsSync(join(dir, out)) ? readFileSync(join(dir, out), "utf8") : undefined])); return { ok: true, out: "", dist: (path) => rendered.get(path) }; } finally { rmSync(dir, { recursive: true, force: true }); @@ -133,6 +131,19 @@ test("other tools' {{…}} ship as written", () => { for (const [path, text] of Object.entries(theirs)) assert.equal(r.dist(path.replace(/^platforms\//, "dist/")), text); }); +// A plugin's README is shared/README.md with the host's slots from readme/. +const copilotReadme = () => readFileSync(join(root, "dist/copilot/README.md"), "utf8"); +test("a plugin host with its own README.md fails", fails({ "platforms/copilot/README.md": "# Copilot\n" }, "a plugin's README comes from shared/README.md")); +test("a plugin host without readme/setup.md fails", fails({ "platforms/copilot/readme/setup.md": null }, "slot {{> setup}} has no platforms/copilot/readme/setup.md")); +test("a readme/ file shared/README.md has no slot for fails", fails({ "platforms/copilot/readme/extra.md": "x\n" }, "platforms/copilot/readme/extra.md: shared/README.md has no slot for it")); +test("the optional more slot is left out without its file and put in with it", () => { + assert.ok(!copilotReadme().includes("{{"), "no slot token left in the rendered README"); + assert.ok(!copilotReadme().includes("For maintainers")); + const r = run({ "platforms/copilot/readme/more.md": "## For maintainers\n\nRelease notes live elsewhere.\n" }, ["dist/copilot/README.md"]); + assert.ok(r.ok, r.out); + assert.match(r.dist("dist/copilot/README.md"), /## Setup\n[\s\S]*## For maintainers\n\nRelease notes live elsewhere\.\n\n## Privacy/); +}); + // Snippets: plain text, one line, used somewhere. const snippet = (text) => ({ ...piece, "shared/text/cost.md": text, "platforms/piece/src/index.ts": "export const cost = {{text:cost}};\n" }); test("a snippet with parameter names and brackets passes", passes(snippet("A request with js_render returns [{...}] and costs 5 credits.\n"))); diff --git a/scripts/validate.mjs b/scripts/validate.mjs index 3f365fc..1bc2888 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -260,6 +260,7 @@ for (const host of hosts()) { // inputs, so a snippet named only there isn't used. const codeHostReadmes = hosts().filter((h) => isCodeHost(vars(h))).map((h) => `${h}/README.md`); const snippetSources = [ + join(root, "shared", "README.md"), ...skillNames().map((n) => join(root, "skills", n, "SKILL.md")), ...listFiles(platformsDir) .filter((f) => !codeHostReadmes.includes(f) && !/\/(vars\.yaml|overrides\.lock)$/.test(f)) diff --git a/shared/README.md b/shared/README.md new file mode 100644 index 0000000..cc439db --- /dev/null +++ b/shared/README.md @@ -0,0 +1,43 @@ +# Zenrows for {{host_full}} + +{{< zenrows-description}}. This plugin connects {{host}} to the Zenrows MCP server and adds skills that tell {{host}} which Zenrows tool fits each task and how to keep requests efficient. + +## What you get + +- The Zenrows MCP server (`https://mcp.zenrows.com/mcp`). You sign in to your Zenrows account with OAuth; there is no API key to copy. +- Skills: + +| Skill | Use it for | +|---|---| +| `{{skill:using-zenrows}}` | Overview: tools, defaults, errors and plan limits | +| `{{skill:getting-started}}` | First-run checks and troubleshooting | +| `{{skill:scrape-webpage}}` | Reading one page | +| `{{skill:extract-structured-data}}` | Named fields or JSON from one page | +| `{{skill:batch}}` | A list of five or more URLs as one job | +| `{{skill:crawl}}` | Many pages under one site section | +| `{{skill:map}}` | Listing a site's URLs without fetching them | +| `{{skill:browser-automation}}` | Clicks, forms, pagination and multi-step flows | +| `{{skill:capture-api-calls}}` | Finding the API calls a page makes | + +## Requirements + +A Zenrows account. Requests use credits on your Zenrows plan. You can [sign up here](https://app.zenrows.com/register). + +## Setup + +{{> setup}} + +{{>? more}} + +## Privacy + +The plugin sends the URLs and options you ask {{host}} to fetch to Zenrows, which processes them under the [Zenrows privacy policy](https://www.zenrows.com/legal/privacy). + +## Support + +- Documentation: [docs.zenrows.com](https://docs.zenrows.com) +- Contact: [integrations@zenrows.com](mailto:integrations@zenrows.com) + +## License + +MIT diff --git a/shared/text/zenrows-description.md b/shared/text/zenrows-description.md index bdb45e5..d550b69 100644 --- a/shared/text/zenrows-description.md +++ b/shared/text/zenrows-description.md @@ -1 +1 @@ -fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows +Live web data: fetch any public page as Markdown, HTML or text, pull structured fields from it as JSON, run batch jobs over many URLs, crawl a site from one start URL and drive a hosted browser for pages that need clicks, forms or pagination, with proxies, JavaScript rendering and anti-bot protection handled by Zenrows diff --git a/skills/using-zenrows/SKILL.md b/skills/using-zenrows/SKILL.md index d2b2c86..4bd1529 100644 --- a/skills/using-zenrows/SKILL.md +++ b/skills/using-zenrows/SKILL.md @@ -5,7 +5,7 @@ description: Overview of the Zenrows tools for live web data. Use when a task ne # Using Zenrows -Zenrows gives {{host}} live web data through the Zenrows MCP server: {{host}} can {{< zenrows-description}}. Zenrows is a paid service: every request uses credits on the user's Zenrows plan. +{{< zenrows-description}}. {{host_full}} reaches it through the Zenrows MCP server. Zenrows is a paid service: every request uses credits on the user's Zenrows plan. The {{connection}} signs in with OAuth. If the Zenrows tools are missing or return `AUTH003`, use the `{{skill:getting-started}}` skill. Never ask the user to paste an API key into the conversation. From f3840bdba63e68d7ca56b297b87d97509a97f69e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Fri, 13 Nov 2026 17:17:26 +0000 Subject: [PATCH 3/5] docs(readme): link the MCP server documentation from every plugin's README --- dist/copilot/README.md | 1 + shared/README.md | 1 + 2 files changed, 2 insertions(+) diff --git a/dist/copilot/README.md b/dist/copilot/README.md index 5e4428f..4cc68ba 100644 --- a/dist/copilot/README.md +++ b/dist/copilot/README.md @@ -48,6 +48,7 @@ The plugin sends the URLs and options you ask Copilot to fetch to Zenrows, which ## Support - Documentation: [docs.zenrows.com](https://docs.zenrows.com) +- MCP server documentation: [docs.zenrows.com/mcp/overview](https://docs.zenrows.com/mcp/overview) - Contact: [integrations@zenrows.com](mailto:integrations@zenrows.com) ## License diff --git a/shared/README.md b/shared/README.md index cc439db..0f2880d 100644 --- a/shared/README.md +++ b/shared/README.md @@ -36,6 +36,7 @@ The plugin sends the URLs and options you ask {{host}} to fetch to Zenrows, whic ## Support - Documentation: [docs.zenrows.com](https://docs.zenrows.com) +- MCP server documentation: [docs.zenrows.com/mcp/overview](https://docs.zenrows.com/mcp/overview) - Contact: [integrations@zenrows.com](mailto:integrations@zenrows.com) ## License From 58796a5cc74722e2b572b9cca0f103a8d1c633a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Fri, 13 Nov 2026 17:57:12 +0000 Subject: [PATCH 4/5] feat(skills): state what AUTH004, AUTH006, AUTH014 and REQS007 mean once, in shared/text/ The using-zenrows and extract-structured-data skills take the wording from four snippets, so code hosts can quote the same facts. Every plugin renders byte-identically. --- shared/text/auth004.md | 1 + shared/text/auth006.md | 1 + shared/text/auth014.md | 1 + shared/text/reqs007-page-not-recognised.md | 1 + skills/extract-structured-data/SKILL.md | 2 +- skills/using-zenrows/SKILL.md | 4 ++-- 6 files changed, 7 insertions(+), 3 deletions(-) create mode 100644 shared/text/auth004.md create mode 100644 shared/text/auth006.md create mode 100644 shared/text/auth014.md create mode 100644 shared/text/reqs007-page-not-recognised.md diff --git a/shared/text/auth004.md b/shared/text/auth004.md new file mode 100644 index 0000000..09426a7 --- /dev/null +++ b/shared/text/auth004.md @@ -0,0 +1 @@ +the account's credit allowance is spent diff --git a/shared/text/auth006.md b/shared/text/auth006.md new file mode 100644 index 0000000..b5f06c3 --- /dev/null +++ b/shared/text/auth006.md @@ -0,0 +1 @@ +too many requests are running at once diff --git a/shared/text/auth014.md b/shared/text/auth014.md new file mode 100644 index 0000000..c8511e2 --- /dev/null +++ b/shared/text/auth014.md @@ -0,0 +1 @@ +this API key reached its credit cap diff --git a/shared/text/reqs007-page-not-recognised.md b/shared/text/reqs007-page-not-recognised.md new file mode 100644 index 0000000..dbc749c --- /dev/null +++ b/shared/text/reqs007-page-not-recognised.md @@ -0,0 +1 @@ +the domain is prepared, but this page has nothing Extract extracts (a home or search page, for example) diff --git a/skills/extract-structured-data/SKILL.md b/skills/extract-structured-data/SKILL.md index 3f25523..cdb1c4c 100644 --- a/skills/extract-structured-data/SKILL.md +++ b/skills/extract-structured-data/SKILL.md @@ -24,7 +24,7 @@ Use the dedicated `extract` tool, not `scrape`. It returns structured JSON direc - **`mode` selects the extraction strategy** (`auto`, `autoparse`, `css`). It is not the adaptive request setting, which is the separate boolean `mode_auto` and is on by default. `scrape` has no `mode` parameter at all. - **`extract` with `mode='auto'` works on any domain Zenrows has prepared, with any API key.** There is no per-account gate. When it can't extract a page it returns `REQS007` (or the older `AUTH010`) with a `reason`, and nothing is charged: - `domain_not_prepared`: call `prepare_domain` with the domain and 1 to 5 example URLs, one per kind of page the user wants (for example one product page and one listing page). It costs no credits and takes a few minutes. Poll `prepared_domain_status` about every 30 seconds until `status` is `ready`, then call `extract` again. Each new domain uses one slot of the account's monthly allowance, which `prepared_domain_status` shows. If the user can't wait, use `mode='css'` meanwhile. - - `page_not_recognised`: the domain is prepared, but this page has nothing Extract extracts (a home or search page, for example). Try a product, article or listing page on the same site. If this page should work, retry it once after a few minutes, not in a loop; otherwise use `mode='css'`. Don't call `prepare_domain`: it doesn't change this answer. + - `page_not_recognised`: {{< reqs007-page-not-recognised}}. Try a product, article or listing page on the same site. If this page should work, retry it once after a few minutes, not in a loop; otherwise use `mode='css'`. Don't call `prepare_domain`: it doesn't change this answer. - `domain_not_enabled`: preparing doesn't help. Use `mode='css'`, or tell the user to contact Zenrows support to have the site enabled. - **`fallback_autoparse` is deprecated and has no effect.** Extract no longer falls back to autoparse. Don't set it; if you do, the result only carries a note saying it was ignored. diff --git a/skills/using-zenrows/SKILL.md b/skills/using-zenrows/SKILL.md index 4bd1529..62cfdf7 100644 --- a/skills/using-zenrows/SKILL.md +++ b/skills/using-zenrows/SKILL.md @@ -78,8 +78,8 @@ Reads the connected account's plan and usage. It changes nothing. - Do not build manual retry loops around a request. Zenrows already escalates to heavier settings when a page needs it. - `401` or `AUTH003`: Zenrows is not connected, or the sign-in is no longer valid (for example the API key was rotated). Use the `{{skill:getting-started}}` skill to reconnect. Do not ask for a key. -- `AUTH004` (the account's credit allowance is spent) or `AUTH014` (this API key reached its credit cap): a plan limit, not a sign-in problem. Reconnecting doesn't clear it. Tell the user; they can check their usage and plan in the Zenrows dashboard, or raise the key's cap. -- `AUTH006`: too many requests are running at once. Wait for running requests to finish, then retry. +- `AUTH004` ({{< auth004}}) or `AUTH014` ({{< auth014}}): a plan limit, not a sign-in problem. Reconnecting doesn't clear it. Tell the user; they can check their usage and plan in the Zenrows dashboard, or raise the key's cap. +- `AUTH006`: {{< auth006}}. Wait for running requests to finish, then retry. - `413`: the response is too large. Narrow it with `css_extractor`. - `REQS007` (or the older `AUTH010`) from `extract`: read its `reason`. `domain_not_prepared`: prepare the domain with `prepare_domain` and wait for `prepared_domain_status` to say `ready`. `page_not_recognised`: try a product, article or listing page, or `mode='css'`; don't prepare the domain again. `domain_not_enabled`: use `mode='css'`. Not charged in any case. Details in the `{{skill:extract-structured-data}}` skill. - Look up any other error code at https://docs.zenrows.com. From 6151fbfed27446e2fa195f9474f14d22b6654513 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?In=C3=AAs=20Moital?= Date: Fri, 13 Nov 2026 18:06:47 +0000 Subject: [PATCH 5/5] build: let a host keep its own tooling in scripts/, and add npm run package platforms//scripts/ is never shipped; validate runs a host's scripts/validate.mjs and npm run package -- its scripts/package.sh. Moved here from the OpenAI PR, since Activepieces uses it too. --- CONTRIBUTING.md | 4 +++- package.json | 1 + scripts/lib.mjs | 5 +++-- scripts/package.mjs | 18 ++++++++++++++++++ scripts/test.mjs | 8 ++++++++ scripts/validate.mjs | 9 +++++++++ 6 files changed, 42 insertions(+), 3 deletions(-) create mode 100644 scripts/package.mjs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 0e0bb28..ef4aa0c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -14,10 +14,12 @@ platforms// skills//.md the host's text for a {{> slot}} in that skill readme/setup.md the host's Setup section of its README (readme/more.md: optional extra sections) overrides.lock whole-file overrides and the shared version each was copied from (only once one exists) + scripts/ the host's own tooling, never shipped; validate.mjs and package.sh are run by + npm run validate and npm run package -- , the rest by the host itself everything else the host's manifests, …, copied into dist// ({{version}} comes from release-please) dist// GENERATED: the plugin the host installs, or a code host's files .github/plugin/marketplace.json Copilot's marketplace catalog (see below) -scripts/ render, validate, override, new-host, mirror, test +scripts/ render, validate, override, new-host, package, mirror, test ``` ### Marketplace catalogs diff --git a/package.json b/package.json index 11d8c95..72266c2 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "override": "node scripts/override.mjs create", "override:ack": "node scripts/override.mjs ack", "new-host": "node scripts/new-host.mjs", + "package": "node scripts/package.mjs", "mirror": "node scripts/mirror.mjs" }, "devDependencies": { diff --git a/scripts/lib.mjs b/scripts/lib.mjs index 286a12a..d941997 100644 --- a/scripts/lib.mjs +++ b/scripts/lib.mjs @@ -15,9 +15,10 @@ export const sharedTextDir = join(root, "shared", "text"); // Files copied into every host's plugin as they are. const sharedFiles = { LICENSE: "LICENSE" }; const sharedDirs = { "shared/assets": "assets" }; -// In platforms//, these are inputs to the render, not files of the plugin. +// In platforms//, these are inputs to the render or the host's own tooling +// (scripts/validate.mjs, scripts/package.sh), not files of the plugin. // Everything else there (manifests, README, …) is copied into dist//. -const reserved = ["vars.yaml", "overrides.lock", "skills/", "readme/"]; +const reserved = ["vars.yaml", "overrides.lock", "skills/", "scripts/", "readme/"]; // A plugin host's README.md is rendered from this template, with the host's // slots from platforms//readme/. const readmeTemplate = join(root, "shared", "README.md"); diff --git a/scripts/package.mjs b/scripts/package.mjs new file mode 100644 index 0000000..274068f --- /dev/null +++ b/scripts/package.mjs @@ -0,0 +1,18 @@ +// npm run package -- : builds what a host uploads by hand (for example +// OpenAI's ZIP) by running platforms//scripts/package.sh. +import { spawnSync } from "node:child_process"; +import { existsSync } from "node:fs"; +import { join } from "node:path"; +import { hosts, platformsDir } from "./lib.mjs"; + +const host = process.argv[2]; +if (!host || !hosts().includes(host)) { + console.error(`Usage: npm run package -- , one of: ${hosts().join(", ")}`); + process.exit(1); +} +const script = join(platformsDir, host, "scripts", "package.sh"); +if (!existsSync(script)) { + console.log(`${host} needs no packaging: it installs from dist/${host}/ (no platforms/${host}/scripts/package.sh).`); + process.exit(0); +} +process.exit(spawnSync("bash", [script], { stdio: "inherit" }).status ?? 1); diff --git a/scripts/test.mjs b/scripts/test.mjs index 61e8dd3..3ec2040 100644 --- a/scripts/test.mjs +++ b/scripts/test.mjs @@ -144,6 +144,14 @@ test("the optional more slot is left out without its file and put in with it", ( assert.match(r.dist("dist/copilot/README.md"), /## Setup\n[\s\S]*## For maintainers\n\nRelease notes live elsewhere\.\n\n## Privacy/); }); +// A host's scripts/ is its own tooling: never shipped, and its validate.mjs runs. +test("a host's scripts/ isn't shipped", () => { + const r = run({ "platforms/copilot/scripts/tool.sh": "echo hi\n" }, ["dist/copilot/scripts/tool.sh"]); + assert.ok(r.ok, r.out); + assert.equal(r.dist("dist/copilot/scripts/tool.sh"), undefined); +}); +test("a host's own scripts/validate.mjs failing fails validate", fails({ "platforms/copilot/scripts/validate.mjs": "console.error('too long'); process.exit(1);\n" }, "copilot: platforms/copilot/scripts/validate.mjs failed:\ntoo long")); + // Snippets: plain text, one line, used somewhere. const snippet = (text) => ({ ...piece, "shared/text/cost.md": text, "platforms/piece/src/index.ts": "export const cost = {{text:cost}};\n" }); test("a snippet with parameter names and brackets passes", passes(snippet("A request with js_render returns [{...}] and costs 5 credits.\n"))); diff --git a/scripts/validate.mjs b/scripts/validate.mjs index 1bc2888..c571499 100644 --- a/scripts/validate.mjs +++ b/scripts/validate.mjs @@ -1,6 +1,7 @@ // Checks what each host ships: skill frontmatter, skill references, manifests // (against the JSON Schema they declare), the marketplace catalogs at the top of // the repo, and that no whole-file override was copied from an older shared file. +import { spawnSync } from "node:child_process"; import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { basename, dirname, join, relative } from "node:path"; import Ajv2020 from "ajv/dist/2020.js"; @@ -236,6 +237,14 @@ for (const host of hosts()) { if (text.includes("TODO(new-host)")) errors.push(`${host}: platforms/${host}/${file} still has a TODO(new-host)`); } + // Checks only one host needs, such as OpenAI's submission limits, live in + // platforms//scripts/validate.mjs and run on the committed dist/. + const extra = join(platformsDir, host, "scripts", "validate.mjs"); + if (existsSync(extra)) { + const run = spawnSync(process.execPath, [extra, join(root, "dist", host)], { encoding: "utf8" }); + if (run.status !== 0) errors.push(`${host}: platforms/${host}/scripts/validate.mjs failed:\n${(run.stderr || run.stdout).trim()}`); + } + // A code host's README stays here (it isn't shipped) and the repo's hosts // list links to it. if (isCodeHost(values)) {