From bdc162b4485bac8ff30ea8ae634a1888aafdc26e Mon Sep 17 00:00:00 2001 From: JonathonRP Date: Wed, 2 Sep 2026 18:11:10 -0500 Subject: [PATCH 1/2] Add Pigments extension Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .gitmodules | 4 ++++ extensions.toml | 5 +++++ extensions/pigments-lsp | 1 + 3 files changed, 10 insertions(+) create mode 160000 extensions/pigments-lsp diff --git a/.gitmodules b/.gitmodules index 1dd9228c7f..3febbbe356 100644 --- a/.gitmodules +++ b/.gitmodules @@ -3938,6 +3938,10 @@ path = extensions/pierre-theme url = https://github.com/pierrecomputer/theme.git +[submodule "extensions/pigments-lsp"] + path = extensions/pigments-lsp + url = https://github.com/JonathonRP/zed-pigments.git + [submodule "extensions/pigs-in-space"] path = extensions/pigs-in-space url = https://github.com/kreek/pigs-in-space-zed.git diff --git a/extensions.toml b/extensions.toml index 06c3ba976d..6fa708246b 100644 --- a/extensions.toml +++ b/extensions.toml @@ -4010,6 +4010,11 @@ submodule = "extensions/pierre-theme" path = "zed" version = "0.0.29" +[pigments-lsp] +submodule = "extensions/pigments-lsp" +path = "zed-pigments" +version = "0.3.1" + [pigs-in-space] submodule = "extensions/pigs-in-space" version = "0.1.0" diff --git a/extensions/pigments-lsp b/extensions/pigments-lsp new file mode 160000 index 0000000000..545ee63ba6 --- /dev/null +++ b/extensions/pigments-lsp @@ -0,0 +1 @@ +Subproject commit 545ee63ba654a57e322e109b09ff249c908c1ec6 From 0da7c7f0e6a44a939987c01f4685a1d132056ccd Mon Sep 17 00:00:00 2001 From: JonathonRP Date: Wed, 2 Sep 2026 18:49:22 -0500 Subject: [PATCH 2/2] Publish RP extension catalog Add a fork-guarded, integrity-checked Pages catalog that mirrors the complete upstream registry and packages the pinned Pigments extension. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .github/workflows/rp-catalog.yml | 138 +++++++++++ README.md | 4 + RP_CATALOG.md | 53 +++++ package.json | 2 + rp-catalog.config.json | 20 ++ rp-catalog.schema.json | 72 ++++++ src/generate-rp-catalog.js | 377 +++++++++++++++++++++++++++++++ src/lib/git.js | 38 ++++ src/lib/rp-catalog.js | 172 ++++++++++++++ src/lib/rp-catalog.test.js | 155 +++++++++++++ src/validate-rp-registry.js | 42 ++++ 11 files changed, 1073 insertions(+) create mode 100644 .github/workflows/rp-catalog.yml create mode 100644 RP_CATALOG.md create mode 100644 rp-catalog.config.json create mode 100644 rp-catalog.schema.json create mode 100644 src/generate-rp-catalog.js create mode 100644 src/lib/rp-catalog.js create mode 100644 src/lib/rp-catalog.test.js create mode 100644 src/validate-rp-registry.js diff --git a/.github/workflows/rp-catalog.yml b/.github/workflows/rp-catalog.yml new file mode 100644 index 0000000000..4abd709ac1 --- /dev/null +++ b/.github/workflows/rp-catalog.yml @@ -0,0 +1,138 @@ +name: RP extension catalog + +on: + pull_request: + branches: + - "release/rp-stable" + push: + branches: + - "release/rp-stable" + schedule: + - cron: "17 5 * * *" + workflow_dispatch: + +concurrency: + group: rp-extension-catalog + cancel-in-progress: false + +env: + ZED_EXTENSION_CLI_SHA: 9ee3c503a4bbbc6b4a0f8a789acca4871d773223 + +jobs: + validate: + if: github.repository == 'JonathonRP/extensions' + runs-on: ubuntu-24.04 + timeout-minutes: 20 + steps: + - name: Checkout catalog + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: 0 + + - name: Fetch upstream + run: | + git remote add upstream https://github.com/zed-industries/extensions.git + git fetch --no-tags upstream main + echo "UPSTREAM_REVISION=$(git merge-base HEAD upstream/main)" >> "$GITHUB_ENV" + + - uses: pnpm/setup@5d160c5bc68a09337ad0d5654e237e03253b5879 # v1 + with: + version: 11 + runtime: node@24.20.0 + cache: true + + - name: Validate registry + run: | + pnpm install --frozen-lockfile + pnpm build + pnpm test + pnpm validate-rp-registry + + - uses: actions-rust-lang/setup-rust-toolchain@166cdcfd11aee3cb47222f9ddb555ce30ddb9659 # v1 + with: + toolchain: "1.90" + target: "wasm32-wasip2" + + - name: Package Pigments with official tooling + run: | + git submodule update --init --depth 1 extensions/pigments-lsp + wget --quiet "https://zed-extension-cli.nyc3.digitaloceanspaces.com/$ZED_EXTENSION_CLI_SHA/x86_64-unknown-linux-gnu/zed-extension" + chmod +x zed-extension + pnpm package-extensions pigments-lsp + env: + REF_NAME: ${{ github.ref_name }} + RUSTUP_TOOLCHAIN: "1.90" + SHOULD_PUBLISH: "false" + + - name: Preserve package + if: github.event_name != 'pull_request' + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4 + with: + name: rp-pigments-package + path: | + output/archive.tar.gz + output/manifest.json + if-no-files-found: error + + publish: + if: github.repository == 'JonathonRP/extensions' && github.event_name != 'pull_request' && github.ref == 'refs/heads/release/rp-stable' + needs: validate + runs-on: ubuntu-24.04 + timeout-minutes: 120 + permissions: + contents: read + pages: write + id-token: write + attestations: write + environment: + name: github-pages + url: ${{ steps.deployment.outputs.page_url }} + steps: + - name: Checkout catalog + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + fetch-depth: 0 + + - name: Fetch upstream + run: | + git remote add upstream https://github.com/zed-industries/extensions.git + git fetch --no-tags upstream main + echo "UPSTREAM_REVISION=$(git merge-base HEAD upstream/main)" >> "$GITHUB_ENV" + echo "FORK_REVISION=$(git rev-parse HEAD)" >> "$GITHUB_ENV" + + - uses: pnpm/setup@5d160c5bc68a09337ad0d5654e237e03253b5879 # v1 + with: + version: 11 + runtime: node@24.20.0 + cache: true + + - name: Restore packaged Pigments + uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4 + with: + name: rp-pigments-package + path: output + + - name: Generate complete integrity catalog + run: | + pnpm install --frozen-lockfile + pnpm generate-rp-catalog + env: + CURRENT_CATALOG_URL: https://jonathonrp.github.io/extensions/rp-catalog/v1/catalog.json + RP_CATALOG_CONCURRENCY: "8" + + - name: Configure Pages + uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5 + + - name: Attest catalog provenance + uses: actions/attest-build-provenance@43d14bc2b83dec42d39ecae14e916627a18bb661 # v3 + with: + subject-path: public/rp-catalog/v1/catalog.json + + - name: Upload Pages artifact + uses: actions/upload-pages-artifact@7b1f4a764d45c48632c6b24a0339c27f5614fb0b # v4 + with: + path: public + + - name: Deploy Pages + id: deployment + uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4 diff --git a/README.md b/README.md index 2f6ee12781..fef2a9de2c 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,10 @@ This is the central repository containing the extensions available for [Zed](https://zed.dev/). +The `JonathonRP/extensions` fork publishes the fork-only +[RP Extensions catalog](RP_CATALOG.md) from its persistent +`release/rp-stable` branch. This does not change the upstream Zed registry. + ## Getting started See the [Developing Extensions](https://zed.dev/docs/extensions/developing-extensions) docs for how to develop your own extension. diff --git a/RP_CATALOG.md b/RP_CATALOG.md new file mode 100644 index 0000000000..e45f4cc568 --- /dev/null +++ b/RP_CATALOG.md @@ -0,0 +1,53 @@ +# RP Extensions catalog + +`JonathonRP/extensions` is a history-preserving fork of +`zed-industries/extensions`. Its `main` branch is reserved for clean upstream +fast-forwards. The default, persistent `release/rp-stable` branch mirrors an +identified upstream revision and carries declared RP additions. + +## Endpoint + +- Catalog: +- Catalog SHA-256: +- JSON Schema: +- Workflow: + +Schema version 1 contains the existing Zed `ExtensionMetadata` fields in +`data`, all known official version metadata in `versions`, and one integrity +record per current package in `packages`. Each package binds its ID, version, +schema/Wasm compatibility, immutable source repository and Git revision, +archive byte size, archive SHA-256, and HTTPS download URL. + +Official entries use Zed's exact-version download route and permit only its API +and extension object-store hosts. RP additions are packaged with the same +pinned `zed-extension` CLI as upstream and are hosted under this Pages site. +RP clients must verify the catalog digest, allowlisted hosts, archive size and +SHA-256, then validate the packaged manifest before installation. + +## Pigments + +```toml +[pigments-lsp] +submodule = "extensions/pigments-lsp" +path = "zed-pigments" +version = "0.3.1" +``` + +The source is `https://github.com/JonathonRP/zed-pigments.git` pinned to +`545ee63ba654a57e322e109b09ff249c908c1ec6`. + +## Publication and sync + +The fork-only workflow is guarded to `JonathonRP/extensions`. It validates pull +requests targeting `release/rp-stable`, and publishes after pushes to that +branch, manual dispatches, and a daily 05:17 UTC refresh. Publication fails if +an upstream entry is removed or modified, an undeclared RP entry appears, the +Pigments pin changes, package metadata is malformed, or an archive cannot be +hashed. + +To sync, fetch `zed-industries/extensions`, merge its `main` into +`release/rp-stable`, resolve only around declared RP files, and open a focused +fork PR. Never force-push the release branch or overwrite an existing +ID/version package. GitHub Pages and the official Zed archive service remain +availability dependencies; RP clients must fail closed rather than silently +fall back to a partial catalog. diff --git a/package.json b/package.json index 4a18b67cd2..53eef41b71 100644 --- a/package.json +++ b/package.json @@ -8,8 +8,10 @@ "build": "tsc -p .", "test": "vitest run", "test:watch": "vitest", + "generate-rp-catalog": "node src/generate-rp-catalog.js", "package-extensions": "node src/package-extensions.js", "sort-extensions": "node src/sort-extensions.js", + "validate-rp-registry": "node src/validate-rp-registry.js", "danger": "danger" }, "dependencies": { diff --git a/rp-catalog.config.json b/rp-catalog.config.json new file mode 100644 index 0000000000..ab02a444fb --- /dev/null +++ b/rp-catalog.config.json @@ -0,0 +1,20 @@ +{ + "schema_version": 1, + "channel": "rp-stable", + "label": "RP Extensions", + "base_url": "https://jonathonrp.github.io/extensions/rp-catalog/v1", + "upstream_repository": "https://github.com/zed-industries/extensions", + "fork_repository": "https://github.com/JonathonRP/extensions", + "upstream_api_url": "https://api.zed.dev", + "upstream_archive_host": "zed-extensions.nyc3.digitaloceanspaces.com", + "additions": { + "pigments-lsp": { + "submodule": "extensions/pigments-lsp", + "path": "zed-pigments", + "version": "0.3.1", + "source_repository": "https://github.com/JonathonRP/zed-pigments.git", + "source_revision": "545ee63ba654a57e322e109b09ff249c908c1ec6", + "published_at": "2026-09-02T23:00:44Z" + } + } +} diff --git a/rp-catalog.schema.json b/rp-catalog.schema.json new file mode 100644 index 0000000000..43dfaef7c6 --- /dev/null +++ b/rp-catalog.schema.json @@ -0,0 +1,72 @@ +{ + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$id": "https://jonathonrp.github.io/extensions/rp-catalog/v1/schema.json", + "title": "RP Extensions catalog", + "type": "object", + "required": [ + "schema_version", + "channel", + "label", + "generated_at", + "source", + "integrity", + "entry_count", + "upstream_entry_count", + "additions", + "data", + "versions", + "packages" + ], + "properties": { + "schema_version": { "const": 1 }, + "channel": { "const": "rp-stable" }, + "label": { "const": "RP Extensions" }, + "generated_at": { "type": "string", "format": "date-time" }, + "source": { + "type": "object", + "required": [ + "fork_repository", + "fork_revision", + "upstream_repository", + "upstream_revision" + ] + }, + "integrity": { + "type": "object", + "required": [ + "catalog_digest_algorithm", + "catalog_digest_url", + "allowed_archive_hosts" + ] + }, + "entry_count": { "type": "integer", "minimum": 1 }, + "upstream_entry_count": { "type": "integer", "minimum": 1 }, + "additions": { + "type": "array", + "items": { + "type": "object", + "required": ["id", "version", "source_repository", "source_revision"] + } + }, + "data": { "type": "array" }, + "versions": { "type": "object" }, + "packages": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "version", + "schema_version", + "wasm_api_version", + "source_repository", + "source_revision", + "archive_url", + "archive_size", + "archive_sha256" + ] + } + } + }, + "additionalProperties": false +} diff --git a/src/generate-rp-catalog.js b/src/generate-rp-catalog.js new file mode 100644 index 0000000000..b70ba1f40d --- /dev/null +++ b/src/generate-rp-catalog.js @@ -0,0 +1,377 @@ +import fs from "node:fs/promises"; +import path from "node:path"; +import toml from "@iarna/toml"; +import { readSubmoduleSources } from "./lib/git.js"; +import { exec } from "./lib/process.js"; +import { + sha256, + validatePackageRecords, + validateRegistryDelta, +} from "./lib/rp-catalog.js"; + +/** + * @typedef {{ + * id: string, + * name: string, + * version: string, + * description?: string, + * authors: string[], + * repository: string, + * schema_version?: number, + * wasm_api_version?: string | null, + * provides?: string[], + * published_at: string, + * download_count: number, + * }} ExtensionMetadata + * + * @typedef {{ + * schema_version: number, + * channel: string, + * label: string, + * base_url: string, + * upstream_repository: string, + * fork_repository: string, + * upstream_api_url: string, + * upstream_archive_host: string, + * additions: Record, + * }} CatalogConfig + * + * @typedef {{ + * packages: import("./lib/rp-catalog.js").PackageRecord[], + * }} PreviousCatalog + */ + +const upstreamRevision = requiredEnv("UPSTREAM_REVISION"); +const forkRevision = requiredEnv("FORK_REVISION"); +const outputRoot = process.env["RP_CATALOG_OUTPUT"] ?? "public/rp-catalog/v1"; +const pigmentsArchive = + process.env["PIGMENTS_ARCHIVE"] ?? "output/archive.tar.gz"; +const pigmentsManifest = + process.env["PIGMENTS_MANIFEST"] ?? "output/manifest.json"; +const currentCatalogUrl = process.env["CURRENT_CATALOG_URL"]; +const concurrency = Number.parseInt( + process.env["RP_CATALOG_CONCURRENCY"] ?? "8", + 10, +); + +/** @type {CatalogConfig} */ +const config = JSON.parse(await fs.readFile("rp-catalog.config.json", "utf8")); +/** @type {Record} */ +const current = + /** @type {Record} */ ( + /** @type {unknown} */ ( + toml.parse(await fs.readFile("extensions.toml", "utf8")) + ) + ); +const { stdout: upstreamToml } = await exec("git", [ + "show", + `${upstreamRevision}:extensions.toml`, +]); +/** @type {Record} */ +const upstream = + /** @type {Record} */ ( + /** @type {unknown} */ (toml.parse(upstreamToml)) + ); +const sources = await readSubmoduleSources(); +const upstreamSources = await readSubmoduleSources(upstreamRevision); +validateRegistryDelta( + current, + upstream, + config.additions, + sources, + upstreamSources, +); + +const previous = await fetchPreviousCatalog(currentCatalogUrl); +const previousPackages = new Map( + (previous?.packages ?? []).map((record) => [ + `${record.id}@${record.version}`, + record, + ]), +); + +const additionManifests = await readAdditionManifests(); +const extensionIds = Object.keys(current).sort(); +const results = await mapConcurrent(extensionIds, concurrency, async (id) => { + const registryEntry = current[id]; + if (!registryEntry) { + throw new Error(`Missing registry entry for ${id}.`); + } + const source = sources[registryEntry.submodule]; + if (!source) { + throw new Error(`Missing source for ${id}.`); + } + + const addition = config.additions[id]; + const versions = addition + ? [additionManifests[id]] + : await fetchJson( + `${config.upstream_api_url}/extensions/${encodeURIComponent(id)}`, + ).then( + /** @param {{data: ExtensionMetadata[]}} response */ (response) => + response.data, + ); + const metadata = + versions.find( + /** @param {any} version */ (version) => + version.version === registryEntry.version, + ) ?? null; + if (!metadata) { + throw new Error( + `Official registry has no ${id}@${registryEntry.version} metadata.`, + ); + } + + const archiveUrl = addition + ? `${config.base_url}/extensions/${id}/${registryEntry.version}/archive.tar.gz` + : `${config.upstream_api_url}/extensions/${encodeURIComponent(id)}/${encodeURIComponent(registryEntry.version)}/download`; + const previousPackage = previousPackages.get( + `${id}@${registryEntry.version}`, + ); + + let archive; + if ( + previousPackage?.source_revision === source.revision && + previousPackage?.archive_url === archiveUrl && + typeof previousPackage?.archive_sha256 === "string" && + Number.isSafeInteger(previousPackage?.archive_size) + ) { + archive = { + size: previousPackage.archive_size, + digest: previousPackage.archive_sha256, + }; + } else if (addition) { + const bytes = await fs.readFile(pigmentsArchive); + archive = { size: bytes.byteLength, digest: sha256(bytes) }; + } else { + archive = await fetchArchiveIntegrity(archiveUrl); + } + + return { + metadata, + versions, + package: { + id, + version: registryEntry.version, + schema_version: metadata.schema_version ?? 0, + wasm_api_version: metadata.wasm_api_version ?? null, + source_repository: source.url, + source_revision: source.revision, + archive_url: archiveUrl, + archive_size: archive.size, + archive_sha256: archive.digest, + }, + }; +}); + +const packages = results.map((result) => result.package); +validatePackageRecords(packages, extensionIds); + +await fs.rm(outputRoot, { recursive: true, force: true }); +await fs.mkdir(outputRoot, { recursive: true }); +for (const [id, addition] of Object.entries(config.additions)) { + const target = path.join( + outputRoot, + "extensions", + id, + addition.version, + "archive.tar.gz", + ); + await fs.mkdir(path.dirname(target), { recursive: true }); + + const previousPackage = previousPackages.get(`${id}@${addition.version}`); + if ( + previousPackage?.source_revision === addition.source_revision && + previousPackage?.archive_sha256 === + packages.find((record) => record.id === id)?.archive_sha256 + ) { + const response = await fetchWithRetry(previousPackage.archive_url); + const finalHost = new URL(response.url).hostname; + if (finalHost !== "jonathonrp.github.io") { + throw new Error(`Unexpected cached RP archive host: ${finalHost}.`); + } + const bytes = new Uint8Array(await response.arrayBuffer()); + if ( + bytes.byteLength !== previousPackage.archive_size || + sha256(bytes) !== previousPackage.archive_sha256 + ) { + throw new Error(`Cached RP archive integrity failed for ${id}.`); + } + await fs.writeFile(target, bytes); + } else { + await fs.copyFile(pigmentsArchive, target); + } +} + +/** @type {Record} */ +const versions = {}; +for (const [index, id] of extensionIds.entries()) { + const result = results[index]; + if (!result) throw new Error(`Missing generated result for ${id}.`); + versions[id] = result.versions; +} + +const catalog = { + schema_version: config.schema_version, + channel: config.channel, + label: config.label, + generated_at: new Date().toISOString(), + source: { + fork_repository: config.fork_repository, + fork_revision: forkRevision, + upstream_repository: config.upstream_repository, + upstream_revision: upstreamRevision, + }, + integrity: { + catalog_digest_algorithm: "sha256", + catalog_digest_url: `${config.base_url}/catalog.json.sha256`, + allowed_archive_hosts: [ + "api.zed.dev", + config.upstream_archive_host, + "jonathonrp.github.io", + ], + }, + entry_count: extensionIds.length, + upstream_entry_count: Object.keys(upstream).length, + additions: Object.entries(config.additions).map(([id, addition]) => ({ + id, + version: addition.version, + source_repository: addition.source_repository, + source_revision: addition.source_revision, + })), + data: results.map((result) => result.metadata), + versions, + packages, +}; + +const catalogJson = `${JSON.stringify(catalog, null, 2)}\n`; +const catalogDigest = sha256(catalogJson); +await Promise.all([ + fs.writeFile(path.join(outputRoot, "catalog.json"), catalogJson), + fs.writeFile( + path.join(outputRoot, "catalog.json.sha256"), + `${catalogDigest} catalog.json\n`, + ), + fs.copyFile("rp-catalog.schema.json", path.join(outputRoot, "schema.json")), + fs.writeFile( + path.join(outputRoot, "index.html"), + 'RP Extensions

RP Extensions

Catalog v1

\n', + ), +]); + +console.log( + `Generated ${extensionIds.length} entries (${Object.keys(upstream).length} upstream + ${Object.keys(config.additions).length} RP) with catalog SHA-256 ${catalogDigest}.`, +); + +async function readAdditionManifests() { + const manifest = JSON.parse(await fs.readFile(pigmentsManifest, "utf8")); + const firstAddition = Object.entries(config.additions)[0]; + if (!firstAddition) { + throw new Error("Catalog must declare an RP addition."); + } + const [id, addition] = firstAddition; + if (manifest.version !== addition.version) { + throw new Error("Packaged RP addition does not match catalog config."); + } + return { + [id]: { + id, + ...manifest, + schema_version: manifest.schema_version ?? 0, + wasm_api_version: manifest.wasm_api_version ?? null, + provides: manifest.provides ?? [], + published_at: addition.published_at, + download_count: 0, + }, + }; +} + +/** + * @param {string | undefined} url + * @returns {Promise} + */ +async function fetchPreviousCatalog(url) { + if (!url) return null; + try { + return await fetchJson(url); + } catch (error) { + console.warn(`No reusable catalog at ${url}: ${error}`); + return null; + } +} + +/** @param {string} url */ +async function fetchArchiveIntegrity(url) { + const response = await fetchWithRetry(url); + const finalHost = new URL(response.url).hostname; + if (finalHost !== config.upstream_archive_host) { + throw new Error(`Unexpected upstream archive host: ${finalHost}.`); + } + const bytes = new Uint8Array(await response.arrayBuffer()); + return { size: bytes.byteLength, digest: sha256(bytes) }; +} + +/** + * @template T + * @param {string} url + * @returns {Promise} + */ +async function fetchJson(url) { + const response = await fetchWithRetry(url); + return /** @type {Promise} */ (response.json()); +} + +/** @param {string} url */ +async function fetchWithRetry(url) { + let lastError; + for (let attempt = 0; attempt < 4; attempt += 1) { + try { + const response = await fetch(url, { + headers: { "user-agent": "JonathonRP/extensions RP catalog" }, + }); + if (!response.ok) { + throw new Error(`${response.status} ${response.statusText}`); + } + return response; + } catch (error) { + lastError = error; + await new Promise((resolve) => setTimeout(resolve, 500 * 2 ** attempt)); + } + } + throw new Error(`Failed to fetch ${url}: ${lastError}`); +} + +/** + * @template T, U + * @param {T[]} values + * @param {number} limit + * @param {(value: T) => Promise} operation + */ +async function mapConcurrent(values, limit, operation) { + if (!Number.isSafeInteger(limit) || limit < 1) { + throw new Error("RP_CATALOG_CONCURRENCY must be a positive integer."); + } + const results = /** @type {U[]} */ (new Array(values.length)); + let next = 0; + await Promise.all( + Array.from({ length: Math.min(limit, values.length) }, async () => { + while (next < values.length) { + const index = next; + next += 1; + const value = values[index]; + if (value === undefined) { + throw new Error(`Missing value at index ${index}.`); + } + results[index] = await operation(value); + } + }), + ); + return results; +} + +/** @param {string} name */ +function requiredEnv(name) { + const value = process.env[name]; + if (!value) throw new Error(`${name} is required.`); + return value; +} diff --git a/src/lib/git.js b/src/lib/git.js index aaac882a18..23a89e1d52 100644 --- a/src/lib/git.js +++ b/src/lib/git.js @@ -41,6 +41,44 @@ export async function readGitmodules(path) { return gitSubmodules.deserialize(gitmodulesContent); } +/** @param {string} [ref] */ +export async function readGitlinkRevisions(ref = "HEAD") { + const { stdout } = await exec("git", ["ls-tree", "-r", ref, "extensions"]); + /** @type {Record} */ + const revisions = {}; + for (const line of stdout.split("\n")) { + const match = line.match( + /^160000 commit ([a-f0-9]{40})\t(extensions\/.+)$/, + ); + if (match?.[1] && match[2]) { + revisions[match[2]] = match[1]; + } + } + return revisions; +} + +/** @param {string} [ref] */ +export async function readSubmoduleSources(ref = "HEAD") { + const { stdout } = await exec("git", ["show", `${ref}:.gitmodules`]); + const gitmodules = gitSubmodules.deserialize(stdout); + const revisions = await readGitlinkRevisions(ref); + /** @type {Record} */ + const sources = {}; + + for (const entry of Object.values(gitmodules)) { + const submodulePath = entry["path"]; + const submoduleUrl = entry["url"]; + if (!submodulePath || !submoduleUrl) continue; + const revision = revisions[submodulePath]; + if (!revision) { + throw new Error(`Missing Git revision for submodule ${submodulePath}.`); + } + sources[submodulePath] = { url: submoduleUrl, revision }; + } + + return sources; +} + /** @param {string} path */ export async function sortGitmodules(path) { const gitmodules = await readGitmodules(path); diff --git a/src/lib/rp-catalog.js b/src/lib/rp-catalog.js new file mode 100644 index 0000000000..740b4d3552 --- /dev/null +++ b/src/lib/rp-catalog.js @@ -0,0 +1,172 @@ +import { createHash } from "node:crypto"; + +/** + * @typedef {{ + * submodule: string, + * path?: string, + * version: string, + * }} RegistryEntry + * + * @typedef {{ + * submodule: string, + * path?: string, + * version: string, + * source_repository: string, + * source_revision: string, + * published_at: string, + * }} Addition + * + * @typedef {{ + * id: string, + * version: string, + * schema_version: number, + * wasm_api_version: string | null, + * source_repository: string, + * source_revision: string, + * archive_url: string, + * archive_size: number, + * archive_sha256: string, + * }} PackageRecord + */ + +/** @param {string | Uint8Array} value */ +export function sha256(value) { + return createHash("sha256").update(value).digest("hex"); +} + +/** + * Ensure the fork branch is an exact upstream snapshot plus declared additions. + * + * @param {Record} current + * @param {Record} upstream + * @param {Record} additions + * @param {Record} sources + * @param {Record} upstreamSources + */ +export function validateRegistryDelta( + current, + upstream, + additions, + sources, + upstreamSources, +) { + const upstreamIds = Object.keys(upstream).sort(); + const currentIds = Object.keys(current).sort(); + const additionIds = Object.keys(additions).sort(); + + for (const id of upstreamIds) { + if (!current[id]) { + throw new Error(`RP registry dropped upstream extension "${id}".`); + } + + if (JSON.stringify(current[id]) !== JSON.stringify(upstream[id])) { + throw new Error(`RP registry modified upstream extension "${id}".`); + } + + const submodule = upstream[id]?.submodule; + const source = submodule ? sources[submodule] : undefined; + const upstreamSource = submodule ? upstreamSources[submodule] : undefined; + if ( + !source || + !upstreamSource || + source.url !== upstreamSource.url || + source.revision !== upstreamSource.revision + ) { + throw new Error(`RP registry modified upstream source for "${id}".`); + } + } + + const unexpectedIds = currentIds.filter( + (id) => !upstream[id] && !additions[id], + ); + if (unexpectedIds.length > 0) { + throw new Error( + `RP registry contains undeclared additions: ${unexpectedIds.join(", ")}.`, + ); + } + + for (const id of additionIds) { + const expected = additions[id]; + const actual = current[id]; + if (!expected) { + throw new Error(`Missing configuration for RP addition "${id}".`); + } + if (!actual) { + throw new Error(`RP registry is missing declared addition "${id}".`); + } + + const fields = [ + ["submodule", expected.submodule, actual.submodule], + ["path", expected.path, actual.path], + ["version", expected.version, actual.version], + ]; + for (const [field, expectedValue, actualValue] of fields) { + if (expectedValue !== actualValue) { + throw new Error( + `RP addition "${id}" has ${field}=${JSON.stringify(actualValue)}; expected ${JSON.stringify(expectedValue)}.`, + ); + } + } + + const source = sources[actual.submodule]; + if (!source) { + throw new Error(`RP addition "${id}" has no Git submodule source.`); + } + if (source.url !== expected.source_repository) { + throw new Error( + `RP addition "${id}" source is ${source.url}; expected ${expected.source_repository}.`, + ); + } + if (source.revision !== expected.source_revision) { + throw new Error( + `RP addition "${id}" revision is ${source.revision}; expected ${expected.source_revision}.`, + ); + } + } + + if (currentIds.length !== upstreamIds.length + additionIds.length) { + throw new Error( + `RP registry has ${currentIds.length} entries; expected ${upstreamIds.length + additionIds.length}.`, + ); + } +} + +/** + * @param {PackageRecord[]} packages + * @param {string[]} extensionIds + */ +export function validatePackageRecords(packages, extensionIds) { + const expected = new Set(extensionIds); + const seen = new Set(); + + for (const record of packages) { + if (!expected.has(record.id)) { + throw new Error(`Package record has unknown extension "${record.id}".`); + } + if (seen.has(record.id)) { + throw new Error(`Package record is duplicated for "${record.id}".`); + } + seen.add(record.id); + + if (!/^[a-f0-9]{40}$/.test(record.source_revision)) { + throw new Error(`Package "${record.id}" has an invalid source revision.`); + } + if (!/^[a-f0-9]{64}$/.test(record.archive_sha256)) { + throw new Error(`Package "${record.id}" has an invalid SHA-256.`); + } + if ( + !Number.isSafeInteger(record.archive_size) || + record.archive_size <= 0 + ) { + throw new Error(`Package "${record.id}" has an invalid archive size.`); + } + if (new URL(record.archive_url).protocol !== "https:") { + throw new Error(`Package "${record.id}" does not use HTTPS.`); + } + } + + const missing = extensionIds.filter((id) => !seen.has(id)); + if (missing.length > 0) { + throw new Error(`Missing package records: ${missing.join(", ")}.`); + } +} diff --git a/src/lib/rp-catalog.test.js b/src/lib/rp-catalog.test.js new file mode 100644 index 0000000000..c8bd49853b --- /dev/null +++ b/src/lib/rp-catalog.test.js @@ -0,0 +1,155 @@ +import { describe, expect, it } from "vitest"; +import { + sha256, + validatePackageRecords, + validateRegistryDelta, +} from "./rp-catalog.js"; + +const upstream = { + rust: { submodule: "extensions/rust", version: "1.0.0" }, +}; +const addition = { + pigments: { + submodule: "extensions/pigments", + path: "zed-pigments", + version: "0.3.1", + source_repository: "https://github.com/example/pigments.git", + source_revision: "a".repeat(40), + published_at: "2026-09-02T23:00:44Z", + }, +}; +const sources = { + "extensions/rust": { + url: "https://github.com/example/rust.git", + revision: "c".repeat(40), + }, + "extensions/pigments": { + url: "https://github.com/example/pigments.git", + revision: "a".repeat(40), + }, +}; + +describe("validateRegistryDelta", () => { + it("accepts an exact upstream mirror with a declared pinned addition", () => { + expect(() => + validateRegistryDelta( + { + ...upstream, + pigments: { + submodule: "extensions/pigments", + path: "zed-pigments", + version: "0.3.1", + }, + }, + upstream, + addition, + sources, + { "extensions/rust": sources["extensions/rust"] }, + ), + ).not.toThrow(); + }); + + it("rejects removal or modification of an upstream entry", () => { + expect(() => + validateRegistryDelta({}, upstream, addition, sources, { + "extensions/rust": sources["extensions/rust"], + }), + ).toThrow('dropped upstream extension "rust"'); + + expect(() => + validateRegistryDelta( + { + rust: { submodule: "extensions/rust", version: "2.0.0" }, + pigments: { + submodule: "extensions/pigments", + path: "zed-pigments", + version: "0.3.1", + }, + }, + upstream, + addition, + sources, + { "extensions/rust": sources["extensions/rust"] }, + ), + ).toThrow('modified upstream extension "rust"'); + }); + + it("rejects a changed addition source revision", () => { + expect(() => + validateRegistryDelta( + { + ...upstream, + pigments: { + submodule: "extensions/pigments", + path: "zed-pigments", + version: "0.3.1", + }, + }, + upstream, + addition, + { + "extensions/rust": sources["extensions/rust"], + "extensions/pigments": { + ...sources["extensions/pigments"], + revision: "b".repeat(40), + }, + }, + { "extensions/rust": sources["extensions/rust"] }, + ), + ).toThrow("revision"); + }); + + it("rejects a changed upstream source revision", () => { + expect(() => + validateRegistryDelta( + { + ...upstream, + pigments: { + submodule: "extensions/pigments", + path: "zed-pigments", + version: "0.3.1", + }, + }, + upstream, + addition, + { + ...sources, + "extensions/rust": { + ...sources["extensions/rust"], + revision: "d".repeat(40), + }, + }, + { "extensions/rust": sources["extensions/rust"] }, + ), + ).toThrow('modified upstream source for "rust"'); + }); +}); + +describe("validatePackageRecords", () => { + it("requires one valid immutable package per extension", () => { + const record = { + id: "rust", + version: "1.0.0", + schema_version: 1, + wasm_api_version: null, + source_repository: "https://github.com/example/rust.git", + source_revision: "a".repeat(40), + archive_url: "https://example.com/rust/1.0.0/archive.tar.gz", + archive_size: 42, + archive_sha256: "b".repeat(64), + }; + + expect(() => validatePackageRecords([record], ["rust"])).not.toThrow(); + expect(() => validatePackageRecords([], ["rust"])).toThrow( + "Missing package records", + ); + }); +}); + +describe("sha256", () => { + it("hashes catalog and archive bytes", () => { + expect(sha256("rp")).toBe( + "796e80c7e5bf8a48cb603f229eeec578ec72443dbfe37710cc80de67228c6713", + ); + }); +}); diff --git a/src/validate-rp-registry.js b/src/validate-rp-registry.js new file mode 100644 index 0000000000..73866b5d32 --- /dev/null +++ b/src/validate-rp-registry.js @@ -0,0 +1,42 @@ +import toml from "@iarna/toml"; +import fs from "node:fs/promises"; +import { readSubmoduleSources } from "./lib/git.js"; +import { exec } from "./lib/process.js"; +import { validateRegistryDelta } from "./lib/rp-catalog.js"; + +const upstreamRevision = process.env["UPSTREAM_REVISION"]; +if (!upstreamRevision) { + throw new Error("UPSTREAM_REVISION is required."); +} + +/** @type {{additions: Record}} */ +const config = JSON.parse(await fs.readFile("rp-catalog.config.json", "utf8")); +/** @type {Record} */ +const current = + /** @type {Record} */ ( + /** @type {unknown} */ ( + toml.parse(await fs.readFile("extensions.toml", "utf8")) + ) + ); +const { stdout: upstreamToml } = await exec("git", [ + "show", + `${upstreamRevision}:extensions.toml`, +]); +/** @type {Record} */ +const upstream = + /** @type {Record} */ ( + /** @type {unknown} */ (toml.parse(upstreamToml)) + ); +const sources = await readSubmoduleSources(); +const upstreamSources = await readSubmoduleSources(upstreamRevision); + +validateRegistryDelta( + current, + upstream, + config.additions, + sources, + upstreamSources, +); +console.log( + `Validated ${Object.keys(upstream).length} upstream entries and ${Object.keys(config.additions).length} RP addition(s).`, +);