diff --git a/RP_CATALOG.md b/RP_CATALOG.md index e45f4cc568..90c446602a 100644 --- a/RP_CATALOG.md +++ b/RP_CATALOG.md @@ -12,12 +12,18 @@ identified upstream revision and carries declared RP additions. - JSON Schema: - Workflow: -Schema version 1 contains the existing Zed `ExtensionMetadata` fields in -`data`, all known official version metadata in `versions`, and one integrity -record per current package in `packages`. Each package binds its ID, version, +Schema version 1 contains every mirrored registry record in `source_entries`, +the existing Zed `ExtensionMetadata` fields for installable releases in `data`, +all known official version metadata in `versions`, and one integrity record per +installable current package in `packages`. Each package binds its ID, version, schema/Wasm compatibility, immutable source repository and Git revision, archive byte size, archive SHA-256, and HTTPS download URL. +An upstream source entry that has no matching package in the official service +remains visible in `source_entries` and is identified in +`unavailable_source_entries`; it is never presented as installable and no +unverifiable archive is synthesized. + Official entries use Zed's exact-version download route and permit only its API and extension object-store hosts. RP additions are packaged with the same pinned `zed-extension` CLI as upstream and are hosted under this Pages site. diff --git a/rp-catalog.schema.json b/rp-catalog.schema.json index 43dfaef7c6..adcda30502 100644 --- a/rp-catalog.schema.json +++ b/rp-catalog.schema.json @@ -12,7 +12,10 @@ "integrity", "entry_count", "upstream_entry_count", + "published_upstream_entry_count", "additions", + "source_entries", + "unavailable_source_entries", "data", "versions", "packages" @@ -41,6 +44,7 @@ }, "entry_count": { "type": "integer", "minimum": 1 }, "upstream_entry_count": { "type": "integer", "minimum": 1 }, + "published_upstream_entry_count": { "type": "integer", "minimum": 0 }, "additions": { "type": "array", "items": { @@ -48,6 +52,8 @@ "required": ["id", "version", "source_repository", "source_revision"] } }, + "source_entries": { "type": "array" }, + "unavailable_source_entries": { "type": "array" }, "data": { "type": "array" }, "versions": { "type": "object" }, "packages": { diff --git a/src/generate-rp-catalog.js b/src/generate-rp-catalog.js index b70ba1f40d..27c53b52eb 100644 --- a/src/generate-rp-catalog.js +++ b/src/generate-rp-catalog.js @@ -117,9 +117,25 @@ const results = await mapConcurrent(extensionIds, concurrency, async (id) => { version.version === registryEntry.version, ) ?? null; if (!metadata) { - throw new Error( - `Official registry has no ${id}@${registryEntry.version} metadata.`, - ); + if (addition) { + throw new Error( + `RP addition has no ${id}@${registryEntry.version} metadata.`, + ); + } + return { + metadata: null, + versions, + package: null, + unavailable: { + id, + version: registryEntry.version, + reason: "not-published-by-upstream", + published_versions: versions.map( + /** @param {ExtensionMetadata} version */ (version) => + version.version, + ), + }, + }; } const archiveUrl = addition @@ -150,6 +166,7 @@ const results = await mapConcurrent(extensionIds, concurrency, async (id) => { return { metadata, versions, + unavailable: null, package: { id, version: registryEntry.version, @@ -164,8 +181,19 @@ const results = await mapConcurrent(extensionIds, concurrency, async (id) => { }; }); -const packages = results.map((result) => result.package); -validatePackageRecords(packages, extensionIds); +const data = results.flatMap((result) => + result.metadata ? [result.metadata] : [], +); +const packages = results.flatMap((result) => + result.package ? [result.package] : [], +); +const unavailableSourceEntries = results.flatMap((result) => + result.unavailable ? [result.unavailable] : [], +); +validatePackageRecords( + packages, + data.map((metadata) => metadata.id), +); await fs.rm(outputRoot, { recursive: true, force: true }); await fs.mkdir(outputRoot, { recursive: true }); @@ -233,13 +261,30 @@ const catalog = { }, entry_count: extensionIds.length, upstream_entry_count: Object.keys(upstream).length, + published_upstream_entry_count: + Object.keys(upstream).length - unavailableSourceEntries.length, additions: Object.entries(config.additions).map(([id, addition]) => ({ id, version: addition.version, source_repository: addition.source_repository, source_revision: addition.source_revision, })), - data: results.map((result) => result.metadata), + source_entries: extensionIds.map((id) => { + const registryEntry = current[id]; + const source = registryEntry ? sources[registryEntry.submodule] : undefined; + if (!registryEntry || !source) { + throw new Error(`Missing source entry for ${id}.`); + } + return { + id, + version: registryEntry.version, + source_repository: source.url, + source_revision: source.revision, + available: !unavailableSourceEntries.some((entry) => entry.id === id), + }; + }), + unavailable_source_entries: unavailableSourceEntries, + data, versions, packages, }; @@ -260,7 +305,7 @@ await Promise.all([ ]); console.log( - `Generated ${extensionIds.length} entries (${Object.keys(upstream).length} upstream + ${Object.keys(config.additions).length} RP) with catalog SHA-256 ${catalogDigest}.`, + `Generated ${extensionIds.length} source entries and ${packages.length} installable packages (${unavailableSourceEntries.length} upstream unavailable) with catalog SHA-256 ${catalogDigest}.`, ); async function readAdditionManifests() {