Handle unpublished upstream registry entries

RP_CATALOG.md

@@ -12,12 +12,18 @@ identified upstream revision and carries declared RP additions.
 - JSON Schema: <https://jonathonrp.github.io/extensions/rp-catalog/v1/schema.json>
 - Workflow: <https://github.com/JonathonRP/extensions/actions/workflows/rp-catalog.yml>
 
-Schema version 1 contains the existing Zed `ExtensionMetadata` fields in
-`data`, all known official version metadata in `versions`, and one integrity
-record per current package in `packages`. Each package binds its ID, version,
+Schema version 1 contains every mirrored registry record in `source_entries`,
+the existing Zed `ExtensionMetadata` fields for installable releases in `data`,
+all known official version metadata in `versions`, and one integrity record per
+installable current package in `packages`. Each package binds its ID, version,
 schema/Wasm compatibility, immutable source repository and Git revision,
 archive byte size, archive SHA-256, and HTTPS download URL.
 
+An upstream source entry that has no matching package in the official service
+remains visible in `source_entries` and is identified in
+`unavailable_source_entries`; it is never presented as installable and no
+unverifiable archive is synthesized.
+
 Official entries use Zed's exact-version download route and permit only its API
 and extension object-store hosts. RP additions are packaged with the same
 pinned `zed-extension` CLI as upstream and are hosted under this Pages site.

rp-catalog.schema.json

@@ -12,7 +12,10 @@
     "integrity",
     "entry_count",
     "upstream_entry_count",
+    "published_upstream_entry_count",
     "additions",
+    "source_entries",
+    "unavailable_source_entries",
     "data",
     "versions",
     "packages"
@@ -41,13 +44,16 @@
     },
     "entry_count": { "type": "integer", "minimum": 1 },
     "upstream_entry_count": { "type": "integer", "minimum": 1 },
+    "published_upstream_entry_count": { "type": "integer", "minimum": 0 },
     "additions": {
       "type": "array",
       "items": {
         "type": "object",
         "required": ["id", "version", "source_repository", "source_revision"]
       }
     },
+    "source_entries": { "type": "array" },
+    "unavailable_source_entries": { "type": "array" },
     "data": { "type": "array" },
     "versions": { "type": "object" },
     "packages": {

src/generate-rp-catalog.js

@@ -117,9 +117,25 @@ const results = await mapConcurrent(extensionIds, concurrency, async (id) => {
         version.version === registryEntry.version,
     ) ?? null;
   if (!metadata) {
-    throw new Error(
-      `Official registry has no ${id}@${registryEntry.version} metadata.`,
-    );
+    if (addition) {
+      throw new Error(
+        `RP addition has no ${id}@${registryEntry.version} metadata.`,
+      );
+    }
+    return {
+      metadata: null,
+      versions,
+      package: null,
+      unavailable: {
+        id,
+        version: registryEntry.version,
+        reason: "not-published-by-upstream",
+        published_versions: versions.map(
+          /** @param {ExtensionMetadata} version */ (version) =>
+            version.version,
+        ),
+      },
+    };
   }
 
   const archiveUrl = addition
@@ -150,6 +166,7 @@ const results = await mapConcurrent(extensionIds, concurrency, async (id) => {
   return {
     metadata,
     versions,
+    unavailable: null,
     package: {
       id,
       version: registryEntry.version,
@@ -164,8 +181,19 @@ const results = await mapConcurrent(extensionIds, concurrency, async (id) => {
   };
 });
 
-const packages = results.map((result) => result.package);
-validatePackageRecords(packages, extensionIds);
+const data = results.flatMap((result) =>
+  result.metadata ? [result.metadata] : [],
+);
+const packages = results.flatMap((result) =>
+  result.package ? [result.package] : [],
+);
+const unavailableSourceEntries = results.flatMap((result) =>
+  result.unavailable ? [result.unavailable] : [],
+);
+validatePackageRecords(
+  packages,
+  data.map((metadata) => metadata.id),
+);
 
 await fs.rm(outputRoot, { recursive: true, force: true });
 await fs.mkdir(outputRoot, { recursive: true });
@@ -233,13 +261,30 @@ const catalog = {
   },
   entry_count: extensionIds.length,
   upstream_entry_count: Object.keys(upstream).length,
+  published_upstream_entry_count:
+    Object.keys(upstream).length - unavailableSourceEntries.length,
   additions: Object.entries(config.additions).map(([id, addition]) => ({
     id,
     version: addition.version,
     source_repository: addition.source_repository,
     source_revision: addition.source_revision,
   })),
-  data: results.map((result) => result.metadata),
+  source_entries: extensionIds.map((id) => {
+    const registryEntry = current[id];
+    const source = registryEntry ? sources[registryEntry.submodule] : undefined;
+    if (!registryEntry || !source) {
+      throw new Error(`Missing source entry for ${id}.`);
+    }
+    return {
+      id,
+      version: registryEntry.version,
+      source_repository: source.url,
+      source_revision: source.revision,
+      available: !unavailableSourceEntries.some((entry) => entry.id === id),
+    };
+  }),
+  unavailable_source_entries: unavailableSourceEntries,
+  data,
   versions,
   packages,
 };
@@ -260,7 +305,7 @@ await Promise.all([
 ]);
 
 console.log(
-  `Generated ${extensionIds.length} entries (${Object.keys(upstream).length} upstream + ${Object.keys(config.additions).length} RP) with catalog SHA-256 ${catalogDigest}.`,
+  `Generated ${extensionIds.length} source entries and ${packages.length} installable packages (${unavailableSourceEntries.length} upstream unavailable) with catalog SHA-256 ${catalogDigest}.`,
 );
 
 async function readAdditionManifests() {