From dcc26326543ef7d9c171b0ed87f2107ae975844f Mon Sep 17 00:00:00 2001 From: ruv Date: Tue, 10 Nov 2026 16:12:37 -0500 Subject: [PATCH] ci(release): version-derived idempotent GitHub release, 10-minute npm visibility wait, ruv.io homepages Release-pipeline fixes found while shipping ruv-swarm 2.0.1 (#209). 1. GitHub release tag. create-github-release.py hardcoded tag v0.2.2 and failed on every run after the first. The tag now comes from the published ruv-swarm npm version (ruv-fann-registry-v). If a published release with that tag already exists and its .crate/.tgz assets are byte-identical, the step records "already-released" and skips. Receipts are excluded from that comparison because they embed the source commit. Different archives, a draft, or a tag without a release still fail; tags and releases are never overwritten. Release notes are built from the receipts instead of the hardcoded PR #202 text. 2. npm visibility. After npm publish the script waited 24 x 5 s (2 min); run 42879324451 published ruv-swarm 2.0.1 correctly but needed longer than that. wait_public() now retries 404s for up to 600 s with 5/10/20/30/60 s backoff, and other HTTP errors are raised immediately. A timeout now reports "not publicly downloadable" rather than a byte mismatch. 3. homepage https://ruv.io for ruv-swarm, cuda-wasm, @ruv-fann/napi and @ruv-fann/integrations (the last two were missing it and are unpublished). The release script refuses an existing npm version with different bytes, so the published packages move to unreleased patch versions: ruv-swarm 2.0.2 and cuda-wasm 2.0.1. Nothing is published here; the next real release carries them. The consumer smoke test now checks the candidate's cuda-wasm version instead of a hardcoded 2.0.0. ruv-swarm package.json and lockfiles are WASM receipt inputs, so sourceSha256 moves to 5a896c96. The WASM binaries are unchanged. packages/core, packages/wasm and packages/wsp already have a homepage and belong to the npm/WSP lanes; they are left unchanged. Checked: scripts/tests 72/72 (4 new release cases, 3 new visibility cases); consumer() run against locally packed ruv-swarm 2.0.2 and cuda-wasm 2.0.1; ruv-swarm npm test 104+4+8 and test:installation pass; cuda-wasm npm test 18/18; verifyWasmBuild OK; npm audit 0 (root, ruv-swarm/npm, fixture, cuda-wasm). Co-Authored-By: claude-flow --- bindings/napi/package.json | 1 + cuda-wasm/CHANGELOG.md | 6 ++ cuda-wasm/README.md | 2 +- cuda-wasm/package-lock.json | 4 +- cuda-wasm/package.json | 4 +- cuda-wasm/tests/node-wrapper.test.cjs | 2 +- integrations/ruv-fann/package.json | 1 + package-lock.json | 2 +- ruv-swarm/npm/CHANGELOG.md | 6 ++ ruv-swarm/npm/README.md | 2 +- ruv-swarm/npm/package-lock.json | 4 +- ruv-swarm/npm/package.json | 4 +- .../test-npm-install-v106/package-lock.json | 8 +-- .../npm/test-npm-install-v106/package.json | 2 +- ruv-swarm/npm/test-npm-install-v106/smoke.mjs | 2 +- ruv-swarm/npm/wasm/build-info.json | 2 +- scripts/create-github-release.py | 47 ++++++++++--- scripts/publish-npm-candidates.py | 44 ++++++++---- scripts/tests/test_github_release.py | 38 +++++++++-- scripts/tests/test_publish_npm_candidates.py | 67 +++++++++++++++++++ 20 files changed, 204 insertions(+), 44 deletions(-) create mode 100644 scripts/tests/test_publish_npm_candidates.py diff --git a/bindings/napi/package.json b/bindings/napi/package.json index f4425993..6b50fad4 100644 --- a/bindings/napi/package.json +++ b/bindings/napi/package.json @@ -12,6 +12,7 @@ "files": ["index.cjs", "index.mjs", "index.d.ts", "ruv_fann.node", "build-info.json", "README.md", "LICENSE-MIT", "LICENSE-APACHE"], "engines": { "node": ">=22.13.0" }, "license": "MIT OR Apache-2.0", + "homepage": "https://ruv.io", "repository": { "type": "git", "url": "https://github.com/ruvnet/ruv-FANN.git", "directory": "bindings/napi" }, "scripts": { "build": "node scripts/build.mjs", diff --git a/cuda-wasm/CHANGELOG.md b/cuda-wasm/CHANGELOG.md index 63c0e6c6..f007ee80 100644 --- a/cuda-wasm/CHANGELOG.md +++ b/cuda-wasm/CHANGELOG.md @@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [2.0.1] - Unreleased + +### Package metadata + +- Set `homepage` to https://ruv.io. No code or dependency change. + ## [2.0.0] - Unreleased ### Breaking changes diff --git a/cuda-wasm/README.md b/cuda-wasm/README.md index ef282873..7d7713d1 100644 --- a/cuda-wasm/README.md +++ b/cuda-wasm/README.md @@ -2,7 +2,7 @@ `cuda-rust-wasm` is an experimental Rust toolkit for reading a subset of CUDA C++, generating Rust or WGSL text, and experimenting with CPU reference kernels. CUDA is an NVIDIA trademark; this project is independent of NVIDIA. -The Rust library and `cuda2rust` command have a staged 0.2.0 release. The `cuda-wasm` npm package has a staged 2.0.0 release with a smaller, working contract: a Node.js wrapper around that command, returning Rust or WGSL **source text**. These versions are prepared in this checkout; registry publication is a separate step. +The Rust library and `cuda2rust` command have a staged 0.2.0 release. The `cuda-wasm` npm package has a staged 2.0.1 release with a smaller, working contract: a Node.js wrapper around that command, returning Rust or WGSL **source text**. These versions are prepared in this checkout; registry publication is a separate step. ![CUDA source passes through the Node wrapper to a trusted cuda2rust executable, which returns Rust or WGSL source for separate validation.](docs/assets/source-flow.svg) diff --git a/cuda-wasm/package-lock.json b/cuda-wasm/package-lock.json index b30433f7..c5213a80 100644 --- a/cuda-wasm/package-lock.json +++ b/cuda-wasm/package-lock.json @@ -1,12 +1,12 @@ { "name": "cuda-wasm", - "version": "2.0.0", + "version": "2.0.1", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "cuda-wasm", - "version": "2.0.0", + "version": "2.0.1", "license": "MIT", "bin": { "cuda-wasm": "cli/index.js" diff --git a/cuda-wasm/package.json b/cuda-wasm/package.json index d61cd29d..8f45029f 100644 --- a/cuda-wasm/package.json +++ b/cuda-wasm/package.json @@ -1,6 +1,6 @@ { "name": "cuda-wasm", - "version": "2.0.0", + "version": "2.0.1", "description": "Node.js wrapper for cuda2rust: generate Rust or WGSL source from a subset of CUDA C++", "main": "./dist/index.js", "module": "./dist/index.mjs", @@ -52,7 +52,7 @@ "bugs": { "url": "https://github.com/ruvnet/ruv-FANN/issues" }, - "homepage": "https://github.com/ruvnet/ruv-FANN/tree/main/cuda-wasm#readme", + "homepage": "https://ruv.io", "funding": { "type": "github", "url": "https://github.com/sponsors/ruvnet" diff --git a/cuda-wasm/tests/node-wrapper.test.cjs b/cuda-wasm/tests/node-wrapper.test.cjs index 350a752b..7b57b1a8 100644 --- a/cuda-wasm/tests/node-wrapper.test.cjs +++ b/cuda-wasm/tests/node-wrapper.test.cjs @@ -25,7 +25,7 @@ function rejectsCode(promise, code) { test('CJS, ESM and package version expose only the documented source API', async () => { const esm = await import(pathToFileURL(path.resolve(__dirname, '../dist/index.mjs')).href); - assert.equal(api.getVersion(), '2.0.0'); + assert.equal(api.getVersion(), '2.0.1'); assert.equal(api.getVersion(), pkg.version); assert.equal(esm.transpileCuda, api.transpileCuda); assert.equal(esm.default.transpileFile, api.transpileFile); diff --git a/integrations/ruv-fann/package.json b/integrations/ruv-fann/package.json index e15bf124..ad43166d 100644 --- a/integrations/ruv-fann/package.json +++ b/integrations/ruv-fann/package.json @@ -14,6 +14,7 @@ "files": ["src/", "README.md", "LICENSE-MIT", "LICENSE-APACHE"], "engines": { "node": ">=22.13.0" }, "license": "MIT OR Apache-2.0", + "homepage": "https://ruv.io", "repository": { "type": "git", "url": "https://github.com/ruvnet/ruv-FANN.git", "directory": "integrations/ruv-fann" }, "publishConfig": { "access": "public" }, "scripts": { diff --git a/package-lock.json b/package-lock.json index de731499..e4ca0ffd 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13568,7 +13568,7 @@ }, "ruv-swarm/npm": { "name": "ruv-swarm", - "version": "2.0.1", + "version": "2.0.2", "hasInstallScript": true, "license": "MIT OR Apache-2.0", "dependencies": { diff --git a/ruv-swarm/npm/CHANGELOG.md b/ruv-swarm/npm/CHANGELOG.md index 2d2f778d..a41175d6 100644 --- a/ruv-swarm/npm/CHANGELOG.md +++ b/ruv-swarm/npm/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## [2.0.2] - Unreleased + +### Package metadata + +- Set `homepage` to https://ruv.io. No code, dependency or WASM change. + ## [2.0.1] - 2026-11-10 ### Release pipeline diff --git a/ruv-swarm/npm/README.md b/ruv-swarm/npm/README.md index b64fee80..39387482 100644 --- a/ruv-swarm/npm/README.md +++ b/ruv-swarm/npm/README.md @@ -388,7 +388,7 @@ Version 2.0 narrows the public API to working capabilities. Use `RuvSwarm.initia MCP clients should refresh `tools/list`: unsupported autonomous, training and invented telemetry tools have been removed. Assignment replies return actual task IDs, `assigned` or `unassigned`, and `assignedAgents`; they contain no estimated completion or fabricated model accuracy. SQLite records survive process restarts. State defaults to `.ruv-swarm/state.sqlite` in the working project; `RUV_SWARM_DB_PATH` selects an explicit location. To use an older database, back it up and point that variable at the existing file. -The npm package is 2.0.1; its bundled core Rust WASM crate has its own version, 1.0.6. See the [changelog](./CHANGELOG.md) for the compatibility changes. +The npm package is 2.0.2; its bundled core Rust WASM crate has its own version, 1.0.6. See the [changelog](./CHANGELOG.md) for the compatibility changes. ## License diff --git a/ruv-swarm/npm/package-lock.json b/ruv-swarm/npm/package-lock.json index 4a51d8cc..bbc27a0e 100644 --- a/ruv-swarm/npm/package-lock.json +++ b/ruv-swarm/npm/package-lock.json @@ -1,12 +1,12 @@ { "name": "ruv-swarm", - "version": "2.0.1", + "version": "2.0.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ruv-swarm", - "version": "2.0.1", + "version": "2.0.2", "hasInstallScript": true, "license": "MIT OR Apache-2.0", "dependencies": { diff --git a/ruv-swarm/npm/package.json b/ruv-swarm/npm/package.json index 6aa8e3e1..d4ac0427 100644 --- a/ruv-swarm/npm/package.json +++ b/ruv-swarm/npm/package.json @@ -1,6 +1,6 @@ { "name": "ruv-swarm", - "version": "2.0.1", + "version": "2.0.2", "description": "Node.js swarm metadata, SQLite persistence, and checked WebAssembly inference", "main": "src/index.js", "module": "src/index.js", @@ -166,7 +166,7 @@ "bugs": { "url": "https://github.com/ruvnet/ruv-FANN/issues" }, - "homepage": "https://github.com/ruvnet/ruv-FANN#readme", + "homepage": "https://ruv.io", "dependencies": { "better-sqlite3": "12.11.1", "node-gyp": "12.4.0", diff --git a/ruv-swarm/npm/test-npm-install-v106/package-lock.json b/ruv-swarm/npm/test-npm-install-v106/package-lock.json index 40e30ea3..a4965be1 100644 --- a/ruv-swarm/npm/test-npm-install-v106/package-lock.json +++ b/ruv-swarm/npm/test-npm-install-v106/package-lock.json @@ -9,7 +9,7 @@ "version": "1.0.0", "license": "MIT OR Apache-2.0", "dependencies": { - "ruv-swarm": "file:../ruv-swarm-2.0.1.tgz" + "ruv-swarm": "file:../ruv-swarm-2.0.2.tgz" }, "engines": { "node": ">=22.13.0" @@ -473,9 +473,9 @@ } }, "node_modules/ruv-swarm": { - "version": "2.0.1", - "resolved": "file:../ruv-swarm-2.0.1.tgz", - "integrity": "sha512-urMqtsVJ8V8fPVh75CaN2Q41XT18wrJxqHrHeid6Yn63+W7KkpA9hFfGqjhnsF6Dfl/G6+RhYbPjEKaWrwKmyg==", + "version": "2.0.2", + "resolved": "file:../ruv-swarm-2.0.2.tgz", + "integrity": "sha512-5SlGjtKdegdasvjWOwRQAMDcL086AYrtfsyKAzHk7FDJmzYznSUdMsYpCiEJo4NmEHWO16NK6eu7jwAH5/RkKA==", "hasInstallScript": true, "license": "MIT OR Apache-2.0", "dependencies": { diff --git a/ruv-swarm/npm/test-npm-install-v106/package.json b/ruv-swarm/npm/test-npm-install-v106/package.json index 69cb10c1..8f1525bb 100644 --- a/ruv-swarm/npm/test-npm-install-v106/package.json +++ b/ruv-swarm/npm/test-npm-install-v106/package.json @@ -12,6 +12,6 @@ "test": "node smoke.mjs" }, "dependencies": { - "ruv-swarm": "file:../ruv-swarm-2.0.1.tgz" + "ruv-swarm": "file:../ruv-swarm-2.0.2.tgz" } } diff --git a/ruv-swarm/npm/test-npm-install-v106/smoke.mjs b/ruv-swarm/npm/test-npm-install-v106/smoke.mjs index 138ad088..521185cc 100644 --- a/ruv-swarm/npm/test-npm-install-v106/smoke.mjs +++ b/ruv-swarm/npm/test-npm-install-v106/smoke.mjs @@ -7,7 +7,7 @@ const Database = apiRequire('better-sqlite3'); const database = new Database(':memory:'); try { assert.equal(database.prepare('SELECT 2 + 3 AS result').get().result, 5); } finally { database.close(); } -assert.equal(RuvSwarm.getVersion(), '2.0.1'); +assert.equal(RuvSwarm.getVersion(), '2.0.2'); const runtime = await RuvSwarm.initialize(); const network = runtime.createNeuralNetwork([2, 1], 'Linear'); try { diff --git a/ruv-swarm/npm/wasm/build-info.json b/ruv-swarm/npm/wasm/build-info.json index fe148a88..7416a3c9 100644 --- a/ruv-swarm/npm/wasm/build-info.json +++ b/ruv-swarm/npm/wasm/build-info.json @@ -1,6 +1,6 @@ { "schemaVersion": 2, - "sourceSha256": "a7a4c4c7467b543e2266e72b474cd088e3e82d4aaa3042b251f358752d58feb4", + "sourceSha256": "5a896c96cff597f89ab174786d44b6fa3a1a70db0b3df423c7c1f87db39c4fd4", "toolchain": { "rustc": "rustc 1.88.0 (6b00bc388 2025-06-23)", "wasmPack": "wasm-pack 0.15.1" diff --git a/scripts/create-github-release.py b/scripts/create-github-release.py index 21c6eeb0..e60d3635 100644 --- a/scripts/create-github-release.py +++ b/scripts/create-github-release.py @@ -3,6 +3,7 @@ import hashlib import json import os +import re from pathlib import Path import shutil import subprocess @@ -17,6 +18,13 @@ def digest(path): return hashlib.sha256(path.read_bytes()).hexdigest() +ARCHIVE_SUFFIXES = ('.crate', '.tgz') + + +def write_receipt(base, source, repo, tag, status, manifest): + (base / 'github-release.json').write_text(json.dumps({'sourceSha': source, 'tag': tag, 'status': status, 'url': f'https://github.com/{repo}/releases/tag/{tag}', 'assets': manifest}, indent=2)) + + def main(): base = Path(sys.argv[1]).resolve() source = os.environ['GITHUB_SHA'] @@ -53,19 +61,40 @@ def main(): manifest = {p.name: digest(p) for p in assets.iterdir()} (assets / 'SHA256SUMS').write_text(''.join(f'{value} {name}\n' for name, value in sorted(manifest.items()))) manifest['SHA256SUMS'] = digest(assets / 'SHA256SUMS') + versions = {r['name']: r.get('version') for r in packages} + swarm = versions.get('ruv-swarm') + if not isinstance(swarm, str) or not re.fullmatch(r'\d+\.\d+\.\d+', swarm): + raise ValueError('Release tag requires the published ruv-swarm npm version') + # One release per ruv-swarm npm version. The npm byte check already + # forces a new ruv-swarm version whenever its packaged bytes change. + repo, tag = 'ruvnet/ruv-FANN', f'ruv-fann-registry-v{swarm}' + archives = {name: value for name, value in manifest.items() if name.endswith(ARCHIVE_SUFFIXES)} + # Never overwrite an existing release or move a tag. A rerun of the same + # version with identical package archives is already released; receipts + # are excluded from the comparison because they embed the source commit. + release = subprocess.run(['gh', 'release', 'view', tag, '--repo', repo, '--json', 'isDraft'], capture_output=True, text=True) + ref = subprocess.run(['gh', 'api', f'repos/{repo}/git/ref/tags/{tag}'], capture_output=True) + if release.returncode == 0 or ref.returncode == 0: + if release.returncode != 0 or json.loads(release.stdout).get('isDraft') is not False: + raise ValueError(f'Tag {tag} exists without a published release; inspect before retrying') + existing = base / 'github-existing-download' + run('release', 'download', tag, '--repo', repo, '--dir', str(existing)) + published = {p.name: digest(p) for p in existing.iterdir() if p.name.endswith(ARCHIVE_SUFFIXES)} + if published != archives: + raise ValueError(f'Release {tag} exists with different package archives; bump the ruv-swarm npm version') + write_receipt(base, source, repo, tag, 'already-released', manifest) + return + published_now = [r for r in crates if r['status'] == 'published-and-byte-verified'] + listing = ', '.join(f"`{r['name']}@{r.get('version', '?')}`" for r in packages) + new_crates = ', '.join(f"`{r['name']} {r.get('version', '?')}`" for r in published_now) or 'none (all already on crates.io)' notes = base / 'release-notes.md' - notes.write_text(f'''Coordinated release from PR #202, commit `{source}`. + notes.write_text(f'''Coordinated registry release from commit `{source}`. -Includes dense autograd training, the Candle adapter, WASM training, and SQLite/MCP fixes. Publishes 27 verified Cargo crates, `ruv-swarm@2.0.0` and `cuda-wasm@2.0.0`. The native Node addon remains a local platform build. +npm packages: {listing}. Cargo crates verified: {len(crates)}; newly published: {new_crates}. All attached package archives are the exact bytes verified from the public registries. Fresh public installs passed core/SIMD training, persistent CLI, CUDA compiler and Rust dense training checks. See the attached publication receipts and SHA256SUMS. ''') - repo, tag = 'ruvnet/ruv-FANN', 'v0.2.2' - # Never overwrite an existing release or move a tag. - exists = subprocess.run(['gh', 'api', f'repos/{repo}/git/ref/tags/{tag}'], capture_output=True) - if exists.returncode == 0: - raise ValueError('Release tag already exists; inspect before retrying') - run('release', 'create', tag, '--repo', repo, '--target', source, '--draft', '--title', 'ruv-FANN 0.2.2 — dense training and coordinated packages', '--notes-file', str(notes), *[str(p) for p in sorted(assets.iterdir())]) + run('release', 'create', tag, '--repo', repo, '--target', source, '--draft', '--title', f'ruv-FANN registry release (ruv-swarm {swarm})', '--notes-file', str(notes), *[str(p) for p in sorted(assets.iterdir())]) def verify(folder): run('release', 'download', tag, '--repo', repo, '--dir', str(folder)) if {p.name: digest(p) for p in folder.iterdir()} != manifest: @@ -73,7 +102,7 @@ def verify(folder): verify(base / 'github-draft-download') run('release', 'edit', tag, '--repo', repo, '--draft=false') verify(base / 'github-public-download') - (base / 'github-release.json').write_text(json.dumps({'sourceSha': source, 'tag': tag, 'status': 'published-and-byte-verified', 'url': f'https://github.com/{repo}/releases/tag/{tag}', 'assets': manifest}, indent=2)) + write_receipt(base, source, repo, tag, 'published-and-byte-verified', manifest) if __name__ == '__main__': diff --git a/scripts/publish-npm-candidates.py b/scripts/publish-npm-candidates.py index 6127f4f0..d0a3a6d9 100644 --- a/scripts/publish-npm-candidates.py +++ b/scripts/publish-npm-candidates.py @@ -12,6 +12,26 @@ import urllib.request ROOT = Path(__file__).resolve().parents[1] +# npm can take several minutes before a new tarball is served publicly. +VISIBILITY_TIMEOUT = 600 +VISIBILITY_DELAYS = (5, 10, 20, 30, 60) + + +def wait_public(url, timeout=VISIBILITY_TIMEOUT, sleep=time.sleep, clock=time.monotonic): + """Return the public tarball bytes, or None if it stays 404 past the deadline.""" + deadline = clock() + timeout + attempt = 0 + while True: + try: + return urllib.request.urlopen(url, timeout=60).read() + except urllib.error.HTTPError as error: + if error.code != 404: + raise + remaining = deadline - clock() + if remaining <= 0: + return None + sleep(min(VISIBILITY_DELAYS[min(attempt, len(VISIBILITY_DELAYS) - 1)], remaining)) + attempt += 1 def sha(path): @@ -31,13 +51,17 @@ def head(): return subprocess.check_output(['git', 'rev-parse', 'HEAD'], cwd=ROOT, text=True).strip() -def consumer(directory, specs, compiler=None): +def consumer(directory, specs, packages, compiler=None): directory.mkdir() (directory / 'package.json').write_text('{"private":true,"type":"module"}') # Consumer lifecycle code never inherits registry publishing credentials. env = {k: v for k, v in os.environ.items() if k not in ('NODE_AUTH_TOKEN', 'NPM_TOKEN', 'CARGO_REGISTRY_TOKEN', 'GH_TOKEN', 'GITHUB_TOKEN')} (directory / '.npmrc').write_text('registry=https://registry.npmjs.org\n') env['NPM_CONFIG_USERCONFIG'] = str(directory / '.npmrc') + cuda = [r['version'] for r in packages if r['name'] == 'cuda-wasm'] + if len(cuda) != 1: raise ValueError('Expected one cuda-wasm candidate') + # The consumer checks the exact version that this release candidate carries. + env['CUDA_WASM_VERSION'] = cuda[0] run(['npm', 'install', '--registry=https://registry.npmjs.org', '--no-audit', '--no-fund', *specs], directory, env) script = r''' import assert from 'node:assert/strict'; @@ -58,7 +82,7 @@ def consumer(directory, specs, compiler=None): const cli=new URL('./node_modules/ruv-swarm/bin/ruv-swarm-secure.js',import.meta.url); const call=args=>JSON.parse(execFileSync(process.execPath,[cli.pathname,...args],{encoding:'utf8',env:{...process.env,RUV_SWARM_DB_PATH:new URL('./state.sqlite',import.meta.url).pathname}})); call(['init','mesh','2']);call(['spawn','coder','Registry consumer']);assert.equal(call(['status']).total_agents,1); -const cjs=require('cuda-wasm');const esm=await import('cuda-wasm');assert.equal(cjs.getVersion(),'2.0.0');assert.equal(esm.getVersion(),'2.0.0'); +const cjs=require('cuda-wasm');const esm=await import('cuda-wasm');assert.equal(cjs.getVersion(),process.env.CUDA_WASM_VERSION);assert.equal(esm.getVersion(),process.env.CUDA_WASM_VERSION); if(process.env.CUDA2RUST_BINARY){ const source='__global__ void copy(float* a, float* b) { int i = threadIdx.x; b[i] = a[i]; }'; assert.match((await cjs.transpileCuda(source)).code,/fn copy/); @@ -101,7 +125,7 @@ def main(): training_archive = list((base / 'npm-training').glob('*.tgz')) if len(training_archive) != 1 or sha(training_archive[0]) != candidates[0]['sha256']: raise ValueError('Training-tested archive differs') - consumer(base / 'npm-local-consumer', [r['archive'] for r in candidates]) + consumer(base / 'npm-local-consumer', [r['archive'] for r in candidates], candidates) save(seal, {'sourceSha': head(), 'packages': candidates}) return record = json.loads(seal.read_text()) @@ -122,21 +146,17 @@ def main(): except urllib.error.HTTPError as error: if error.code != 404: raise run(['npm','publish',row['archive'],'--ignore-scripts','--access','public','--provenance','--registry=https://registry.npmjs.org'],ROOT) - public = None - for _ in range(24): - try: - public = urllib.request.urlopen(url, timeout=60).read(); break - except urllib.error.HTTPError as retry: - if retry.code != 404: raise - time.sleep(5) - if public is None or hashlib.sha256(public).hexdigest() != row['sha256']: + public = wait_public(url) + if public is None: + raise RuntimeError(f'Published, but {url} was not publicly downloadable within {VISIBILITY_TIMEOUT}s; rerun the failed job to verify the bytes') + if hashlib.sha256(public).hexdigest() != row['sha256']: raise ValueError('Published npm bytes do not match candidate: '+row['name']) published.append({**row, 'status':'published-and-byte-verified'}) stream.seek(0);json.dump({'sourceSha':head(),'packages':published},stream,indent=2);stream.truncate();stream.flush() return # Install the Cargo compiler and dense example from public crates.io too. run(['cargo','install','cuda-rust-wasm','--version','=0.2.0','--locked','--bin','cuda2rust','--root',str(base/'public-compiler')],base) - consumer(base/'npm-public-consumer',[r['name']+'@'+r['version'] for r in record['packages']],base/'public-compiler/bin/cuda2rust') + consumer(base/'npm-public-consumer',[r['name']+'@'+r['version'] for r in record['packages']],record['packages'],base/'public-compiler/bin/cuda2rust') rust = base/'rust-public-consumer';(rust/'src').mkdir(parents=True) (rust/'Cargo.toml').write_text('[package]\nname="release-consumer"\nversion="0.0.0"\nedition="2021"\n[dependencies]\nruv-fann={version="=0.2.2",default-features=false,features=["std","serde"]}\nruv-fann-candle="=0.1.0"\n') (rust/'src/main.rs').write_text((ROOT/'examples/dense_autograd.rs').read_text()) diff --git a/scripts/tests/test_github_release.py b/scripts/tests/test_github_release.py index 8e743b95..3ac6c978 100644 --- a/scripts/tests/test_github_release.py +++ b/scripts/tests/test_github_release.py @@ -14,7 +14,7 @@ class GitHubReleaseTests(unittest.TestCase): - def exercise(self, corrupt=False, consumer_passed=True): + def exercise(self, corrupt=False, consumer_passed=True, existing=None, swarm_version='2.0.2'): with tempfile.TemporaryDirectory() as temporary: base = Path(temporary) (base / 'cargo-candidates').mkdir() @@ -22,34 +22,64 @@ def exercise(self, corrupt=False, consumer_passed=True): for index in range(29): archive = base / ('cargo-candidates' if index < 27 else '.') / f'package-{index}.{"crate" if index < 27 else "tgz"}' archive.write_bytes(f'exact published bytes {index}'.encode()) - row = {'name': f'package-{index}', 'sha256': module.digest(archive), 'archive': str(archive), 'status': 'published-and-byte-verified'} + name = f'package-{index}' if index < 27 else ('ruv-swarm', 'cuda-wasm')[index - 27] + version = '1.0.0' if index < 27 else (swarm_version, '2.0.1')[index - 27] + row = {'name': name, 'version': version, 'sha256': module.digest(archive), 'archive': str(archive), 'status': 'published-and-byte-verified'} (crates if index < 27 else packages).append(row) for name, field, rows in [('cargo-publication', 'crates', crates), ('npm-publication', 'packages', packages)]: (base / (name + '.json')).write_text(json.dumps({'sourceSha': 'reviewed', field: rows})) (base / 'public-consumers.json').write_text(json.dumps({'sourceSha': 'reviewed', 'status': 'passed' if consumer_passed else 'failed', **dict.fromkeys(('npmNormalInstall','coreAndSimdTraining','persistentCli','cudaPublicCompiler','cargoPublicDenseTraining'), True)})) calls = [] + def gh_lookup(argv): + if existing is None: + return SimpleNamespace(returncode=1, stdout='') + return SimpleNamespace(returncode=0, stdout=json.dumps({'isDraft': existing == 'draft'})) def gh(*args): calls.append(args) if args[:2] == ('release', 'download'): folder = Path(args[args.index('--dir') + 1]) + if existing is not None and folder.name == 'github-existing-download': + shutil.copytree(base / 'github-release-assets', folder) + (folder / 'cargo-publication.json').write_text('{"sourceSha": "older commit"}') + if existing == 'different': + (folder / 'package-3.crate').write_bytes(b'other release bytes') + return '' shutil.copytree(base / 'github-release-assets', folder) if corrupt: (folder / 'package-0.crate').write_bytes(b'wrong bytes') return '' - with patch.dict(os.environ, {'GITHUB_ACTIONS': 'true', 'GITHUB_WORKFLOW': 'protected-release', 'GITHUB_REF': 'refs/heads/main', 'GITHUB_SHA': 'reviewed'}), patch.object(module.sys, 'argv', ['script', str(base)]), patch.object(module, 'run', side_effect=gh), patch.object(module.subprocess, 'run', return_value=SimpleNamespace(returncode=1)): - if corrupt or not consumer_passed: + with patch.dict(os.environ, {'GITHUB_ACTIONS': 'true', 'GITHUB_WORKFLOW': 'protected-release', 'GITHUB_REF': 'refs/heads/main', 'GITHUB_SHA': 'reviewed'}), patch.object(module.sys, 'argv', ['script', str(base)]), patch.object(module, 'run', side_effect=gh), patch.object(module.subprocess, 'run', side_effect=lambda argv, **kw: gh_lookup(argv)): + if existing == 'same': + module.main() + receipt = json.loads((base / 'github-release.json').read_text()) + self.assertEqual(receipt['status'], 'already-released') + self.assertEqual(receipt['tag'], f'ruv-fann-registry-v{swarm_version}') + self.assertFalse(any(c[:2] in (('release', 'create'), ('release', 'edit')) for c in calls)) + elif corrupt or not consumer_passed or existing in ('different', 'draft') or swarm_version is None: with self.assertRaises(ValueError): module.main() self.assertFalse(any(c[:2] == ('release', 'edit') for c in calls)) + if not corrupt: + self.assertFalse(any(c[:2] == ('release', 'create') for c in calls)) else: module.main() receipt = json.loads((base / 'github-release.json').read_text()) self.assertEqual(receipt['status'], 'published-and-byte-verified') self.assertEqual(len(receipt['assets']), 33) self.assertEqual(sum(c[:2] == ('release', 'download') for c in calls), 2) + create = next(c for c in calls if c[:2] == ('release', 'create')) + self.assertEqual(create[2], f'ruv-fann-registry-v{swarm_version}') def test_exact_assets_are_verified_before_and_after_publication(self): self.exercise() def test_corrupt_draft_cannot_be_published(self): self.exercise(corrupt=True) def test_failed_consumers_cannot_create_release(self): self.exercise(consumer_passed=False) + def test_rerun_with_identical_archives_is_already_released(self): + self.exercise(existing='same') + def test_existing_release_with_different_archives_fails(self): + self.exercise(existing='different') + def test_existing_draft_release_fails(self): + self.exercise(existing='draft') + def test_missing_ruv_swarm_version_fails(self): + self.exercise(swarm_version=None) diff --git a/scripts/tests/test_publish_npm_candidates.py b/scripts/tests/test_publish_npm_candidates.py new file mode 100644 index 00000000..91c91807 --- /dev/null +++ b/scripts/tests/test_publish_npm_candidates.py @@ -0,0 +1,67 @@ +import importlib.util +from pathlib import Path +import unittest +import urllib.error +from unittest.mock import patch + +spec = importlib.util.spec_from_file_location('publish_npm', Path(__file__).resolve().parents[1] / 'publish-npm-candidates.py') +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + + +class Clock: + def __init__(self): + self.now = 0.0 + self.sleeps = [] + + def time(self): + return self.now + + def sleep(self, seconds): + self.sleeps.append(seconds) + self.now += seconds + + +def not_found(url): + return urllib.error.HTTPError(url, 404, 'Not Found', {}, None) + + +class Response: + def __init__(self, data): + self.data = data + + def read(self): + return self.data + + +class VisibilityTests(unittest.TestCase): + def test_waits_past_two_minutes_with_backoff(self): + clock = Clock() + def urlopen(url, timeout): + if clock.now < 300: + raise not_found(url) + return Response(b'public bytes') + with patch.object(module.urllib.request, 'urlopen', side_effect=urlopen): + data = module.wait_public('https://registry.example/pkg.tgz', sleep=clock.sleep, clock=clock.time) + self.assertEqual(data, b'public bytes') + self.assertEqual(clock.sleeps[:5], [5, 10, 20, 30, 60]) + self.assertTrue(all(delay <= 60 for delay in clock.sleeps)) + + def test_gives_up_at_the_deadline(self): + clock = Clock() + with patch.object(module.urllib.request, 'urlopen', side_effect=lambda url, timeout: (_ for _ in ()).throw(not_found(url))): + data = module.wait_public('https://registry.example/pkg.tgz', sleep=clock.sleep, clock=clock.time) + self.assertIsNone(data) + self.assertEqual(clock.now, module.VISIBILITY_TIMEOUT) + + def test_other_http_errors_are_not_retried(self): + clock = Clock() + error = urllib.error.HTTPError('u', 500, 'Server Error', {}, None) + with patch.object(module.urllib.request, 'urlopen', side_effect=error): + with self.assertRaises(urllib.error.HTTPError): + module.wait_public('https://registry.example/pkg.tgz', sleep=clock.sleep, clock=clock.time) + self.assertEqual(clock.sleeps, []) + + +if __name__ == '__main__': + unittest.main()