diff --git a/.github/workflows/datasets.yml b/.github/workflows/datasets.yml index ae9f5a8..b38dbdc 100644 --- a/.github/workflows/datasets.yml +++ b/.github/workflows/datasets.yml @@ -2,16 +2,11 @@ name: datasets # On GitHub's hosted runners only. This never runs a recipe or pulls a dataset. check: every pull # request, the scripts' unit tests and, for each changed dataset, its rules, a version not yet on -# the Hub, its tools in the toolshed, and a passing record whose files the Hub holds exactly. -# promote: a push to main that changes a dataset, tagging its record's commit as the version on the -# Hub and deleting the candidate branch; it needs the repository secret HF_TOKEN, with write on the -# org's datasets. +# the Hub, its tools in the toolshed, and a passing record whose files the Hub holds exactly. CI holds +# no Hub token: whoever merges runs scripts/promote with their own. on: pull_request: - push: - branches: [main] - paths: ["datasets/**"] permissions: contents: read @@ -22,7 +17,6 @@ defaults: jobs: check: - if: github.event_name == 'pull_request' runs-on: ubuntu-latest steps: - uses: actions/checkout@v7 @@ -38,25 +32,3 @@ jobs: failed=0 for d in $datasets; do python3 -I scripts/check.py --hub "datasets/$d" || failed=1; done exit "$failed" - - promote: - if: github.event_name == 'push' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v7 - with: - fetch-depth: 0 - - uses: actions/setup-python@v7 - with: - python-version: "3.12" - - run: python3 -m pip install -q huggingface_hub - - env: - BEFORE: ${{ github.event.before }} - HF_TOKEN: ${{ secrets.HF_TOKEN }} - run: | - datasets=$(scripts/changed-datasets "$BEFORE" "$GITHUB_SHA") - [[ -n "$datasets" ]] || { echo "no dataset to promote"; exit 0; } - for d in $datasets; do - VERSION=$(python3 -I -c "import sys; sys.path.insert(0, 'scripts'); import check, pathlib; print(check.frontmatter(pathlib.Path('datasets/$d/SKILL.md'))['metadata']['version'])") - python3 -I scripts/promote "$d" "$VERSION" | tail -n 1 | tee -a "$GITHUB_STEP_SUMMARY" - done diff --git a/AGENTS.md b/AGENTS.md index b2128f8..447feec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -25,9 +25,9 @@ hub.toml the Hub org this repository publishes to scripts/check.py a dataset directory's rules, its record, and whether its version is published scripts/push upload the cook's output and the card to the Hub's candidate branch; prints the manifest and `commit:` scripts/record < output write a record from the test's and push's output -scripts/promote tag the record's commit on the Hub, never moving a tag; delete the candidate branch +scripts/promote after the merge, by whoever merged: tag the record's commit on the Hub, never moving a tag; delete the candidate branch tests/ unit tests of the scripts, with the Hub stubbed -.github/workflows/datasets.yml check and promote, on hosted runners +.github/workflows/datasets.yml check, on hosted runners ``` ## Commands @@ -52,7 +52,7 @@ datasets//recipe/build && datasets//recipe/test && pytho ## Rules the repository keeps -- `scripts/check.py` and `scripts/record` run with the Python standard library alone (`python3 -I`): flyloft's host runs them, and so do login nodes. `scripts/push` and `scripts/promote` need `huggingface_hub`, which `datascience` and CI have. +- `scripts/check.py` and `scripts/record` run with the Python standard library alone (`python3 -I`): flyloft's host runs them, and so do login nodes. `scripts/push` and `scripts/promote` need `huggingface_hub`, which `datascience` has. - Nothing in `scripts/` or `.github/` names the lab: the org comes from `hub.toml`, so a fork with its own org is its own catalogue and store. - CI runs on hosted runners and never runs a recipe or pulls a dataset. Whoever cooks a dataset tests its output where it was built, files it with `scripts/push`, and commits the record; nobody can claim a version exists without one. - Every dataset is reviewed by someone other than its builder before it merges. @@ -63,4 +63,4 @@ A dataset exists because a question needed it. Do not add datasets, parameters, ## Commits and pull requests -A commit message is one sentence stating the behavior. A dataset arrives as a pull request whose record is committed with it. Merge with a merge commit (`gh pr merge --merge`): `promote` runs on the merge commit, and flyloft watches that check. +A commit message is one sentence stating the behavior. A dataset arrives as a pull request whose record is committed with it. Merge with a merge commit (`gh pr merge --merge`), then pull `main` and run `scripts/promote ` with your own `HF_TOKEN`, in `datascience`: CI holds no Hub token. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 02178d4..84c2a67 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -103,4 +103,11 @@ Any change under `datasets//` bumps `metadata.version`, prose included. A 1. Push your branch and open the pull request, the record committed in it. 2. `check` runs: the dataset's rules, an unpublished version, each tool in the toolshed at its version, and a record of a passing test whose files the Hub holds exactly. 3. Someone other than you reviews it, a critic agent included: does the skill describe the files the record lists and state the recipe's inputs; does the record show the test passed where the build ran; is the version new; are the tools the ones the record shows. They merge with a merge commit (`gh pr merge --merge`). -4. `promote` tags the record's commit `v` on the Hub and deletes the candidate branch. +4. Whoever merged promotes it: pull `main`, then, from the clone's root, + + ```bash + HF_TOKEN=$(< ~/.cache/huggingface/token) podman run --rm -e HF_TOKEN -v "$PWD":/work -w /work \ + ghcr.io/brineylab/toolshed/datascience:1.0.0 python3 -I scripts/promote + ``` + + It tags the record's commit `v` on the Hub and deletes the candidate branch; its last line says what it did. It refuses a record that is not on `origin/main`. Run it from a full clone, not a `git worktree`, whose `.git` points outside the mounted directory. CI holds no Hub token, so nothing is published until someone runs it. diff --git a/README.md b/README.md index 3c9816d..7d1f50f 100644 --- a/README.md +++ b/README.md @@ -38,4 +38,4 @@ For Codex and OpenCode, link into `~/.agents/skills/` instead. `git pull` update A dataset's version is a tag, `v`, on its Hub repository, and a published tag is never moved. A version changes when the source is re-pulled, a parameter moves, or a tool moves; a variant (another species, another threshold) is another dataset. -CI checks every pull request: the dataset's rules, an unpublished version, the recipe's tools in the toolshed, and a record of the candidate's passing test whose files are exactly the Hub's. On merge it tags the record's commit. CI checks records and never pulls a dataset; whoever builds a dataset tests it and files it. +CI checks every pull request: the dataset's rules, an unpublished version, the recipe's tools in the toolshed, and a record of the candidate's passing test whose files are exactly the Hub's. After the merge, whoever merged tags the record's commit with `scripts/promote`. CI checks records and never pulls a dataset; whoever builds a dataset tests it and files it. diff --git a/docs/DESIGN.md b/docs/DESIGN.md index e1b4f3b..06be4c2 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -1,6 +1,6 @@ # The datasets repository: Design -Status: approved 2026-11-04; revised 2026-11-05 by [superpowers/specs/2026-11-05-recipes-tools-and-jaffe-paired-design.md](superpowers/specs/2026-11-05-recipes-tools-and-jaffe-paired-design.md), so that a recipe is instructions any cook runs, names its tools, and runs in the toolshed's `datascience` (§2, §3, §4, §5, §6). This is the repository's own copy of the design that lives in flyloft as [`docs/design/datasets.md`](https://github.com/brineylab/flyloft/blob/main/docs/design/datasets.md), with the lab's decisions on flyloft issue [#174](https://github.com/brineylab/flyloft/issues/174) (D1–D7, all option A) marked by number. §1 to §4 are this repository's; §5 summarizes flyloft's side, which that document settles. Calls made in writing that nobody has discussed are marked **(call)**. The toolshed's design ([brineylab/toolshed, docs/DESIGN.md](https://github.com/brineylab/toolshed/blob/main/docs/DESIGN.md)) is the sibling this one mirrors; where this document says "as the toolshed's", that is the reference. +Status: approved 2026-11-04; revised 2026-11-05 by [superpowers/specs/2026-11-05-recipes-tools-and-jaffe-paired-design.md](superpowers/specs/2026-11-05-recipes-tools-and-jaffe-paired-design.md), so that a recipe is instructions any cook runs, names its tools, and runs in the toolshed's `datascience` (§2, §3, §4, §5, §6); revised 2026-11-06 so that whoever merges promotes with their own Hub token, and CI holds none, since a Hub token is a person's, not an org's (§3, §5). This is the repository's own copy of the design that lives in flyloft as [`docs/design/datasets.md`](https://github.com/brineylab/flyloft/blob/main/docs/design/datasets.md), with the lab's decisions on flyloft issue [#174](https://github.com/brineylab/flyloft/issues/174) (D1–D7, all option A) marked by number. §1 to §4 are this repository's; §5 summarizes flyloft's side, which that document settles. Calls made in writing that nobody has discussed are marked **(call)**. The toolshed's design ([brineylab/toolshed, docs/DESIGN.md](https://github.com/brineylab/toolshed/blob/main/docs/DESIGN.md)) is the sibling this one mirrors; where this document says "as the toolshed's", that is the reference. ## 1. Scenario @@ -63,8 +63,8 @@ results//.txt the record of the candidate's test: the Hub commi scripts/check.py a dataset directory's rules, its record, and whether its version is published scripts/push upload the cook's output and the card to the Hub on the candidate branch; prints the manifest and the commit scripts/record a record from the test's and the push's output, for any cook -scripts/promote tag the record's commit as the version on the Hub; delete the candidate branch -.github/workflows/datasets.yml check and promote +scripts/promote run by whoever merges: tag the merged record's commit as the version on the Hub; delete the candidate branch +.github/workflows/datasets.yml check ``` **Versions.** `metadata.version` is the Hub tag, `v` on `/`, with the org from `hub.toml`, so a fork with its own org is its own catalogue and store with one edit. A published tag is never moved: any change under `datasets//` bumps the version, prose included, and CI refuses a pull request whose tag is already on the Hub. Agents and flyloft always name a version, never `main`. @@ -75,10 +75,11 @@ scripts/promote tag the record's commit as the version on the Hub **The card.** The Hub asks every large dataset for a card and serves its viewer from parquet. `scripts/push` writes `README.md` on the candidate branch from the skill: a header with the license and the name, then the skill's body. One source, two readers. -**The workflows**, on GitHub's hosted runners only, with `huggingface_hub` installed and never a dataset downloaded: +**The workflow**, on GitHub's hosted runners only, never downloading a dataset and holding no Hub token: - `check`, on every pull request, for the datasets it changes: the skill's frontmatter parses, its name is the directory's, each of its tools is in the toolshed's catalogue at that version (read from the shed's `main` by API); `recipe/build`, `recipe/params.toml`, and `recipe/test` exist; the version's tag is not on the Hub; the record exists for this version, names a commit on the Hub whose tree holds exactly the record's files with the record's sha256 (the Hub's file listing, no download), and shows exit code 0. -- `promote`, on a push to `main`, for the datasets the push changed: the record's commit tagged `v` on the Hub, never moving a tag that exists, and the candidate branch deleted. It runs on the push, so its check attaches to the merge commit, which is what flyloft watches (§5). It needs a Hub token with write on the org's datasets, a repository secret; the toolshed's promote needs none, which is the one place the two repositories' CI differ. + +**Promoting** is the merger's, as pushing is the cook's: after the merge, whoever merged pulls `main` and runs `scripts/promote ` with their own `HF_TOKEN`, which tags the record's commit `v` on the Hub, never moving a tag that exists, and deletes the candidate branch; it refuses a record that is not on `origin/main`, so only a reviewed, merged record is published, and its last line says what it did. It runs from a full clone, since a worktree's `.git` points outside the directory a container mounts. A Hub token is a person's, so CI holds none; this is the one place the repository differs from the toolshed, whose CI promotes with GitHub's own token. **Guards.** Nothing a pull request contains runs on the lab's machines: CI runs the check script and its unit tests. Nothing that goes public holds a credential: `check.py` refuses a dataset directory or a record that holds the shape of one, in `scripts/push` before anything goes up, in `scripts/record` before anything is written, and in `check` as the backstop; `scripts/push` reads its token from `HF_TOKEN` and never prints it. The record's file list is the Hub's: what a version holds is what the record says, and `check` proves it. `main` requires `check` and no other approval: flyloft merges as the campaign's owner after its critic's review, and a person's pull request is merged by a member other than its builder. @@ -95,8 +96,8 @@ A recipe runs in toolshed images named in its skill's `metadata.tools`, most oft Settled in flyloft's [`docs/design/datasets.md`](https://github.com/brineylab/flyloft/blob/main/docs/design/datasets.md) §4 to §5; this repository needs none of it to work, and none of it changes what this repository is. - **Reading (D6: jobs pull).** Every job script exports `HF_HUB_CACHE` under the cluster's shared data directory, and the Hub library does the rest: a cached version is returned at once, a missing one is downloaded, two jobs pulling at once cannot leave a half file. The cache is disposable. The implementer's assignment carries the catalogue rendered from this repository's skills. -- **Building.** A campaign's implementer gets a worktree of this repository, writes a dataset, and calls `submit_dataset`; flyloft cooks the recipe as its skill says (one job, or several), then calls `scripts/push` and writes the record with `scripts/record` from the test's and push's output, pushes the branch, opens the pull request, watches `check`, wakes its critic to review, merges, and watches `promote`. The same states as a tool build. -- **flyloft needs from this repository:** `scripts/record` and `scripts/check.py` runnable with the Python standard library alone on its host (its tier one carries copies of them), `scripts/push` printing the manifest and a `commit:` line, and the five frontmatter fields, `metadata.tools` among them. +- **Building.** A campaign's implementer gets a worktree of this repository, writes a dataset, and calls `submit_dataset`; flyloft cooks the recipe as its skill says (one job, or several), then calls `scripts/push` and writes the record with `scripts/record` from the test's and push's output, pushes the branch, opens the pull request, watches `check`, wakes its critic to review, merges, and runs `scripts/promote` with the token it holds for the campaign's owner, reading its last line. The same states as a tool build, with promoting done by flyloft rather than watched. +- **flyloft needs from this repository:** `scripts/record` and `scripts/check.py` runnable with the Python standard library alone on its host (its tier one carries copies of them), `scripts/push` printing the manifest and a `commit:` line, `scripts/promote` saying on its last line what it did, and the five frontmatter fields, `metadata.tools` among them. ## 6. Not here, and deferred @@ -110,4 +111,4 @@ Settled in flyloft's [`docs/design/datasets.md`](https://github.com/brineylab/fl ## 7. The decisions, for the record -From flyloft issue #174, decided 2026-11-04: D1 the Hub is the store; D2 recipes live here, in a GitHub repository mirroring the toolshed's split; D3 variants are datasets, versions are time; D4 a recipe is one program running in one tool's image (revised 2026-11-05 by the spec above: a recipe names its tools; one job is the default, not the rule); D5 publishing mirrors the toolshed's flow (contributor builds, CI never builds, merge promotes); D6 jobs pull into the cluster's cache; D7 flyloft's part is the catalogue, reading, and `submit_dataset`. The two CoreWeave clusters have GPU partitions only; a no-GPU run there takes a GPU node's spare CPUs (flyloft's [`docs/design/spare-cpus.md`](https://github.com/brineylab/flyloft/blob/main/docs/design/spare-cpus.md)), which is where a dataset build on CoreWeave goes; the lab builds data on the rtx6000 cluster, not on b200. +From flyloft issue #174, decided 2026-11-04: D1 the Hub is the store; D2 recipes live here, in a GitHub repository mirroring the toolshed's split; D3 variants are datasets, versions are time; D4 a recipe is one program running in one tool's image (revised 2026-11-05 by the spec above: a recipe names its tools; one job is the default, not the rule); D5 publishing mirrors the toolshed's flow (contributor builds, CI never builds, merge promotes; revised 2026-11-06: the merger runs promote, since CI holds no Hub token); D6 jobs pull into the cluster's cache; D7 flyloft's part is the catalogue, reading, and `submit_dataset`. The two CoreWeave clusters have GPU partitions only; a no-GPU run there takes a GPU node's spare CPUs (flyloft's [`docs/design/spare-cpus.md`](https://github.com/brineylab/flyloft/blob/main/docs/design/spare-cpus.md)), which is where a dataset build on CoreWeave goes; the lab builds data on the rtx6000 cluster, not on b200. diff --git a/docs/plans/datasets-a.md b/docs/plans/datasets-a.md index be99cc1..8f93bd8 100644 --- a/docs/plans/datasets-a.md +++ b/docs/plans/datasets-a.md @@ -2,7 +2,7 @@ > **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. -> Tasks 5–7 are replaced by `datasets-a2.md`; Task 8 waits for flyloft's plan B. +> Tasks 5–7 are replaced by `datasets-a2.md`; Task 8 waits for flyloft's plan B. DESIGN's 2026-11-06 revision supersedes this plan's `promote` job and its `HF_TOKEN` repository secret: whoever merges runs `scripts/promote` with their own token, and CI holds none. **Goal:** The lab's datasets repository exists as `brineylab/datasets`, with its check script, the contributor's build, test, push, record, and promote scripts, a workflow that checks a pull request's record against the Hub and promotes on merge, a recipe image in the toolshed, and two datasets published to the Hub under `brineylab`: `hello`, a synthetic one of kilobytes that exercises the whole path, and `oas-paired-human`, the real one. diff --git a/scripts/promote b/scripts/promote index e21602a..92bde1d 100755 --- a/scripts/promote +++ b/scripts/promote @@ -1,10 +1,14 @@ #!/usr/bin/env -S python3 -I """scripts/promote : tag the commit results//.txt names as v on -/, then delete the candidate branch. A tag is never moved: one already naming another commit -exits 1; one already naming the record's commit passes, so a promote run again after it succeeded passes. -The last line of stdout says what happened. The token is HF_TOKEN's. CI runs it on a push to main. +/, then delete the candidate branch. Whoever merges the dataset's pull request runs it after +pulling main, with their own HF_TOKEN; a record that is not on origin/main as it is here is refused, so only +a reviewed, merged record is published (a guard against mistakes, not misuse: an org token can tag +the Hub directly). Run it from a full clone: a worktree's .git points outside a mounted directory. A tag is never moved: one already naming another commit exits 1; +one already naming the record's commit passes, so a promote run again after it succeeded passes. The last +line of stdout says what happened. """ import os +import subprocess import sys from pathlib import Path @@ -19,7 +23,14 @@ def main() -> int: print("usage: scripts/promote ", file=sys.stderr) return 2 name, version = sys.argv[1:] - rec = check.parse_record(check.ROOT / "results" / name / f"{version}.txt") + rel = f"results/{name}/{version}.txt" + merged = subprocess.run(["git", "show", f"origin/main:{rel}"], cwd=check.ROOT, capture_output=True) + here = check.ROOT / rel + if merged.returncode or not here.is_file() or merged.stdout != here.read_bytes(): + why = f" ({merged.stderr.decode(errors='replace').strip()})" if merged.returncode else "" + print(f"{rel} is not on origin/main{why}; merge its pull request and pull first, in a full clone", file=sys.stderr) + return 1 + rec = check.parse_record(check.ROOT / rel) repo, tag, commit = f"{check.hub_org()}/{name}", f"v{version}", rec["commit"] api = HfApi(token=os.environ["HF_TOKEN"]) have = {t.name: t.target_commit for t in api.list_repo_refs(repo, repo_type="dataset").tags} diff --git a/tests/test_push_promote.py b/tests/test_push_promote.py index 1a4b406..a5ccdbb 100644 --- a/tests/test_push_promote.py +++ b/tests/test_push_promote.py @@ -97,10 +97,35 @@ def test_the_card_carries_the_license_and_the_skills_body(self): self.assertTrue(card.startswith("---\nlicense: mit\npretty_name: tiny\n---\n")); self.assertIn("## Files", card) self.assertNotIn("hello@1.0.1", card) # the skill's frontmatter stays in the skill - def record(self, commit): + def record(self, commit, merged=True): + """The record, committed; merged, origin/main holds that commit, as after the pull request's merge.""" (self.repo / "results/tiny").mkdir(parents=True) (self.repo / "results/tiny/1.0.0.txt").write_text(f"---\ndataset: tiny\nversion: 1.0.0\nhub: brineylab/tiny\nbranch: {self.branch}\n" f"commit: {commit}\ntree: {self.tree}\nexit: 0\n---\ncommit: {commit}\n") + git(self.repo, "add", "results"); git(self.repo, "commit", "-qm", "record") + if merged: + git(self.repo, "update-ref", "refs/remotes/origin/main", "HEAD") + + def test_promote_refuses_a_record_that_is_not_on_main(self): + self.hub.write_text(json.dumps({"branches": {self.branch: "c" * 40}})) + git(self.repo, "update-ref", "refs/remotes/origin/main", "HEAD") + self.record("c" * 40, merged=False) + r = self.run_script("scripts/promote", "tiny", "1.0.0") + self.assertEqual(r.returncode, 1); self.assertIn("results/tiny/1.0.0.txt is not on origin/main (", r.stderr); self.assertIn("merge its pull request and pull first", r.stderr) + self.assertEqual(self.state().get("calls", []), []) # nothing reached the Hub + + def test_promote_says_why_git_cannot_find_the_record_on_main(self): + self.record("c" * 40) + shutil.rmtree(self.repo / ".git") # as in a worktree whose main clone is not mounted + r = self.run_script("scripts/promote", "tiny", "1.0.0") + self.assertEqual(r.returncode, 1); self.assertIn("results/tiny/1.0.0.txt is not on origin/main (", r.stderr) + self.assertIn("not a git repository", r.stderr); self.assertNotIn("Traceback", r.stderr) + + def test_promote_refuses_a_merged_record_missing_here(self): + self.record("c" * 40) + (self.repo / "results/tiny/1.0.0.txt").unlink() + r = self.run_script("scripts/promote", "tiny", "1.0.0") + self.assertEqual(r.returncode, 1); self.assertIn("is not on origin/main", r.stderr); self.assertNotIn("Traceback", r.stderr) def test_promote_tags_the_records_commit_and_deletes_the_branch(self): self.hub.write_text(json.dumps({"branches": {"main": "m" * 40, self.branch: "c" * 40}}))