From a9d965a06ac672a9f5a207dd99506b2e5e0b3866 Mon Sep 17 00:00:00 2001 From: Bryan Briney Date: Fri, 20 Nov 2026 09:51:10 -0800 Subject: [PATCH] Optional metadata.count and metadata.unit in each dataset's skill flyloft's Datasets page shows a dataset's entry count when its skill names one. Two optional frontmatter fields, together: count (digits only) and unit (what an entry is). - check.py: count is digits, and count and unit come together. - changed-datasets: a change only to count/unit lines is not a change, so adding them to a published version needs no new version. - A unit test runs the rules on every dataset, so a count-only edit is still checked. - Every existing skill gets its count, from its skill or record. - CONTRIBUTING, AGENTS, DESIGN: the fields and the version exception. Closes #20 Co-Authored-By: Claude Opus 5.5 --- AGENTS.md | 4 ++-- CONTRIBUTING.md | 8 +++++--- datasets/afdb-rep/SKILL.md | 2 ++ datasets/afdb-swissprot/SKILL.md | 2 ++ .../SKILL.md | 2 ++ datasets/hello/SKILL.md | 2 ++ datasets/jaffe-paired/SKILL.md | 2 ++ datasets/oplm-antibody-sequences/SKILL.md | 2 ++ datasets/uniref100/SKILL.md | 2 ++ datasets/uniref50/SKILL.md | 2 ++ datasets/uniref90/SKILL.md | 2 ++ docs/DESIGN.md | 4 ++-- scripts/changed-datasets | 4 +++- scripts/check.py | 5 +++++ tests/test_changed_datasets.py | 14 +++++++++++++ tests/test_check.py | 20 ++++++++++++++++++- 16 files changed, 68 insertions(+), 9 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index fd65249..7d54df7 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -45,8 +45,8 @@ datasets//recipe/build && datasets//recipe/test && pytho - **The recipe's tools are toolshed tools**, named in the skill's `metadata.tools` as one or more `name@version` separated by single spaces; the one that pushes is built without `HF_HUB_OFFLINE` and holds `huggingface_hub` and python3 (the shed's `datascience`). A recipe never pip-installs. - **The identity is the recipe's inputs.** The skill states the source, its pull date, every parameter, and the tools' versions in prose; the record carries them. A change to any of them is a new version; a change that makes the result incomparable (a threshold, a species) is a new dataset. - **Variants are datasets, versions are time.** Copy the sibling's directory, change what differs, give it its own name. -- **Any change under `datasets//` bumps `metadata.version`**, prose included. A published tag is never moved; readers always name a version. -- **The skill's frontmatter is `name`, `description`, `metadata.version`, `metadata.tools`, `metadata.license`.** Everything else is prose. No declared schemas. +- **Any change under `datasets//` bumps `metadata.version`**, prose included, except `metadata.count`/`metadata.unit` alone (`scripts/changed-datasets` ignores them). A published tag is never moved; readers always name a version. +- **The skill's frontmatter is `name`, `description`, `metadata.version`, `metadata.tools`, `metadata.license`, and optionally `metadata.count` (digits) with `metadata.unit`.** Everything else is prose. No declared schemas. - **The test judges the real output**, in seconds, from footers and samples: the files, the schema, row counts above the recipe's floor, the recipe's own checks. It is not the recipe on a slice. - **Never a credential in the repository**: `scripts/push` reads `HF_TOKEN` and never prints it; `scripts/push`, `scripts/record`, and `check` refuse what looks like one. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 20703f4..d106f0f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -23,7 +23,7 @@ The record, `results//.txt`, is written by `scripts/record` after ## The skill -Frontmatter has five fields a machine reads: +Frontmatter has five fields a machine reads, and two optional ones: ```markdown --- @@ -33,10 +33,12 @@ metadata: version: 1.0.0 tools: datascience@1.0.0 license: + count: + unit: --- ``` -`name` is the directory's and the Hub repository's, `description` is the catalogue line, `metadata.version` is the Hub tag (`v` prefixed), `metadata.tools` is one or more `name@version` separated by single spaces (a string: the skill format's `metadata` maps strings to strings), and `metadata.license` goes on the card. Then the body, in this order: `Hub:` (the repository and tag), `## Files` (each file, its rows, its columns), `## Reading it` (the `snapshot_download` line at this version), `## Recipe` (the source, its pull date, every parameter, the tools, what the test checks), `## Compute` (CPUs, memory, hours, disk, measured), `## Gotchas`. The body is also the Hub's card. No declared schemas: prose. +`name` is the directory's and the Hub repository's, `description` is the catalogue line, `metadata.version` is the Hub tag (`v` prefixed), `metadata.tools` is one or more `name@version` separated by single spaces (a string: the skill format's `metadata` maps strings to strings), and `metadata.license` goes on the card. `metadata.count` and `metadata.unit`, optional and together, are the catalogue's size line (flyloft's board shows "542k structures"): `count` is the version's whole number of entries in digits, no commas, across every file, and `unit` is what one entry is. Write them when you record a version; a version without them still works. Then the body, in this order: `Hub:` (the repository and tag), `## Files` (each file, its rows, its columns), `## Reading it` (the `snapshot_download` line at this version), `## Recipe` (the source, its pull date, every parameter, the tools, what the test checks), `## Compute` (CPUs, memory, hours, disk, measured), `## Gotchas`. The body is also the Hub's card. No declared schemas: prose. ## The recipe's shape @@ -96,7 +98,7 @@ A change after cooking changes the tree, so cook, push, and record again. A pros ## Versions -Any change under `datasets//` bumps `metadata.version`, prose included. A published tag is never moved, and `check` refuses a pull request whose tag is already on the Hub. Readers always name a version, never `main`, which is a copy of the version last promoted, for the Hub's page. +Any change under `datasets//` bumps `metadata.version`, prose included, except adding or correcting `metadata.count` and `metadata.unit`: they describe the published version, so CI does not count that as a change. A published tag is never moved, and `check` refuses a pull request whose tag is already on the Hub. Readers always name a version, never `main`, which is a copy of the version last promoted, for the Hub's page. ## The pull request diff --git a/datasets/afdb-rep/SKILL.md b/datasets/afdb-rep/SKILL.md index 64f444e..8802cba 100644 --- a/datasets/afdb-rep/SKILL.md +++ b/datasets/afdb-rep/SKILL.md @@ -5,6 +5,8 @@ metadata: version: 1.0.0 tools: datascience@1.2.0 license: cc-by-4.0 + count: 2278845 + unit: structures --- # afdb-rep diff --git a/datasets/afdb-swissprot/SKILL.md b/datasets/afdb-swissprot/SKILL.md index 4e99bf6..3d46712 100644 --- a/datasets/afdb-swissprot/SKILL.md +++ b/datasets/afdb-swissprot/SKILL.md @@ -5,6 +5,8 @@ metadata: version: 1.0.0 tools: datascience@1.2.0 license: cc-by-4.0 + count: 542378 + unit: structures --- # afdb-swissprot diff --git a/datasets/antigen-specific-antibody-database/SKILL.md b/datasets/antigen-specific-antibody-database/SKILL.md index cd77058..475c534 100644 --- a/datasets/antigen-specific-antibody-database/SKILL.md +++ b/datasets/antigen-specific-antibody-database/SKILL.md @@ -5,6 +5,8 @@ metadata: version: 1.0.1 tools: datascience@1.2.0 license: cc-by-nc-4.0 + count: 1177819 + unit: records --- # Antigen Specific Antibody Database diff --git a/datasets/hello/SKILL.md b/datasets/hello/SKILL.md index f03e1d3..0df3dda 100644 --- a/datasets/hello/SKILL.md +++ b/datasets/hello/SKILL.md @@ -5,6 +5,8 @@ metadata: version: 1.0.0 tools: datascience@1.0.0 license: mit + count: 300 + unit: pairs --- # hello diff --git a/datasets/jaffe-paired/SKILL.md b/datasets/jaffe-paired/SKILL.md index fa438a6..9084bd5 100644 --- a/datasets/jaffe-paired/SKILL.md +++ b/datasets/jaffe-paired/SKILL.md @@ -5,6 +5,8 @@ metadata: version: 1.0.0 tools: datascience@1.2.0 license: mit + count: 1186496 + unit: pairs --- # jaffe-paired diff --git a/datasets/oplm-antibody-sequences/SKILL.md b/datasets/oplm-antibody-sequences/SKILL.md index d0ae198..09ecec1 100644 --- a/datasets/oplm-antibody-sequences/SKILL.md +++ b/datasets/oplm-antibody-sequences/SKILL.md @@ -5,6 +5,8 @@ metadata: version: 2.0.0 tools: datascience@1.2.0 license: cc-by-4.0 + count: 299748717 + unit: sequences --- # oplm-antibody-sequences diff --git a/datasets/uniref100/SKILL.md b/datasets/uniref100/SKILL.md index b2cecfd..b707acc 100644 --- a/datasets/uniref100/SKILL.md +++ b/datasets/uniref100/SKILL.md @@ -5,6 +5,8 @@ metadata: version: 1.0.0 tools: datascience@1.2.0 license: cc-by-4.0 + count: 227086028 + unit: sequences --- # uniref100 diff --git a/datasets/uniref50/SKILL.md b/datasets/uniref50/SKILL.md index 9ecf60c..6e27c4c 100644 --- a/datasets/uniref50/SKILL.md +++ b/datasets/uniref50/SKILL.md @@ -5,6 +5,8 @@ metadata: version: 1.0.0 tools: datascience@1.2.0 license: cc-by-4.0 + count: 41243524 + unit: sequences --- # uniref50 diff --git a/datasets/uniref90/SKILL.md b/datasets/uniref90/SKILL.md index aa65544..c7d4b46 100644 --- a/datasets/uniref90/SKILL.md +++ b/datasets/uniref90/SKILL.md @@ -5,6 +5,8 @@ metadata: version: 1.0.0 tools: datascience@1.2.0 license: cc-by-4.0 + count: 125724408 + unit: sequences --- # uniref90 diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 4729b55..fb725aa 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -45,7 +45,7 @@ What the build takes: CPUs, memory, hours, disk. ## Gotchas ``` -The frontmatter has five fields a machine reads: `name` (the directory's name and the Hub repository's), `description` (the catalogue line), `metadata.version` (the Hub tag, `v` prefixed), `metadata.tools` (the toolshed tools the recipe runs in, one or more `name@version` separated by single spaces, as a string: the format's `metadata` maps strings to strings), and `metadata.license` (the card's). Everything else is prose for the agent. The Hub's card is generated from it (§3). +The frontmatter has five fields a machine reads: `name` (the directory's name and the Hub repository's), `description` (the catalogue line), `metadata.version` (the Hub tag, `v` prefixed), `metadata.tools` (the toolshed tools the recipe runs in, one or more `name@version` separated by single spaces, as a string: the format's `metadata` maps strings to strings), and `metadata.license` (the card's); optionally `metadata.count` (the version's entries, digits only) with `metadata.unit` (what an entry is), for flyloft's catalogue. Everything else is prose for the agent. The Hub's card is generated from it (§3). ## 3. The repository @@ -67,7 +67,7 @@ scripts/promote run by whoever merges: tag the merged record's co .github/workflows/datasets.yml check ``` -**Versions.** `metadata.version` is the Hub tag, `v` on `/`, with the org from `hub.toml`, so a fork with its own org is its own catalogue and store with one edit. A published tag is never moved: any change under `datasets//` bumps the version, prose included, and CI refuses a pull request whose tag is already on the Hub. Agents and flyloft always name a version, never `main`. `main` is for people browsing the Hub: a copy of the version last promoted, one commit per promote with the version's files, not the tagged commit itself, since the Hub moves a branch only by a commit. +**Versions.** `metadata.version` is the Hub tag, `v` on `/`, with the org from `hub.toml`, so a fork with its own org is its own catalogue and store with one edit. A published tag is never moved: any change under `datasets//` bumps the version, prose included (but not one only to `metadata.count`/`metadata.unit`, which describe the published version), and CI refuses a pull request whose tag is already on the Hub. Agents and flyloft always name a version, never `main`. `main` is for people browsing the Hub: a copy of the version last promoted, one commit per promote with the version's files, not the tagged commit itself, since the Hub moves a branch only by a commit. **The Hub** is the store (D1). One dataset repository per dataset, public, created by the first push when it does not exist. The Hub gives what the toolshed's registry gives and more: every commit immutable, a tag per version, every file's sha256, hosting, a dataset viewer, and a download-and-cache library every agent already knows. A dataset built from public data is public from the start, which is also where reviewers look when the lab publishes. A dataset the lab could not make public is the exception, handled when it arrives (a private repository and a read token), not designed for now. A few hundred GB per dataset, the largest foreseen, is within the Hub's guidance (no per-repository cap for public repositories; files under 200 GB, parquet preferred, under 100k files). Free organizations get best-effort public storage beyond the first gigabytes, so before plan A's large dataset the lab confirms its Hub plan holds what it will publish. diff --git a/scripts/changed-datasets b/scripts/changed-datasets index a97d4bd..59fb57a 100755 --- a/scripts/changed-datasets +++ b/scripts/changed-datasets @@ -1,7 +1,9 @@ #!/usr/bin/env bash # scripts/changed-datasets : the datasets whose directories changed and still exist, one per line. +# A change only to a skill's metadata.count or metadata.unit lines is not one: they describe the published +# version, so adding or correcting them needs no new version. set -euo pipefail cd "$(dirname "$0")/.." -git diff --name-only "$@" -- datasets/ | cut -d/ -f2 | sort -u | while read -r d; do +git diff --name-only -I '^ (count|unit):' "$@" -- datasets/ | cut -d/ -f2 | sort -u | while read -r d; do if [[ -d "datasets/$d" ]]; then echo "$d"; fi done diff --git a/scripts/check.py b/scripts/check.py index 930eb7c..72e76a7 100644 --- a/scripts/check.py +++ b/scripts/check.py @@ -161,6 +161,11 @@ def rules(dataset_dir: Path) -> list[str]: reasons.append(f"tools {tools!r} is not one or more toolshed tools' name@MAJOR.MINOR.PATCH separated by single spaces") if not meta.get("license"): reasons.append("license is empty") + if "count" in meta or "unit" in meta: # optional, together: the catalogue's "542k structures" + if not meta.get("count", "").isdigit(): + reasons.append(f"count {meta.get('count')!r} is not a whole number, digits only") + if not meta.get("unit"): + reasons.append("unit is empty; count and unit go together") for script in ("recipe/build", "recipe/test"): p = dataset_dir / script if not p.is_file(): diff --git a/tests/test_changed_datasets.py b/tests/test_changed_datasets.py index b037a54..8c3780e 100644 --- a/tests/test_changed_datasets.py +++ b/tests/test_changed_datasets.py @@ -15,5 +15,19 @@ def test_names_each_changed_dataset_once_and_skips_a_removed_one(self): r = subprocess.run(["bash", str(repo / "scripts/changed-datasets"), "HEAD~1", "HEAD"], capture_output=True, text=True, check=True) self.assertEqual(r.stdout, "a\n") + def test_a_count_or_unit_only_change_is_not_a_change(self): + repo = Path(tempfile.mkdtemp()); self.addCleanup(shutil.rmtree, repo) + (repo / "scripts").mkdir(); shutil.copy(ROOT / "scripts" / "changed-datasets", repo / "scripts") + skill = "---\nname: {}\nmetadata:\n version: 1.0.0\n license: mit\n---\nbody\n" + for d in ("a", "b"): + (repo / "datasets" / d).mkdir(parents=True); (repo / "datasets" / d / "SKILL.md").write_text(skill.format(d)) + git(repo, "init", "-q"); git(repo, "add", "-A"); git(repo, "commit", "-qm", "1") + for d in ("a", "b"): + (repo / "datasets" / d / "SKILL.md").write_text(skill.format(d).replace(" license: mit\n", " license: mit\n count: 300\n unit: pairs\n")) + (repo / "datasets/b/SKILL.md").write_text((repo / "datasets/b/SKILL.md").read_text() + "more prose\n") + git(repo, "add", "-A"); git(repo, "commit", "-qm", "2") + r = subprocess.run(["bash", str(repo / "scripts/changed-datasets"), "HEAD~1", "HEAD"], capture_output=True, text=True, check=True) + self.assertEqual(r.stdout, "b\n") + if __name__ == "__main__": unittest.main() diff --git a/tests/test_check.py b/tests/test_check.py index 08c7126..dac3297 100644 --- a/tests/test_check.py +++ b/tests/test_check.py @@ -1,5 +1,5 @@ import unittest -from helpers import TINY, copy_of, git +from helpers import ROOT, TINY, copy_of, git import check # noqa: E402 ORG = check.hub_org() # hub.toml's organization: a fork with its own is its own store @@ -42,6 +42,24 @@ def test_rules_refuse_a_credential_and_a_params_file_that_does_not_parse(self): (d / "recipe" / "params.toml").write_text("seed = \n") self.assertTrue(any(r.startswith("recipe/params.toml does not parse") for r in check.rules(d))) + def test_count_and_unit_are_optional_together_and_count_is_digits(self): + add = lambda d, lines: (d / "SKILL.md").write_text((d / "SKILL.md").read_text().replace(" license: mit\n", " license: mit\n" + lines)) + with copy_of(TINY) as d: + add(d, " count: 4\n unit: lines\n"); self.assertEqual(check.rules(d), []) + with copy_of(TINY) as d: + add(d, " count: 1,389,546\n unit: pairs\n") + self.assertEqual(check.rules(d), ["count '1,389,546' is not a whole number, digits only"]) + with copy_of(TINY) as d: + add(d, " count: 4\n"); self.assertEqual(check.rules(d), ["unit is empty; count and unit go together"]) + with copy_of(TINY) as d: + add(d, " unit: lines\n"); self.assertEqual(check.rules(d), ["count None is not a whole number, digits only"]) + + def test_every_dataset_in_the_repository_keeps_the_rules(self): + # CI's --hub check runs only on datasets whose change bumps a version; a count/unit-only edit is checked here + for d in sorted((ROOT / "datasets").iterdir()): + with self.subTest(dataset=d.name): + self.assertEqual(check.rules(d), []) + def test_the_minimal_toml_reader_reads_the_recipes_shapes(self): text = 'seed = 1\nidentity = 0.9\nspecies = "human"\nlimit_on = false\nsplit = [0.8, 0.1, 0.1] # fractions\n\n# a comment\n' self.assertEqual(check.read_toml(text, minimal=True),