From 3f31a87401a0661259456ffebddd42df3c4102fa Mon Sep 17 00:00:00 2001 From: Bryan Briney <821533+briney@users.noreply.github.com> Date: Wed, 11 Nov 2026 00:47:36 +0000 Subject: [PATCH 1/2] uniref50: publish v1.0.0 from dev/uniref50@dab7434 The recipe (build, params.toml, test) is taken whole and unchanged from the uniref-datasets campaign's dev/uniref50 at commit dab7434, whose measurement run 640cdc78 built and tested it on bcr: 42 shards, 41,243,524 rows. SKILL.md is written for it, its Compute measured from that build. --- datasets/uniref50/SKILL.md | 48 ++++++++++ datasets/uniref50/recipe/build | 133 +++++++++++++++++++++++++++ datasets/uniref50/recipe/params.toml | 6 ++ datasets/uniref50/recipe/test | 72 +++++++++++++++ 4 files changed, 259 insertions(+) create mode 100644 datasets/uniref50/SKILL.md create mode 100755 datasets/uniref50/recipe/build create mode 100644 datasets/uniref50/recipe/params.toml create mode 100755 datasets/uniref50/recipe/test diff --git a/datasets/uniref50/SKILL.md b/datasets/uniref50/SKILL.md new file mode 100644 index 0000000..9ecf60c --- /dev/null +++ b/datasets/uniref50/SKILL.md @@ -0,0 +1,48 @@ +--- +name: uniref50 +description: "UniRef50 of UniProt release 2026_04 as sharded two-column parquet: one row per entry (a 50%-identity cluster's representative), sequence_id and sequence, in the FASTA's order. Reach for it to pre-train or evaluate a protein language model on UniRef50 at a pinned release." +metadata: + version: 1.0.0 + tools: datascience@1.2.0 + license: cc-by-4.0 +--- +# uniref50 + +Hub: `brineylab/uniref50`, tag `v1.0.0` + +## Files + +`data/00000.parquet` … `data/00041.parquet`: 42 shards holding 41,243,524 rows, one row per UniRef50 entry, in the FASTA's order. Every shard but the last holds exactly 1,000,000 rows (the last 243,524); zstd, row groups of about 100,000 rows. Columns: + +- `sequence_id`: the FASTA header's first token, the entry id (`UniRef50_A0A5A9P0L4`). +- `sequence`: the entry's amino acids, its lines after the header joined. + +## Reading it + +```python +from huggingface_hub import snapshot_download +import polars as pl +path = snapshot_download("brineylab/uniref50", repo_type="dataset", revision="v1.0.0") +shard = pl.read_parquet(f"{path}/data/00000.parquet", columns=["sequence_id", "sequence"]) +``` + +Rows keep the FASTA's order, so a subset of shards is a contiguous slice of the release, not a sample; shuffle yourself for training. + +## Recipe + +**Source:** UniRef release 2026_04 (published 2026-11-04), CC BY 4.0, pulled 2026-11-11: `uniref50.fasta.gz` (9,182,717,121 bytes, MD5 `0dd988dc772143255691f30c0fb9e396`) at https://ftp.uniprot.org/pub/databases/uniprot/uniref/uniref50/uniref50.fasta.gz, holding the release's 41,243,524 entries. The recipe refuses a download whose size or MD5 differs. + +One job in `datascience@1.2.0`: download the FASTA into `/.work/`, resuming a partial one by range; then stream it, never holding the whole file in memory, into `/data/00000.parquet`, `00001.parquet`, … in the FASTA's order, `shard_rows` rows per shard (the last one partial), zstd, row groups of about 100k rows. Stages resume: a rerun keeps the verified FASTA and the shards already written, and a finished build only rereads the footers. + +`params.toml`: `release = "2026_04"`; the file's `url`, `bytes`, and `md5`; `sequences = 41243524`, the release's count; `shard_rows = 1000000`. + +**The test** reads the shards' parquet footers and three sampled row groups, in seconds: the shards are `00000.parquet` onward without a gap; the schema is exactly `sequence_id` and `sequence`, strings; every shard but the last holds exactly 1,000,000 rows; the total is exactly the release's 41,243,524; and in the samples every `sequence_id` starts with `UniRef50_` and every `sequence` is uppercase letters only. + +## Compute + +One job, CPU only: 8 CPUs and 32G asked, on bcr's `cpu` partition. From nothing it takes about 14 min: the download (8.6 GB, one stream from ftp.uniprot.org at about 18 MB/s) 8.5 min; the MD5 check and the 42 shards 5.5 min. Peak memory 5.8 GB. Disk: 6.6 GB of shards and 8.6 GB under `/.work/`, the FASTA all of it, kept so a rerun resumes; the test reads only footers and three row groups, in 3 s. + +## Gotchas + +- The URL serves UniProt's current release only: once 2026_05 is out, the pinned MD5 refuses it, and a rebuild is a new version. +- The FASTA header's other fields (`n=`, `Tax=`, `TaxID=`, `RepID=`) are dropped; cluster sizes and taxonomy are in UniRef's XML, not here. diff --git a/datasets/uniref50/recipe/build b/datasets/uniref50/recipe/build new file mode 100755 index 0000000..77660d5 --- /dev/null +++ b/datasets/uniref50/recipe/build @@ -0,0 +1,133 @@ +#!/usr/bin/env -S python3 -I +"""recipe/build : UniRef50 of UniProt release 2026_04 as sharded two-column parquet. Downloads +uniref50.fasta.gz into /.work/, refusing a size or MD5 that differs from params, then streams it, never +holding the whole file in memory, into /data/00000.parquet, 00001.parquet, ...: the FASTA's order, +shard_rows rows per shard (the last one partial), zstd, row groups of about 100k rows. sequence_id is the +header's first token, sequence the amino acids of the header's lines joined. Stages resume: a rerun keeps +the verified FASTA and the shards already written. Prints the shard and row totals last. +""" +import gzip +import hashlib +import os +import sys +import time +import tomllib +import urllib.request +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + +params = tomllib.loads((Path(__file__).parent / "params.toml").read_text()) +out = Path(sys.argv[1]).resolve() +work, data = out / ".work", out / "data" +SCHEMA = pa.schema([("sequence_id", pa.string()), ("sequence", pa.string())]) + + +def shard_paths() -> list[Path]: + """data/'s shards in order; a gap ends the list, so a partial build resumes at its end.""" + paths, i = [], 0 + while (data / f"{i:05d}.parquet").is_file(): + paths.append(data / f"{i:05d}.parquet") + i += 1 + return paths + + +def rows(paths: list[Path]) -> int: + return sum(pq.ParquetFile(p).metadata.num_rows for p in paths) + + +def download() -> Path: + """The FASTA in .work/, resumed by range from where a partial one stopped and given three attempts at + a connection; refuses a size or MD5 that differs from params.""" + name = params["url"].rsplit("/", 1)[-1] + path, part = work / name, Path(f"{work / name}.part") + if path.is_file(): + return path + for attempt in range(3): + have = part.stat().st_size if part.exists() else 0 + if have >= params["bytes"]: + break + try: + headers = {"User-Agent": "brineylab-datasets"} + if have: + headers["Range"] = f"bytes={have}-" + request = urllib.request.Request(params["url"], headers=headers) + with urllib.request.urlopen(request, timeout=120) as r, open(part, "ab") as f: + if have and r.status != 206: + sys.exit(f"{params['url']} answered {r.status} to a range request at {have} bytes") + while block := r.read(1 << 22): + f.write(block) + break + except OSError as e: + if attempt == 2: + raise + print(f"{name}: {e}; retrying from {part.stat().st_size} bytes", file=sys.stderr) + time.sleep(30) + if part.stat().st_size != params["bytes"]: + sys.exit(f"{name}: {part.stat().st_size} bytes is not {params['bytes']}") + digest = hashlib.md5() + with open(part, "rb") as f: + while block := f.read(1 << 24): + digest.update(block) + if digest.hexdigest() != params["md5"]: + sys.exit(f"{name}: MD5 {digest.hexdigest()} is not {params['md5']}") + part.rename(path) + return path + + +def records(gz: Path): + """(sequence_id, sequence) per FASTA record, in the file's order; the whole file is never held.""" + with gzip.open(gz, "rt", encoding="ascii") as f: + sid, parts = None, [] + for line in f: + if line.startswith(">"): + if sid is not None: + yield sid, "".join(parts) + sid, parts = line[1:].split(None, 1)[0], [] + else: + parts.append(line.rstrip()) + if sid is not None: + yield sid, "".join(parts) + + +def write(i: int, ids: list[str], seqs: list[str]) -> None: + """Shard i whole or not at all: written under .work/ then moved into data/.""" + pq.write_table(pa.table({"sequence_id": ids, "sequence": seqs}, schema=SCHEMA), work / "shard.part", + row_group_size=100_000, compression="zstd") + os.replace(work / "shard.part", data / f"{i:05d}.parquet") + + +def shard(gz: Path, first: int) -> None: + """data/'s shards from first on, in the FASTA's order, skipping the rows of the shards already there.""" + data.mkdir(parents=True, exist_ok=True) + skip = first * params["shard_rows"] + i, ids, seqs = first, [], [] + for sid, seq in records(gz): + if skip: + skip -= 1 + continue + ids.append(sid) + seqs.append(seq) + if len(ids) == params["shard_rows"]: + write(i, ids, seqs) + i, ids, seqs = i + 1, [], [] + if ids: + write(i, ids, seqs) + + +def main() -> None: + if len(sys.argv) != 2: + sys.exit(f"usage: {Path(sys.argv[0]).name} ") + out.mkdir(parents=True, exist_ok=True) + work.mkdir(parents=True, exist_ok=True) + paths = shard_paths() + if rows(paths) != params["sequences"]: + shard(download(), len(paths)) + paths = shard_paths() + print(f"shards: {len(paths)}") + print(f"rows: {rows(paths)}") + + +if __name__ == "__main__": + main() diff --git a/datasets/uniref50/recipe/params.toml b/datasets/uniref50/recipe/params.toml new file mode 100644 index 0000000..2500c72 --- /dev/null +++ b/datasets/uniref50/recipe/params.toml @@ -0,0 +1,6 @@ +release = "2026_04" +url = "https://ftp.uniprot.org/pub/databases/uniprot/uniref/uniref50/uniref50.fasta.gz" +bytes = 9182717121 +md5 = "0dd988dc772143255691f30c0fb9e396" +sequences = 41243524 +shard_rows = 1000000 diff --git a/datasets/uniref50/recipe/test b/datasets/uniref50/recipe/test new file mode 100755 index 0000000..7deb30e --- /dev/null +++ b/datasets/uniref50/recipe/test @@ -0,0 +1,72 @@ +#!/usr/bin/env -S python3 -I +"""recipe/test : exit 0 iff holds a version of uniref50. From the shards' parquet footers and +three sampled row groups, in seconds: the shards are numbered without a gap, the schema is exactly +sequence_id and sequence (strings), every shard but the last is exactly shard_rows rows, the total is +exactly the release's count, and in the samples every sequence_id starts with UniRef50_ and every sequence +is uppercase letters only. One line per failure, naming the check and the first offending row.""" +import re +import sys +import tomllib +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + +params = tomllib.loads((Path(__file__).parent / "params.toml").read_text()) +PREFIX = "UniRef" + Path(__file__).resolve().parent.parent.name.removeprefix("uniref") + "_" +LETTERS = re.compile(r"[A-Z]+") + +out, problems = Path(sys.argv[1]), [] +files = sorted((out / "data").glob("*.parquet")) if (out / "data").is_dir() else [] +if not files: + problems.append("files: data/ holds no shards") +if [p.name for p in files] != [f"{i:05d}.parquet" for i in range(len(files))]: + problems.append("files: data/'s shards are not 00000.parquet onward without a gap") + +total, last, parsed = 0, 0, 0 +for n, p in enumerate(files): + try: + pf = pq.ParquetFile(p) + except Exception as e: + problems.append(f"parses: {p.name} is not a parquet file: {e}") + continue + schema = pf.schema_arrow + if schema.names != ["sequence_id", "sequence"] or any(schema.field(c).type != pa.string() for c in schema.names): + problems.append(f"columns: {p.name} has {schema}, not exactly sequence_id and sequence as strings") + continue + parsed += 1 + shard_rows = pf.metadata.num_rows + total += shard_rows + last = shard_rows + if n + 1 < len(files) and shard_rows != params["shard_rows"]: + problems.append(f"rows: {p.name} holds {shard_rows}, not {params['shard_rows']}") + +if parsed: + if not 0 < last <= params["shard_rows"]: + problems.append(f"rows: the last shard holds {last}, not at most {params['shard_rows']} and at least 1") + if total != params["sequences"]: + problems.append(f"rows: the shards hold {total} in all, not the release's {params['sequences']}") + picks = sorted({0, len(files) // 2, len(files) - 1}) + sampled = 0 + for n in picks: + p = files[n] + try: + pf = pq.ParquetFile(p) + except Exception: + continue + groups = pf.metadata.num_row_groups + g = 0 if n == 0 else groups - 1 if n == len(files) - 1 else groups // 2 + table = pf.read_row_group(g) + ids, seqs = table.column("sequence_id").to_pylist(), table.column("sequence").to_pylist() + sampled += len(ids) + bad = next((s for s in ids if not s.startswith(PREFIX)), None) + if bad is not None: + problems.append(f"sequence_id: {p.name} row group {g} holds {bad!r}, not starting with {PREFIX}") + bad = next((s for s in seqs if s is None or not LETTERS.fullmatch(s)), None) + if bad is not None: + problems.append(f"sequence: {p.name} row group {g} holds {bad!r}, not uppercase letters only") + print(f"sampled: {len(picks)} row groups ({sampled} rows)") + +print(f"data: {len(files)} shards, {total} rows") +print("\n".join(problems) or "ok") +sys.exit(1 if problems else 0) From 78b8da386a99bfab84669025f0ca08f10c0f962c Mon Sep 17 00:00:00 2001 From: Bryan Briney <821533+briney@users.noreply.github.com> Date: Tue, 10 Nov 2026 17:04:14 -0800 Subject: [PATCH 2/2] uniref50 1.0.0's record: its candidate passed on bcr --- results/uniref50/1.0.0.txt | 63 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 results/uniref50/1.0.0.txt diff --git a/results/uniref50/1.0.0.txt b/results/uniref50/1.0.0.txt new file mode 100644 index 0000000..c7d49d1 --- /dev/null +++ b/results/uniref50/1.0.0.txt @@ -0,0 +1,63 @@ +--- +dataset: uniref50 +version: 1.0.0 +hub: brineylab/uniref50 +branch: candidate-d05afc8f0883cb3be26d50e502c3133baea14290 +commit: 6e0c895ffb7ce90066bd6ff37bbd2e5bd541c45b +tree: d05afc8f0883cb3be26d50e502c3133baea14290 +tools: datascience@1.2.0 +runner: sbatch --partition=cpu --cpus-per-task=8 --mem=32G --time=02:00:00 +host: bcr.scripps.edu +started: 2026-11-11T00:48:22Z +exit: 0 +--- +pyxis: importing docker image: ghcr.io#brineylab/toolshed/datascience:1.2.0 +pyxis: imported docker image: ghcr.io#brineylab/toolshed/datascience:1.2.0 +shards: 42 +rows: 41243524 +sampled: 3 row groups (243524 rows) +data: 42 shards, 41243524 rows +ok +file data/00000.parquet 973596124 d180ecffa0445617c5e32bbec60012dd8fdc434708d61dcbb1bc30a0578d15f6 +file data/00001.parquet 543052263 b6211498b6fb068bf2cf4d2f90b8855e3427ebed1ef77f052b30daf446b1902a +file data/00002.parquet 435508552 1941967a5647574289a0311ff2eda76dc70a92c6412a2af41a049f3a893e9160 +file data/00003.parquet 372961498 a2eb612fe1959836fb334dfe5651e0ffd1139141dba1efe2af9dd7fcc8eb148f +file data/00004.parquet 329624313 83a098c2bee3094f5b6af992015ea60cbc911c4bc2cc103cbd9c37e2fca01f08 +file data/00005.parquet 298046315 1e05ccad6a837aebaec553319f5f2df65fb97f08cca91bb2d0849115eef4b07c +file data/00006.parquet 272846137 f6fdca3b1bc3f558f00941f67d785e6c5f46585a80d69233984bacce7fd10080 +file data/00007.parquet 251989069 69f6517a2360905b898f05e73b944092176d6fa9429554bc3f7569b94a2e35eb +file data/00008.parquet 234536251 833947e76cfbd7a0a099e244bcfeb439e6b069e6f99bceaff6c11adbc3ac0392 +file data/00009.parquet 219564969 b6e14d24322ba5f05a6833d2de1c8252179c1b0ffaeb24027d9a9d1a2de95ce9 +file data/00010.parquet 206457030 7554f805e1c54deb54cac3e691692a3992fece4c8d393552759e54f5bec660f6 +file data/00011.parquet 194247728 d8bea035173379a35e644017914b914bc19bd7215ae2dc81babfce868ce594d3 +file data/00012.parquet 183061907 d621e5b32d0ec43385ad3ff44dcd15349c8f30dc8df002f4daec7cf62cd01386 +file data/00013.parquet 173608242 49b3c9a73d5867952d2ac5cd375c4c3d46ab0e3bd6d44c58343a0773a6807738 +file data/00014.parquet 164518290 64f0962def3708fa22badbac692d7bdfafad86c46fb8832a4b6eef8b06a760b9 +file data/00015.parquet 155804731 de320f48de6711ccd7d4336abbe9064b8c276fa658fd5e94b56229449949e9bd +file data/00016.parquet 147628398 89d8bf7b31bedaf69ab496e2376233353233900f8e3aee839188cecdec1e4058 +file data/00017.parquet 139674493 35638a733df90433462e456ca8daffe323a9ee937c5ace2aea79cb9ad2f6e18c +file data/00018.parquet 132367938 9185ce5b5b99d6f3161cc07714181b64eb0c5d34569806914572662f09027882 +file data/00019.parquet 125517586 18d3a16c6a2d9e986f9a092ecb088c61184e09e7a098aae306b36e28e983f1e5 +file data/00020.parquet 119068558 29af1afdbacc388c4d4aae0c60a828d1a6bb79b44bcad25b640bb8eb52abd0c1 +file data/00021.parquet 113065952 966680f279b0808e6a015c2798aeef1b52b34c976441eb9457ae77b3242cb631 +file data/00022.parquet 107186516 1681e324416c03dd9723784053652544e17a8f606ed9e8c792433d0158b1672f +file data/00023.parquet 101744983 1072eeeaa5ede85ee3d42f27b5f6fcc747caf9fe73c5ef2b5e1bc71d5b3a78a7 +file data/00024.parquet 96596892 e647de97226ca851d910954f55b33410de6eead141017e2b251cc9c70a08c9fd +file data/00025.parquet 91667363 c0100aaf280c989bd73cdf5ad37465e226c7eb28e74741c92e1f5a215249456b +file data/00026.parquet 87142157 3d47c090c2cce6a2d02f3671e90e23808d89282c86b039ad0f42cdf97cb04ee5 +file data/00027.parquet 82715909 1b8c259c65c21fbef81a4b6797da0aa05aca5393b7e9016196cc1f9ac6635f64 +file data/00028.parquet 78395831 a8f971e79a94a43c0782e40f0573e404ae1b535641d93d0ef7fa2fd37e0305da +file data/00029.parquet 73885536 60fe74a7da944fe999b73dc41f2a2aed47dbdb96f723b56f9c9d22ed0f5beebd +file data/00030.parquet 69813445 157da9e231ab121ecb39baa4ee87476da19a7ac3d8490619279e6abf22470460 +file data/00031.parquet 66028823 8fb93a971892837e4141be09b06f166c8315713aefcd14f96799a36d9b4f4283 +file data/00032.parquet 62424901 2ea0a686f226d53e2f03cfa8525581de41d5c475fa95939637ebd2e9cac6cc0c +file data/00033.parquet 58798469 4a0261d279de0b7a85f52ab09d1b3cdd0025c04312b6d3c1fe4707125db56aba +file data/00034.parquet 55178397 4aeda63072f86f624464847eb68001826dfdea419a4a9e4e7b4a92e0677ab9ef +file data/00035.parquet 51593490 a1e27a7502e4081cd7ef7872573bd05aec511919234fee24b56bcc073b34e593 +file data/00036.parquet 48201484 cfa796322c561a0680a172cac9a3cc2397695cd392db6181360f51156edd749b +file data/00037.parquet 44904027 b9177808e354d59456f190793b001375cbaa04ded9c1ec521f61ac98139601a5 +file data/00038.parquet 41566833 20962028a9e6a820ce85b7128b5e4d3daec1f510d83b429321c2424c12a7c8ba +file data/00039.parquet 37297181 ac656f9574940538555712dc2bbc73f4eca9e9b8d8a23d43aae0f785458c444c +file data/00040.parquet 30929048 d423701c3ab3f707662c991e79abeb3b0ad934daf1e90c2ffa8c15d6d606c0c0 +file data/00041.parquet 5605327 26c649e0678859dcf43f2d20ad3f5891b6f1b4a69244327f723eea3eb2109874 +commit: 6e0c895ffb7ce90066bd6ff37bbd2e5bd541c45b