From e2fde6c2c6031e691de003c6f2ec09b8d024cf7a Mon Sep 17 00:00:00 2001 From: Bryan Briney <821533+briney@users.noreply.github.com> Date: Wed, 11 Nov 2026 01:11:38 +0000 Subject: [PATCH 1/6] Stage uniref90's and uniref100's recipes, copies of the published uniref50's --- datasets/uniref100/recipe/build | 133 ++++++++++++++++++++++++++ datasets/uniref100/recipe/params.toml | 6 ++ datasets/uniref100/recipe/test | 72 ++++++++++++++ datasets/uniref90/recipe/build | 133 ++++++++++++++++++++++++++ datasets/uniref90/recipe/params.toml | 6 ++ datasets/uniref90/recipe/test | 72 ++++++++++++++ 6 files changed, 422 insertions(+) create mode 100755 datasets/uniref100/recipe/build create mode 100644 datasets/uniref100/recipe/params.toml create mode 100755 datasets/uniref100/recipe/test create mode 100755 datasets/uniref90/recipe/build create mode 100644 datasets/uniref90/recipe/params.toml create mode 100755 datasets/uniref90/recipe/test diff --git a/datasets/uniref100/recipe/build b/datasets/uniref100/recipe/build new file mode 100755 index 0000000..c9713b4 --- /dev/null +++ b/datasets/uniref100/recipe/build @@ -0,0 +1,133 @@ +#!/usr/bin/env -S python3 -I +"""recipe/build : UniRef100 of UniProt release 2026_04 as sharded two-column parquet. Downloads +uniref100.fasta.gz into /.work/, refusing a size or MD5 that differs from params, then streams it, never +holding the whole file in memory, into /data/00000.parquet, 00001.parquet, ...: the FASTA's order, +shard_rows rows per shard (the last one partial), zstd, row groups of about 100k rows. sequence_id is the +header's first token, sequence the amino acids of the header's lines joined. Stages resume: a rerun keeps +the verified FASTA and the shards already written. Prints the shard and row totals last. +""" +import gzip +import hashlib +import os +import sys +import time +import tomllib +import urllib.request +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + +params = tomllib.loads((Path(__file__).parent / "params.toml").read_text()) +out = Path(sys.argv[1]).resolve() +work, data = out / ".work", out / "data" +SCHEMA = pa.schema([("sequence_id", pa.string()), ("sequence", pa.string())]) + + +def shard_paths() -> list[Path]: + """data/'s shards in order; a gap ends the list, so a partial build resumes at its end.""" + paths, i = [], 0 + while (data / f"{i:05d}.parquet").is_file(): + paths.append(data / f"{i:05d}.parquet") + i += 1 + return paths + + +def rows(paths: list[Path]) -> int: + return sum(pq.ParquetFile(p).metadata.num_rows for p in paths) + + +def download() -> Path: + """The FASTA in .work/, resumed by range from where a partial one stopped and given three attempts at + a connection; refuses a size or MD5 that differs from params.""" + name = params["url"].rsplit("/", 1)[-1] + path, part = work / name, Path(f"{work / name}.part") + if path.is_file(): + return path + for attempt in range(3): + have = part.stat().st_size if part.exists() else 0 + if have >= params["bytes"]: + break + try: + headers = {"User-Agent": "brineylab-datasets"} + if have: + headers["Range"] = f"bytes={have}-" + request = urllib.request.Request(params["url"], headers=headers) + with urllib.request.urlopen(request, timeout=120) as r, open(part, "ab") as f: + if have and r.status != 206: + sys.exit(f"{params['url']} answered {r.status} to a range request at {have} bytes") + while block := r.read(1 << 22): + f.write(block) + break + except OSError as e: + if attempt == 2: + raise + print(f"{name}: {e}; retrying from {part.stat().st_size} bytes", file=sys.stderr) + time.sleep(30) + if part.stat().st_size != params["bytes"]: + sys.exit(f"{name}: {part.stat().st_size} bytes is not {params['bytes']}") + digest = hashlib.md5() + with open(part, "rb") as f: + while block := f.read(1 << 24): + digest.update(block) + if digest.hexdigest() != params["md5"]: + sys.exit(f"{name}: MD5 {digest.hexdigest()} is not {params['md5']}") + part.rename(path) + return path + + +def records(gz: Path): + """(sequence_id, sequence) per FASTA record, in the file's order; the whole file is never held.""" + with gzip.open(gz, "rt", encoding="ascii") as f: + sid, parts = None, [] + for line in f: + if line.startswith(">"): + if sid is not None: + yield sid, "".join(parts) + sid, parts = line[1:].split(None, 1)[0], [] + else: + parts.append(line.rstrip()) + if sid is not None: + yield sid, "".join(parts) + + +def write(i: int, ids: list[str], seqs: list[str]) -> None: + """Shard i whole or not at all: written under .work/ then moved into data/.""" + pq.write_table(pa.table({"sequence_id": ids, "sequence": seqs}, schema=SCHEMA), work / "shard.part", + row_group_size=100_000, compression="zstd") + os.replace(work / "shard.part", data / f"{i:05d}.parquet") + + +def shard(gz: Path, first: int) -> None: + """data/'s shards from first on, in the FASTA's order, skipping the rows of the shards already there.""" + data.mkdir(parents=True, exist_ok=True) + skip = first * params["shard_rows"] + i, ids, seqs = first, [], [] + for sid, seq in records(gz): + if skip: + skip -= 1 + continue + ids.append(sid) + seqs.append(seq) + if len(ids) == params["shard_rows"]: + write(i, ids, seqs) + i, ids, seqs = i + 1, [], [] + if ids: + write(i, ids, seqs) + + +def main() -> None: + if len(sys.argv) != 2: + sys.exit(f"usage: {Path(sys.argv[0]).name} ") + out.mkdir(parents=True, exist_ok=True) + work.mkdir(parents=True, exist_ok=True) + paths = shard_paths() + if rows(paths) != params["sequences"]: + shard(download(), len(paths)) + paths = shard_paths() + print(f"shards: {len(paths)}") + print(f"rows: {rows(paths)}") + + +if __name__ == "__main__": + main() diff --git a/datasets/uniref100/recipe/params.toml b/datasets/uniref100/recipe/params.toml new file mode 100644 index 0000000..9ab8b30 --- /dev/null +++ b/datasets/uniref100/recipe/params.toml @@ -0,0 +1,6 @@ +release = "2026_04" +url = "https://ftp.uniprot.org/pub/databases/uniprot/uniref/uniref100/uniref100.fasta.gz" +bytes = 63065088280 +md5 = "923e539321e9bfe6a9b74d343284d600" +sequences = 227086028 +shard_rows = 1000000 diff --git a/datasets/uniref100/recipe/test b/datasets/uniref100/recipe/test new file mode 100755 index 0000000..3ab3032 --- /dev/null +++ b/datasets/uniref100/recipe/test @@ -0,0 +1,72 @@ +#!/usr/bin/env -S python3 -I +"""recipe/test : exit 0 iff holds a version of uniref100. From the shards' parquet footers and +three sampled row groups, in seconds: the shards are numbered without a gap, the schema is exactly +sequence_id and sequence (strings), every shard but the last is exactly shard_rows rows, the total is +exactly the release's count, and in the samples every sequence_id starts with UniRef100_ and every sequence +is uppercase letters only. One line per failure, naming the check and the first offending row.""" +import re +import sys +import tomllib +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + +params = tomllib.loads((Path(__file__).parent / "params.toml").read_text()) +PREFIX = "UniRef" + Path(__file__).resolve().parent.parent.name.removeprefix("uniref") + "_" +LETTERS = re.compile(r"[A-Z]+") + +out, problems = Path(sys.argv[1]), [] +files = sorted((out / "data").glob("*.parquet")) if (out / "data").is_dir() else [] +if not files: + problems.append("files: data/ holds no shards") +if [p.name for p in files] != [f"{i:05d}.parquet" for i in range(len(files))]: + problems.append("files: data/'s shards are not 00000.parquet onward without a gap") + +total, last, parsed = 0, 0, 0 +for n, p in enumerate(files): + try: + pf = pq.ParquetFile(p) + except Exception as e: + problems.append(f"parses: {p.name} is not a parquet file: {e}") + continue + schema = pf.schema_arrow + if schema.names != ["sequence_id", "sequence"] or any(schema.field(c).type != pa.string() for c in schema.names): + problems.append(f"columns: {p.name} has {schema}, not exactly sequence_id and sequence as strings") + continue + parsed += 1 + shard_rows = pf.metadata.num_rows + total += shard_rows + last = shard_rows + if n + 1 < len(files) and shard_rows != params["shard_rows"]: + problems.append(f"rows: {p.name} holds {shard_rows}, not {params['shard_rows']}") + +if parsed: + if not 0 < last <= params["shard_rows"]: + problems.append(f"rows: the last shard holds {last}, not at most {params['shard_rows']} and at least 1") + if total != params["sequences"]: + problems.append(f"rows: the shards hold {total} in all, not the release's {params['sequences']}") + picks = sorted({0, len(files) // 2, len(files) - 1}) + sampled = 0 + for n in picks: + p = files[n] + try: + pf = pq.ParquetFile(p) + except Exception: + continue + groups = pf.metadata.num_row_groups + g = 0 if n == 0 else groups - 1 if n == len(files) - 1 else groups // 2 + table = pf.read_row_group(g) + ids, seqs = table.column("sequence_id").to_pylist(), table.column("sequence").to_pylist() + sampled += len(ids) + bad = next((s for s in ids if not s.startswith(PREFIX)), None) + if bad is not None: + problems.append(f"sequence_id: {p.name} row group {g} holds {bad!r}, not starting with {PREFIX}") + bad = next((s for s in seqs if s is None or not LETTERS.fullmatch(s)), None) + if bad is not None: + problems.append(f"sequence: {p.name} row group {g} holds {bad!r}, not uppercase letters only") + print(f"sampled: {len(picks)} row groups ({sampled} rows)") + +print(f"data: {len(files)} shards, {total} rows") +print("\n".join(problems) or "ok") +sys.exit(1 if problems else 0) diff --git a/datasets/uniref90/recipe/build b/datasets/uniref90/recipe/build new file mode 100755 index 0000000..1c4f88d --- /dev/null +++ b/datasets/uniref90/recipe/build @@ -0,0 +1,133 @@ +#!/usr/bin/env -S python3 -I +"""recipe/build : UniRef90 of UniProt release 2026_04 as sharded two-column parquet. Downloads +uniref90.fasta.gz into /.work/, refusing a size or MD5 that differs from params, then streams it, never +holding the whole file in memory, into /data/00000.parquet, 00001.parquet, ...: the FASTA's order, +shard_rows rows per shard (the last one partial), zstd, row groups of about 100k rows. sequence_id is the +header's first token, sequence the amino acids of the header's lines joined. Stages resume: a rerun keeps +the verified FASTA and the shards already written. Prints the shard and row totals last. +""" +import gzip +import hashlib +import os +import sys +import time +import tomllib +import urllib.request +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + +params = tomllib.loads((Path(__file__).parent / "params.toml").read_text()) +out = Path(sys.argv[1]).resolve() +work, data = out / ".work", out / "data" +SCHEMA = pa.schema([("sequence_id", pa.string()), ("sequence", pa.string())]) + + +def shard_paths() -> list[Path]: + """data/'s shards in order; a gap ends the list, so a partial build resumes at its end.""" + paths, i = [], 0 + while (data / f"{i:05d}.parquet").is_file(): + paths.append(data / f"{i:05d}.parquet") + i += 1 + return paths + + +def rows(paths: list[Path]) -> int: + return sum(pq.ParquetFile(p).metadata.num_rows for p in paths) + + +def download() -> Path: + """The FASTA in .work/, resumed by range from where a partial one stopped and given three attempts at + a connection; refuses a size or MD5 that differs from params.""" + name = params["url"].rsplit("/", 1)[-1] + path, part = work / name, Path(f"{work / name}.part") + if path.is_file(): + return path + for attempt in range(3): + have = part.stat().st_size if part.exists() else 0 + if have >= params["bytes"]: + break + try: + headers = {"User-Agent": "brineylab-datasets"} + if have: + headers["Range"] = f"bytes={have}-" + request = urllib.request.Request(params["url"], headers=headers) + with urllib.request.urlopen(request, timeout=120) as r, open(part, "ab") as f: + if have and r.status != 206: + sys.exit(f"{params['url']} answered {r.status} to a range request at {have} bytes") + while block := r.read(1 << 22): + f.write(block) + break + except OSError as e: + if attempt == 2: + raise + print(f"{name}: {e}; retrying from {part.stat().st_size} bytes", file=sys.stderr) + time.sleep(30) + if part.stat().st_size != params["bytes"]: + sys.exit(f"{name}: {part.stat().st_size} bytes is not {params['bytes']}") + digest = hashlib.md5() + with open(part, "rb") as f: + while block := f.read(1 << 24): + digest.update(block) + if digest.hexdigest() != params["md5"]: + sys.exit(f"{name}: MD5 {digest.hexdigest()} is not {params['md5']}") + part.rename(path) + return path + + +def records(gz: Path): + """(sequence_id, sequence) per FASTA record, in the file's order; the whole file is never held.""" + with gzip.open(gz, "rt", encoding="ascii") as f: + sid, parts = None, [] + for line in f: + if line.startswith(">"): + if sid is not None: + yield sid, "".join(parts) + sid, parts = line[1:].split(None, 1)[0], [] + else: + parts.append(line.rstrip()) + if sid is not None: + yield sid, "".join(parts) + + +def write(i: int, ids: list[str], seqs: list[str]) -> None: + """Shard i whole or not at all: written under .work/ then moved into data/.""" + pq.write_table(pa.table({"sequence_id": ids, "sequence": seqs}, schema=SCHEMA), work / "shard.part", + row_group_size=100_000, compression="zstd") + os.replace(work / "shard.part", data / f"{i:05d}.parquet") + + +def shard(gz: Path, first: int) -> None: + """data/'s shards from first on, in the FASTA's order, skipping the rows of the shards already there.""" + data.mkdir(parents=True, exist_ok=True) + skip = first * params["shard_rows"] + i, ids, seqs = first, [], [] + for sid, seq in records(gz): + if skip: + skip -= 1 + continue + ids.append(sid) + seqs.append(seq) + if len(ids) == params["shard_rows"]: + write(i, ids, seqs) + i, ids, seqs = i + 1, [], [] + if ids: + write(i, ids, seqs) + + +def main() -> None: + if len(sys.argv) != 2: + sys.exit(f"usage: {Path(sys.argv[0]).name} ") + out.mkdir(parents=True, exist_ok=True) + work.mkdir(parents=True, exist_ok=True) + paths = shard_paths() + if rows(paths) != params["sequences"]: + shard(download(), len(paths)) + paths = shard_paths() + print(f"shards: {len(paths)}") + print(f"rows: {rows(paths)}") + + +if __name__ == "__main__": + main() diff --git a/datasets/uniref90/recipe/params.toml b/datasets/uniref90/recipe/params.toml new file mode 100644 index 0000000..fc31c43 --- /dev/null +++ b/datasets/uniref90/recipe/params.toml @@ -0,0 +1,6 @@ +release = "2026_04" +url = "https://ftp.uniprot.org/pub/databases/uniprot/uniref/uniref90/uniref90.fasta.gz" +bytes = 32692271968 +md5 = "9410e1bb6269000a1b1c332854c75322" +sequences = 125724408 +shard_rows = 1000000 diff --git a/datasets/uniref90/recipe/test b/datasets/uniref90/recipe/test new file mode 100755 index 0000000..6311dce --- /dev/null +++ b/datasets/uniref90/recipe/test @@ -0,0 +1,72 @@ +#!/usr/bin/env -S python3 -I +"""recipe/test : exit 0 iff holds a version of uniref90. From the shards' parquet footers and +three sampled row groups, in seconds: the shards are numbered without a gap, the schema is exactly +sequence_id and sequence (strings), every shard but the last is exactly shard_rows rows, the total is +exactly the release's count, and in the samples every sequence_id starts with UniRef90_ and every sequence +is uppercase letters only. One line per failure, naming the check and the first offending row.""" +import re +import sys +import tomllib +from pathlib import Path + +import pyarrow as pa +import pyarrow.parquet as pq + +params = tomllib.loads((Path(__file__).parent / "params.toml").read_text()) +PREFIX = "UniRef" + Path(__file__).resolve().parent.parent.name.removeprefix("uniref") + "_" +LETTERS = re.compile(r"[A-Z]+") + +out, problems = Path(sys.argv[1]), [] +files = sorted((out / "data").glob("*.parquet")) if (out / "data").is_dir() else [] +if not files: + problems.append("files: data/ holds no shards") +if [p.name for p in files] != [f"{i:05d}.parquet" for i in range(len(files))]: + problems.append("files: data/'s shards are not 00000.parquet onward without a gap") + +total, last, parsed = 0, 0, 0 +for n, p in enumerate(files): + try: + pf = pq.ParquetFile(p) + except Exception as e: + problems.append(f"parses: {p.name} is not a parquet file: {e}") + continue + schema = pf.schema_arrow + if schema.names != ["sequence_id", "sequence"] or any(schema.field(c).type != pa.string() for c in schema.names): + problems.append(f"columns: {p.name} has {schema}, not exactly sequence_id and sequence as strings") + continue + parsed += 1 + shard_rows = pf.metadata.num_rows + total += shard_rows + last = shard_rows + if n + 1 < len(files) and shard_rows != params["shard_rows"]: + problems.append(f"rows: {p.name} holds {shard_rows}, not {params['shard_rows']}") + +if parsed: + if not 0 < last <= params["shard_rows"]: + problems.append(f"rows: the last shard holds {last}, not at most {params['shard_rows']} and at least 1") + if total != params["sequences"]: + problems.append(f"rows: the shards hold {total} in all, not the release's {params['sequences']}") + picks = sorted({0, len(files) // 2, len(files) - 1}) + sampled = 0 + for n in picks: + p = files[n] + try: + pf = pq.ParquetFile(p) + except Exception: + continue + groups = pf.metadata.num_row_groups + g = 0 if n == 0 else groups - 1 if n == len(files) - 1 else groups // 2 + table = pf.read_row_group(g) + ids, seqs = table.column("sequence_id").to_pylist(), table.column("sequence").to_pylist() + sampled += len(ids) + bad = next((s for s in ids if not s.startswith(PREFIX)), None) + if bad is not None: + problems.append(f"sequence_id: {p.name} row group {g} holds {bad!r}, not starting with {PREFIX}") + bad = next((s for s in seqs if s is None or not LETTERS.fullmatch(s)), None) + if bad is not None: + problems.append(f"sequence: {p.name} row group {g} holds {bad!r}, not uppercase letters only") + print(f"sampled: {len(picks)} row groups ({sampled} rows)") + +print(f"data: {len(files)} shards, {total} rows") +print("\n".join(problems) or "ok") +sys.exit(1 if problems else 0) From 4a5975c15f20455a719e4e58b076f154ebbfa93b Mon Sep 17 00:00:00 2001 From: Bryan Briney <821533+briney@users.noreply.github.com> Date: Wed, 11 Nov 2026 03:22:03 +0000 Subject: [PATCH 2/6] uniref90: publish v1.0.0 from dev/uniref90@c26df69 --- datasets/uniref90/SKILL.md | 48 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 datasets/uniref90/SKILL.md diff --git a/datasets/uniref90/SKILL.md b/datasets/uniref90/SKILL.md new file mode 100644 index 0000000..319a789 --- /dev/null +++ b/datasets/uniref90/SKILL.md @@ -0,0 +1,48 @@ +--- +name: uniref90 +description: "UniRef90 of UniProt release 2026_04 as sharded two-column parquet: one row per entry (a 90%-identity cluster's representative), sequence_id and sequence, in the FASTA's order. Reach for it to pre-train or evaluate a protein language model on UniRef90 at a pinned release." +metadata: + version: 1.0.0 + tools: datascience@1.2.0 + license: cc-by-4.0 +--- +# uniref90 + +Hub: `brineylab/uniref90`, tag `v1.0.0` + +## Files + +`data/00000.parquet` … `data/00125.parquet`: 126 shards holding 125,724,408 rows, one row per UniRef90 entry, in the FASTA's order. Every shard but the last holds exactly 1,000,000 rows (the last 224,408); zstd, row groups of about 100,000 rows. Columns: + +- `sequence_id`: the FASTA header's first token, the entry id (`UniRef90_A0A5A9P0L4`). +- `sequence`: the entry's amino acids, its lines after the header joined. + +## Reading it + +```python +from huggingface_hub import snapshot_download +import polars as pl +path = snapshot_download("brineylab/uniref90", repo_type="dataset", revision="v1.0.0") +shard = pl.read_parquet(f"{path}/data/00000.parquet", columns=["sequence_id", "sequence"]) +``` + +Rows keep the FASTA's order, so a subset of shards is a contiguous slice of the release, not a sample; shuffle yourself for training. + +## Recipe + +**Source:** UniRef release 2026_04 (published 2026-11-04), CC BY 4.0, pulled 2026-11-11: `uniref90.fasta.gz` (32,692,271,968 bytes, MD5 `9410e1bb6269000a1b1c332854c75322`) at https://ftp.uniprot.org/pub/databases/uniprot/uniref/uniref90/uniref90.fasta.gz, holding the release's 125,724,408 entries. The recipe refuses a download whose size or MD5 differs. + +One job in `datascience@1.2.0`: download the FASTA into `/.work/`, resuming a partial one by range; then stream it, never holding the whole file in memory, into `/data/00000.parquet`, `00001.parquet`, … in the FASTA's order, `shard_rows` rows per shard (the last one partial), zstd, row groups of about 100k rows. Stages resume: a rerun keeps the verified FASTA and the shards already written, and a finished build only rereads the footers. + +`params.toml`: `release = "2026_04"`; the file's `url`, `bytes`, and `md5`; `sequences = 125724408`, the release's count; `shard_rows = 1000000`. + +**The test** reads the shards' parquet footers and three sampled row groups, in seconds: the shards are `00000.parquet` onward without a gap; the schema is exactly `sequence_id` and `sequence`, strings; every shard but the last holds exactly 1,000,000 rows; the total is exactly the release's 125,724,408; and in the samples every `sequence_id` starts with `UniRef90_` and every `sequence` is uppercase letters only. + +## Compute + +One job, CPU only: 8 CPUs and 32G asked, on bcr's `cpu` partition. From nothing it takes about 44 min: the download (30.5 GB, one stream from ftp.uniprot.org at about 23 MB/s) 24 min; the MD5 check and the 126 shards 20 min. Peak memory 7.6 GB. Disk: 23.5 GB of shards and 30.5 GB under `/.work/`, the FASTA all of it, kept so a rerun resumes; the test reads only footers and three row groups, in 7 s. + +## Gotchas + +- The URL serves UniProt's current release only: once 2026_05 is out, the pinned MD5 refuses it, and a rebuild is a new version. +- The FASTA header's other fields (`n=`, `Tax=`, `TaxID=`, `RepID=`) are dropped; cluster sizes and taxonomy are in UniRef's XML, not here. From 7711f48a8fabeaeb161734dcc87bc5c8bb994d68 Mon Sep 17 00:00:00 2001 From: Bryan Briney <821533+briney@users.noreply.github.com> Date: Wed, 11 Nov 2026 03:22:03 +0000 Subject: [PATCH 3/6] One dataset a pull request: uniref100 waits off the branch until uniref90's is merged --- datasets/uniref100/recipe/build | 133 -------------------------- datasets/uniref100/recipe/params.toml | 6 -- datasets/uniref100/recipe/test | 72 -------------- 3 files changed, 211 deletions(-) delete mode 100755 datasets/uniref100/recipe/build delete mode 100644 datasets/uniref100/recipe/params.toml delete mode 100755 datasets/uniref100/recipe/test diff --git a/datasets/uniref100/recipe/build b/datasets/uniref100/recipe/build deleted file mode 100755 index c9713b4..0000000 --- a/datasets/uniref100/recipe/build +++ /dev/null @@ -1,133 +0,0 @@ -#!/usr/bin/env -S python3 -I -"""recipe/build : UniRef100 of UniProt release 2026_04 as sharded two-column parquet. Downloads -uniref100.fasta.gz into /.work/, refusing a size or MD5 that differs from params, then streams it, never -holding the whole file in memory, into /data/00000.parquet, 00001.parquet, ...: the FASTA's order, -shard_rows rows per shard (the last one partial), zstd, row groups of about 100k rows. sequence_id is the -header's first token, sequence the amino acids of the header's lines joined. Stages resume: a rerun keeps -the verified FASTA and the shards already written. Prints the shard and row totals last. -""" -import gzip -import hashlib -import os -import sys -import time -import tomllib -import urllib.request -from pathlib import Path - -import pyarrow as pa -import pyarrow.parquet as pq - -params = tomllib.loads((Path(__file__).parent / "params.toml").read_text()) -out = Path(sys.argv[1]).resolve() -work, data = out / ".work", out / "data" -SCHEMA = pa.schema([("sequence_id", pa.string()), ("sequence", pa.string())]) - - -def shard_paths() -> list[Path]: - """data/'s shards in order; a gap ends the list, so a partial build resumes at its end.""" - paths, i = [], 0 - while (data / f"{i:05d}.parquet").is_file(): - paths.append(data / f"{i:05d}.parquet") - i += 1 - return paths - - -def rows(paths: list[Path]) -> int: - return sum(pq.ParquetFile(p).metadata.num_rows for p in paths) - - -def download() -> Path: - """The FASTA in .work/, resumed by range from where a partial one stopped and given three attempts at - a connection; refuses a size or MD5 that differs from params.""" - name = params["url"].rsplit("/", 1)[-1] - path, part = work / name, Path(f"{work / name}.part") - if path.is_file(): - return path - for attempt in range(3): - have = part.stat().st_size if part.exists() else 0 - if have >= params["bytes"]: - break - try: - headers = {"User-Agent": "brineylab-datasets"} - if have: - headers["Range"] = f"bytes={have}-" - request = urllib.request.Request(params["url"], headers=headers) - with urllib.request.urlopen(request, timeout=120) as r, open(part, "ab") as f: - if have and r.status != 206: - sys.exit(f"{params['url']} answered {r.status} to a range request at {have} bytes") - while block := r.read(1 << 22): - f.write(block) - break - except OSError as e: - if attempt == 2: - raise - print(f"{name}: {e}; retrying from {part.stat().st_size} bytes", file=sys.stderr) - time.sleep(30) - if part.stat().st_size != params["bytes"]: - sys.exit(f"{name}: {part.stat().st_size} bytes is not {params['bytes']}") - digest = hashlib.md5() - with open(part, "rb") as f: - while block := f.read(1 << 24): - digest.update(block) - if digest.hexdigest() != params["md5"]: - sys.exit(f"{name}: MD5 {digest.hexdigest()} is not {params['md5']}") - part.rename(path) - return path - - -def records(gz: Path): - """(sequence_id, sequence) per FASTA record, in the file's order; the whole file is never held.""" - with gzip.open(gz, "rt", encoding="ascii") as f: - sid, parts = None, [] - for line in f: - if line.startswith(">"): - if sid is not None: - yield sid, "".join(parts) - sid, parts = line[1:].split(None, 1)[0], [] - else: - parts.append(line.rstrip()) - if sid is not None: - yield sid, "".join(parts) - - -def write(i: int, ids: list[str], seqs: list[str]) -> None: - """Shard i whole or not at all: written under .work/ then moved into data/.""" - pq.write_table(pa.table({"sequence_id": ids, "sequence": seqs}, schema=SCHEMA), work / "shard.part", - row_group_size=100_000, compression="zstd") - os.replace(work / "shard.part", data / f"{i:05d}.parquet") - - -def shard(gz: Path, first: int) -> None: - """data/'s shards from first on, in the FASTA's order, skipping the rows of the shards already there.""" - data.mkdir(parents=True, exist_ok=True) - skip = first * params["shard_rows"] - i, ids, seqs = first, [], [] - for sid, seq in records(gz): - if skip: - skip -= 1 - continue - ids.append(sid) - seqs.append(seq) - if len(ids) == params["shard_rows"]: - write(i, ids, seqs) - i, ids, seqs = i + 1, [], [] - if ids: - write(i, ids, seqs) - - -def main() -> None: - if len(sys.argv) != 2: - sys.exit(f"usage: {Path(sys.argv[0]).name} ") - out.mkdir(parents=True, exist_ok=True) - work.mkdir(parents=True, exist_ok=True) - paths = shard_paths() - if rows(paths) != params["sequences"]: - shard(download(), len(paths)) - paths = shard_paths() - print(f"shards: {len(paths)}") - print(f"rows: {rows(paths)}") - - -if __name__ == "__main__": - main() diff --git a/datasets/uniref100/recipe/params.toml b/datasets/uniref100/recipe/params.toml deleted file mode 100644 index 9ab8b30..0000000 --- a/datasets/uniref100/recipe/params.toml +++ /dev/null @@ -1,6 +0,0 @@ -release = "2026_04" -url = "https://ftp.uniprot.org/pub/databases/uniprot/uniref/uniref100/uniref100.fasta.gz" -bytes = 63065088280 -md5 = "923e539321e9bfe6a9b74d343284d600" -sequences = 227086028 -shard_rows = 1000000 diff --git a/datasets/uniref100/recipe/test b/datasets/uniref100/recipe/test deleted file mode 100755 index 3ab3032..0000000 --- a/datasets/uniref100/recipe/test +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env -S python3 -I -"""recipe/test : exit 0 iff holds a version of uniref100. From the shards' parquet footers and -three sampled row groups, in seconds: the shards are numbered without a gap, the schema is exactly -sequence_id and sequence (strings), every shard but the last is exactly shard_rows rows, the total is -exactly the release's count, and in the samples every sequence_id starts with UniRef100_ and every sequence -is uppercase letters only. One line per failure, naming the check and the first offending row.""" -import re -import sys -import tomllib -from pathlib import Path - -import pyarrow as pa -import pyarrow.parquet as pq - -params = tomllib.loads((Path(__file__).parent / "params.toml").read_text()) -PREFIX = "UniRef" + Path(__file__).resolve().parent.parent.name.removeprefix("uniref") + "_" -LETTERS = re.compile(r"[A-Z]+") - -out, problems = Path(sys.argv[1]), [] -files = sorted((out / "data").glob("*.parquet")) if (out / "data").is_dir() else [] -if not files: - problems.append("files: data/ holds no shards") -if [p.name for p in files] != [f"{i:05d}.parquet" for i in range(len(files))]: - problems.append("files: data/'s shards are not 00000.parquet onward without a gap") - -total, last, parsed = 0, 0, 0 -for n, p in enumerate(files): - try: - pf = pq.ParquetFile(p) - except Exception as e: - problems.append(f"parses: {p.name} is not a parquet file: {e}") - continue - schema = pf.schema_arrow - if schema.names != ["sequence_id", "sequence"] or any(schema.field(c).type != pa.string() for c in schema.names): - problems.append(f"columns: {p.name} has {schema}, not exactly sequence_id and sequence as strings") - continue - parsed += 1 - shard_rows = pf.metadata.num_rows - total += shard_rows - last = shard_rows - if n + 1 < len(files) and shard_rows != params["shard_rows"]: - problems.append(f"rows: {p.name} holds {shard_rows}, not {params['shard_rows']}") - -if parsed: - if not 0 < last <= params["shard_rows"]: - problems.append(f"rows: the last shard holds {last}, not at most {params['shard_rows']} and at least 1") - if total != params["sequences"]: - problems.append(f"rows: the shards hold {total} in all, not the release's {params['sequences']}") - picks = sorted({0, len(files) // 2, len(files) - 1}) - sampled = 0 - for n in picks: - p = files[n] - try: - pf = pq.ParquetFile(p) - except Exception: - continue - groups = pf.metadata.num_row_groups - g = 0 if n == 0 else groups - 1 if n == len(files) - 1 else groups // 2 - table = pf.read_row_group(g) - ids, seqs = table.column("sequence_id").to_pylist(), table.column("sequence").to_pylist() - sampled += len(ids) - bad = next((s for s in ids if not s.startswith(PREFIX)), None) - if bad is not None: - problems.append(f"sequence_id: {p.name} row group {g} holds {bad!r}, not starting with {PREFIX}") - bad = next((s for s in seqs if s is None or not LETTERS.fullmatch(s)), None) - if bad is not None: - problems.append(f"sequence: {p.name} row group {g} holds {bad!r}, not uppercase letters only") - print(f"sampled: {len(picks)} row groups ({sampled} rows)") - -print(f"data: {len(files)} shards, {total} rows") -print("\n".join(problems) or "ok") -sys.exit(1 if problems else 0) From 75285c3d164fd156bebbc37bc0a017ab0a9dfb1d Mon Sep 17 00:00:00 2001 From: Bryan Briney <821533+briney@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:08:57 -0800 Subject: [PATCH 4/6] uniref90 1.0.0's record: its candidate passed on bcr --- results/uniref90/1.0.0.txt | 147 +++++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) create mode 100644 results/uniref90/1.0.0.txt diff --git a/results/uniref90/1.0.0.txt b/results/uniref90/1.0.0.txt new file mode 100644 index 0000000..02de150 --- /dev/null +++ b/results/uniref90/1.0.0.txt @@ -0,0 +1,147 @@ +--- +dataset: uniref90 +version: 1.0.0 +hub: brineylab/uniref90 +branch: candidate-c1ba4f4c5d2e72f4e7b8731078da60197ad01924 +commit: 3a7f6d6bb1961be9ad56989f31165c2892d223b5 +tree: c1ba4f4c5d2e72f4e7b8731078da60197ad01924 +tools: datascience@1.2.0 +runner: sbatch --partition=cpu --cpus-per-task=8 --mem=32G --time=04:00:00 +host: bcr.scripps.edu +started: 2026-11-11T03:22:35Z +exit: 0 +--- +pyxis: importing docker image: ghcr.io#brineylab/toolshed/datascience:1.2.0 +pyxis: imported docker image: ghcr.io#brineylab/toolshed/datascience:1.2.0 +shards: 126 +rows: 125724408 +sampled: 3 row groups (224408 rows) +data: 126 shards, 125724408 rows +ok +file data/00000.parquet 1342533916 1a8814fa8314ee3fb737f8b333e093c9142a761c63fb86b5f6fff91cb7d086ef +file data/00001.parquet 834643324 67e05c599056c56f57b15283b8003c5997f3ac9fafff6236a0194f334f00b9ad +file data/00002.parquet 703896262 78590d5f10665088ba141a1b1db7dc7b9ee192b3fb5cbcea2374cd008a89eeae +file data/00003.parquet 622383816 97d308d3c569bd33605cdd5b1657885a53e9cf235366b157effc223ee3ae6b7e +file data/00004.parquet 573870475 508c620fc1e9fcffdaca10628afbbb3f86f3e13e6bd60243f704503685bb2959 +file data/00005.parquet 535828452 48108073e11ea3ea3e09680a8a70dd4bd048fbfe17c4eaffbabd111c0ce80961 +file data/00006.parquet 497696294 e0cb29674569209546df900699ab7aa4d4d44aba9938ddc700b14ff23d88e0b3 +file data/00007.parquet 470231266 299168547554687338909015f34e2f58824317a921196848a2fadc20ef14247d +file data/00008.parquet 447034199 c8c9ce3c4e2d2de6479f7becdee2727b875a366dde75072177701a2a17f1a18a +file data/00009.parquet 429322534 d25b865d6a6c3efc67546191afe614de14fdca40dd8ebbfdaebd8c402bd8e640 +file data/00010.parquet 412434682 94dee40be9fc9418f87e3256ef8be407bc8844e6fea01628e1145d5c59e229b0 +file data/00011.parquet 395508478 cfbe281e56b0e61f904347ae53e4bb986a219abdfa691a2248928eaa589d9d22 +file data/00012.parquet 382159424 616e67f62861dcd0ac98db80c9a0fddaef2f9e50535143d180ecbeaa422d0f85 +file data/00013.parquet 368847359 9a14eab64cd5d4cbd3d58ba74e92d7ba18fec33fb370c7762e078a3d6fc68b61 +file data/00014.parquet 356628947 65e81cf984d04b3ba2e195957326324c95e042971f93cd4adb5585b290eb9fda +file data/00015.parquet 345795975 2cd311ec58ba5d958dde68b0f3658b8372eba512ddc329ecba8c8abdb7f2c73e +file data/00016.parquet 335171783 6c3b8384c27777e71b5d4d68f220fdbc4e8333ec23fffe520adf9e55c9f0d08f +file data/00017.parquet 325136719 ef8fe3963f1689571f90607e60f400da9413b2f84d237fc900278409eb1ad0d8 +file data/00018.parquet 316792090 ffd7a81df5910837c7be9a54defb75d02af7f122d3b44f77928f644a1810e336 +file data/00019.parquet 309528998 65adf19a97803839eb79a3e4f043e3f17d92a62907d117dd04b2197557393dcc +file data/00020.parquet 302304105 3108cd694483c7ac84ddee99255d9e8c1ac48c5f1f6fa506c9d762120b3461b4 +file data/00021.parquet 295439110 f19a0ffd96127f0a8966b9a52eb69e6ada1b31456dbfb8a076ec163a2189e9c8 +file data/00022.parquet 289489919 0b77ffcdc0549be14c387a8789db1161bcbbe79452fe8a362c304c4359b064c9 +file data/00023.parquet 283662860 20f9ba67f571f12b6081a7d8b299e1070d7d97feacef17d3887c0f6f6a31ebcb +file data/00024.parquet 277496653 92fec5c0074a76c598648ae96c04fa024ae678463ce0627d0a849de8d9bbaf7f +file data/00025.parquet 271953697 b6cd86b2ffe9f6595b98e8b07210a8187b4157c41a5e160b1ddb4aa222945b8a +file data/00026.parquet 266643386 df2a4602fd20a37ea4497df3404aafe1b718e9345d85632cd8aa6afafc4b821f +file data/00027.parquet 261266269 e61551482da45e9f92efac78b636771d2e01435b670fbb60a7b1ee8dea7224e4 +file data/00028.parquet 255682168 b741636e192d8f6e1234c072c5ef9a2f41d80b38bfe594c26993e455374e73e9 +file data/00029.parquet 252683319 1febc0ca827fc8fcdd11ec0aa1d90b542300cde592162b0c87112a307e407bd2 +file data/00030.parquet 248048401 68984fc19fa2948b66b6098404e5f60abc636c90af855ada94d55b8eec5351cf +file data/00031.parquet 244210254 ad87708b514dfe0dba7296e3c7f6fd7a8ff892ed40fad2269741683857ee28dc +file data/00032.parquet 240240672 3a47aab1b8ddc8970a49173a3b6e7c6438803fe2abb781deff2839af9a07ec74 +file data/00033.parquet 237376585 1cf1d8defb82082324013df0cf5718f970fda902cc96a0463399e666a7923393 +file data/00034.parquet 232923634 4c33dbd8159c1063d314597fc5200ef9998993b448745bf67af13c2c6fd16f0f +file data/00035.parquet 229320538 53dc8ee43a582a9add6b1ae629d756d6c3b9e48dc794420a04775b039d549904 +file data/00036.parquet 226713904 ec267d5e684bd24c4e26a53569088dd1afeb20c4f4853fb4dc26abec900afedd +file data/00037.parquet 223468038 b2eefdcceda392373c80a65bd533bd2d8cfb32c1a34722a474580362b3467279 +file data/00038.parquet 220092455 5cb29f71e5b47d3c7ad8d386779ff216e9806ab711a758e6f99bf89588664678 +file data/00039.parquet 217272478 94dc0b85eeca529d00c615d21717711aa5f39987817703e5ee43af80cb291dfa +file data/00040.parquet 214996186 868b1bd95b225cf6f2e6750e0912af8c517c8964e6faf1939daf39405a4e94df +file data/00041.parquet 212082114 a8eec108b2aa850fb299ebe2800a10d26906f8d9f02d44a007dde4718cf8f44e +file data/00042.parquet 209174534 34662728109f789668c99e0ad09ee691fa02a5b98e74a8a6c5e2f0be3f3c70f9 +file data/00043.parquet 206144886 c6c22c778a7a32d35717dd1b654c1b398805425e2db45443df4b198c52ddf3b1 +file data/00044.parquet 203231811 ffc55d71629e63946e59ef67b6d420876abae96c16ec94f0095cff069f672e9a +file data/00045.parquet 199972136 f6a819c00a25e3519ca1b561733d413823533a58936bc280879be6113ee4119b +file data/00046.parquet 197154108 b8fc99a49cd93000f25d419798c4143c29ab645f4747e8cf21bca45270875110 +file data/00047.parquet 194234975 ed9ea1633a997e00c684de40d64279b66207cb3e3f9e537f2ae7eb9e247746df +file data/00048.parquet 191335327 b431573a51151853078d5105ac4abe60f867756b210d2de512410de6c628b709 +file data/00049.parquet 188074903 831a29a6cab852b387b788cd28f7d712f86b8d78bcaadd1a55b0089cce4950b1 +file data/00050.parquet 185395005 7825a7713a9028eb32058bdbad49d0ebf0530e8474e1cbc3d05ae79915f5093c +file data/00051.parquet 183006911 056abd0f9d6ac2ba760251c89f20e5ab63ac43105ae5a1d8194f5aba74b1b24e +file data/00052.parquet 180538283 733e9093c9d1318c53f7eabe6dd0f56c1b4d8916058d93656e883bd430a84b31 +file data/00053.parquet 178264970 f5cf778217bdfae65d3f8f594e4a8f46ad6d9388506486b0583abb06a0bf57b7 +file data/00054.parquet 176313246 afbabec735b874e1f018cbbd29cd31c193d42421a3b9b030eaa8961b5899170d +file data/00055.parquet 174086368 dc4ff018c2dd6de9e28a5ddd705922a187a5913191f7faa27dc8eebec33c69d9 +file data/00056.parquet 171822633 73035cd7002b5042c717d19c60e5236d8c6d3aca24d52faee338cf28e9ce9953 +file data/00057.parquet 169445385 26f464ea9a0746936ac9156edf7a49308f752c8386689231d94f3f85a14f6b56 +file data/00058.parquet 167929028 bb6f17ea527351801836e091267cef5e2dd392ea1d89e9500452f736d9df9376 +file data/00059.parquet 165739209 3fe7a0f36749b8ffbb33beb4ea3602c5cb2ef8e0273eead4615a75a44275046c +file data/00060.parquet 163580970 344348e85a9122b2861a954a3cccc59eef7820354a8107a18cb84fd37c4780df +file data/00061.parquet 161482786 92b45183263082429db4e797ea286743f429d441775076e59beabb0dd7d0338c +file data/00062.parquet 159352379 f90ca7469dbc3af00bc05a6c63adbd16d074d88716d85c080896bd9e0a8171ec +file data/00063.parquet 157077621 41cd06cf194e6cfc67def12d7674edcfadc7f986cbfcbb4e9df2e88892a4e2c9 +file data/00064.parquet 154983379 c4da6d11d77d2e53cd6305f2136370544f8e473e20eb356655af20a81fa5bc55 +file data/00065.parquet 152884968 8c5b64e4386410ce27ff8e9e9305960b56bd5e5945bc97fa525b73910abceda8 +file data/00066.parquet 150344163 dc0e8363dd9f034f01088b695e6cef8778957c96fb8f60aaa668df706182fdf4 +file data/00067.parquet 148082962 be9c1b3f6fba38e7748baef505a947a0b389999a6ae112028e8b4203bd7f8dc5 +file data/00068.parquet 145822924 0ae2337f3b261f8866dbd893dfd6af2365eead31c3a805eb4b19974673dd7ed5 +file data/00069.parquet 143624821 367cd83066906e4cefd90a2ec8a4ecfdb8ee1034e9c0f26f2f6e2f565a77cb31 +file data/00070.parquet 141077001 12d41830365bbb22ef0e5dfcc60d30322256769f3ccb649564f23654b042d9fa +file data/00071.parquet 139035927 9623a60f10c53965cdc8f7f57860204046973e1b763c27ca2a12d12f0efe4d45 +file data/00072.parquet 137090428 a0db2b512827c8f29001d7b54db6bc467108dd2ecc669c13d13b2a7a7fa0718d +file data/00073.parquet 135387061 781b2776fee2ef001803b73dc9c262e5a27c366ab56ab3864258af8dff802273 +file data/00074.parquet 133129671 23b96f2169e49a92b732eb91224c9be85c4f6dda24616312ba7348556d3910d2 +file data/00075.parquet 130863432 e4d265ea5f82a6ee69cef617695b54dfab7b9dc8131c45987290cdc6f37b2654 +file data/00076.parquet 128734237 13cc9e659a74e9f74725f72cc2cd52db7ea6cfa78325517406487c47d33c4494 +file data/00077.parquet 126513516 dbc76a29fa53819842ff1a05b63075a8e4d2ed5cf9b0e2dcdf73fff505dcd4a2 +file data/00078.parquet 124285591 1987d96abbebc6d6c8d2e9911cac24e03b45df851f56219e9a92a172633ffe3e +file data/00079.parquet 122471641 f2611836fdfe0079f950e506c07234a60dacf9e21c7b6be6f82fc2487a8eaf7d +file data/00080.parquet 120689597 2d486b984acb66cf432009f43a20797e8e09bfe28ab047a5307a6a38376b1e82 +file data/00081.parquet 118547656 4f50d3ca7b741bc6b265ae7e669eb89e9734ebe415578d1a07a75cede5f84f58 +file data/00082.parquet 116363202 1bdc9dec5c1415404e69737597b38c1c2b12b52998b6b229c5b03545a64e453a +file data/00083.parquet 114790020 fa7e9c6997bc957db72fe1b8825b99bc1d115bd0d259af3f9d6acbca45d663bb +file data/00084.parquet 112754170 075408cda8ba7c79060cf3ba963f4cc71054ee4a6fedced2833c2bfee194c4dc +file data/00085.parquet 110718198 282ddbfee20ea80ecf4f022f8ea50ef6c48182ce5ebb9ae07043d8964395f879 +file data/00086.parquet 108563792 366461510fc779c1ed5334cb759cb31029fe71b1046f85c861825b019a16122c +file data/00087.parquet 106883450 883436040ce220e4ab0ad3c29f5d370452cdff5647261bca046cbb1f446abd85 +file data/00088.parquet 104941761 8e2d656887f96f07643e7a5eb369b3419fc99c485c11d969770445c883772c93 +file data/00089.parquet 102874823 39f0f74abe129e5a2c011cdffe6c7e921e26dc8c3e598e0e45fc4287b3f9d884 +file data/00090.parquet 100854056 d2deae9c5a83ec6afa05c45268bba42046b54e2f570d96a3b2af26c3f2742dcd +file data/00091.parquet 98947952 3796e569d2ac24d47ade251e97103d973802bb9f4bbfd5a782501a322e2f1090 +file data/00092.parquet 97030984 7493168db0576319bb047f01102afdc20d1324294cc238c402f34db90ce3f4d9 +file data/00093.parquet 94849716 0e5553696f79fb2426fb48369d2422f4949c0a7d7dde3f66ae716b6819a72cae +file data/00094.parquet 92831579 0e079fcfe15e41c403c2576a9f7b4248b7d84b86f235cdbec00759d2e04f6891 +file data/00095.parquet 90943726 88206ff904a2b75095fcc173f046898b112c0774874d3cac54df3a29cf238afa +file data/00096.parquet 88989797 2119b63486d687e631227954e8af622f2f5e6bf9649c1744c04e07314cfa7491 +file data/00097.parquet 87513742 a77021ab0c14566fcecbce4b375a47944c843c0a2f0314acad6d20d339ffd7de +file data/00098.parquet 85548401 4163fe5cded55ec6c80065cbb52c6414070a515ed5425f1295b39a4e104e26f8 +file data/00099.parquet 83797844 27db1d26c46e360e64007c715b352e8b8451e23397312651d7b8bb842807f2cc +file data/00100.parquet 82079164 a9fd9feb8e22acca0b62ce7652398f14df83775750899d44df0c370b5d8bc9bf +file data/00101.parquet 80251478 651a80e0a71e34be2f6a5c4d75e6a5ed41fb40f82a090a68813d53885b41de17 +file data/00102.parquet 78393024 26d87b4d89e8e5eead4e1ee4a5c30e7f2100be22a8e8416d99e47d3f8eb51fb9 +file data/00103.parquet 76560808 fc0d47018028c4fd724166d0d7338a61f09883cd71d8db644c75bcc1300dbec9 +file data/00104.parquet 74459042 c2d8d480170595a6e9f29f6fa8f5a54280a7a9e96881b4cda712d6371db755cf +file data/00105.parquet 72305382 64a15bcdd96d62e30014f0f2f7e51587fa6fa504177e2c7a69d01a38a4d9bfd0 +file data/00106.parquet 70399797 2ae428974cef27482d70c7c0cc7444c43239172af6157f4a453b47aad426b19e +file data/00107.parquet 68592683 9c91ed63e49c7b2b930f5669d07612cac1290e951648adc21b653acb11f5f624 +file data/00108.parquet 66859230 64386f462e5ee22dbfe0442895b88c9004114b546ca3c0c290892e5f091cacc3 +file data/00109.parquet 64990028 acd85343b05ea8d0a5c7a2024bccaa5483aedbde4434f903809cede74b4dcb8f +file data/00110.parquet 63049186 0a7e9b5da298a5b8ca96df6047dd184797dcb833ac5f93721b9f7d2683a61f3a +file data/00111.parquet 61321617 77daa1286484cd9c63ea9a7ae63e4a5ab3fe2903b4a41a4de89cce6f9467247d +file data/00112.parquet 59404972 2d3762e2b0e8fe34d423903f8e4557e12d5bb00827357c936423b5dd9b1202a9 +file data/00113.parquet 57474531 6b869a4f1958a1a7cb42877e044044165d3781d219aa63b540741505639149ef +file data/00114.parquet 55591490 408e0429780f25c4904e0f08cc12711bb664b99603a9fc78f8c30fcb0e3c471a +file data/00115.parquet 53427347 bb0fe6e9c5f628fcccf5d4b989e15397c079b9bf1ee3d5da8e39a633266c1b9f +file data/00116.parquet 51576176 977f0bb3fe87ff14038cf05c37536d5ab8a50f33cc8758666399378a23c55ff1 +file data/00117.parquet 49629526 7db40a393422647b464fa08c14034c94d6eabc749ffbb7f8118d047e9a21c563 +file data/00118.parquet 47642352 637815543ae682e28fa32da927e6f76f899185595a070cee2d2d08f357861371 +file data/00119.parquet 45711099 32252b698ba83d9eb2eee7b5f04e201a567ec91fee5d823399d14da52128cd1b +file data/00120.parquet 43666379 e90b1ff8a00698857fd7d2cd06a5f8a8e4481551e3b085a25d37b30b2c95a80f +file data/00121.parquet 41653510 09fa26c10c398a1e00b49c3ddd78492edf22a0d6972e370da80b1ac243c60da8 +file data/00122.parquet 39347515 8f0d9ac4b8d9213f899553fc7a7f94b40533f528a6338f9c95744e3ae58337fd +file data/00123.parquet 36447332 a2772ff9f3bea7e0317e3a5c87209709a232e0e70bea9905b52fb35fb1f9889b +file data/00124.parquet 32565083 bd8a9464944bb77fb0bece509fea67e0de1f1627b217756e18450a8aee964366 +file data/00125.parquet 18772323 d3df32f9197eb12f194a04f26b665aae43daa505ae8a144ba888ac2b5ca99be5 +commit: 3a7f6d6bb1961be9ad56989f31165c2892d223b5 From 397690b0cfe4dd66075641832bd7bdbf22a7ce9f Mon Sep 17 00:00:00 2001 From: Bryan Briney <821533+briney@users.noreply.github.com> Date: Wed, 11 Nov 2026 04:13:37 +0000 Subject: [PATCH 5/6] uniref90: the last shard holds 724,408 rows, not 224,408 125,724,408 entries in 126 shards, all but the last exactly 1,000,000 rows, leaves 724,408 rows in data/00125.parquet. 224,408 was the test's sampled-row-group count, which for uniref50 coincides with its last shard's rows and was carried over as such. A prose fix only: the recipe tree, the candidate, and the record stand. --- datasets/uniref90/SKILL.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/datasets/uniref90/SKILL.md b/datasets/uniref90/SKILL.md index 319a789..aa65544 100644 --- a/datasets/uniref90/SKILL.md +++ b/datasets/uniref90/SKILL.md @@ -12,7 +12,7 @@ Hub: `brineylab/uniref90`, tag `v1.0.0` ## Files -`data/00000.parquet` … `data/00125.parquet`: 126 shards holding 125,724,408 rows, one row per UniRef90 entry, in the FASTA's order. Every shard but the last holds exactly 1,000,000 rows (the last 224,408); zstd, row groups of about 100,000 rows. Columns: +`data/00000.parquet` … `data/00125.parquet`: 126 shards holding 125,724,408 rows, one row per UniRef90 entry, in the FASTA's order. Every shard but the last holds exactly 1,000,000 rows (the last 724,408); zstd, row groups of about 100,000 rows. Columns: - `sequence_id`: the FASTA header's first token, the entry id (`UniRef90_A0A5A9P0L4`). - `sequence`: the entry's amino acids, its lines after the header joined. From 536320a72be13d1263876234928c8aeb272e85df Mon Sep 17 00:00:00 2001 From: Bryan Briney <821533+briney@users.noreply.github.com> Date: Tue, 10 Nov 2026 20:56:20 -0800 Subject: [PATCH 6/6] uniref90 1.0.0's record: its candidate passed on bcr --- results/uniref90/1.0.0.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/results/uniref90/1.0.0.txt b/results/uniref90/1.0.0.txt index 02de150..f24ff19 100644 --- a/results/uniref90/1.0.0.txt +++ b/results/uniref90/1.0.0.txt @@ -3,12 +3,12 @@ dataset: uniref90 version: 1.0.0 hub: brineylab/uniref90 branch: candidate-c1ba4f4c5d2e72f4e7b8731078da60197ad01924 -commit: 3a7f6d6bb1961be9ad56989f31165c2892d223b5 +commit: 0a74e637bd557ed275864271ffd6fb313f8dd7db tree: c1ba4f4c5d2e72f4e7b8731078da60197ad01924 tools: datascience@1.2.0 runner: sbatch --partition=cpu --cpus-per-task=8 --mem=32G --time=04:00:00 host: bcr.scripps.edu -started: 2026-11-11T03:22:35Z +started: 2026-11-11T04:14:35Z exit: 0 --- pyxis: importing docker image: ghcr.io#brineylab/toolshed/datascience:1.2.0 @@ -144,4 +144,4 @@ file data/00122.parquet 39347515 8f0d9ac4b8d9213f899553fc7a7f94b40533f528a6338f9 file data/00123.parquet 36447332 a2772ff9f3bea7e0317e3a5c87209709a232e0e70bea9905b52fb35fb1f9889b file data/00124.parquet 32565083 bd8a9464944bb77fb0bece509fea67e0de1f1627b217756e18450a8aee964366 file data/00125.parquet 18772323 d3df32f9197eb12f194a04f26b665aae43daa505ae8a144ba888ac2b5ca99be5 -commit: 3a7f6d6bb1961be9ad56989f31165c2892d223b5 +commit: 0a74e637bd557ed275864271ffd6fb313f8dd7db