diff --git a/results/stok/1.0.0.txt b/results/stok/1.0.0.txt new file mode 100644 index 0000000..552f6b8 --- /dev/null +++ b/results/stok/1.0.0.txt @@ -0,0 +1,35 @@ +--- +tool: stok +version: 1.0.0 +image: ghcr.io/brineylab/toolshed/stok@sha256:127b07d19f1309d96ff6d70314fbb97fdf28ecc94c4d64ffede19daa93a3203c +digest: sha256:127b07d19f1309d96ff6d70314fbb97fdf28ecc94c4d64ffede19daa93a3203c +runner: sbatch --partition=flyloft --gres=gpu:1 --time=00:30:00 +host: sunk.6af0ac-eu-south-03b-briney-b200.coreweave.app +started: 2026-11-21T03:44:29Z +exit: 0 +--- +Job 1401: SUNK job dashboard: https://cks-grafana.coreweave.com/d/slurm-job-metrics/slurm-job-metrics?var-slurm_cluster=slurm&var-job_id=1401 +pyxis: importing docker image: ghcr.io#brineylab/toolshed/stok@sha256:127b07d19f1309d96ff6d70314fbb97fdf28ecc94c4d64ffede19daa93a3203c +pyxis: imported docker image: ghcr.io#brineylab/toolshed/stok@sha256:127b07d19f1309d96ff6d70314fbb97fdf28ecc94c4d64ffede19daa93a3203c +node: slurm-b200-205-097 +GPU 0: NVIDIA B200 (UUID: GPU-c7ebec8b-2ac7-c4d8-d9df-18d37031f95f) +[2026-11-21T03:47:18Z] slurm-b200-205-097: GPU 0: NVIDIA B200 (UUID: GPU-c7ebec8b-2ac7-c4d8-d9df-18d37031f95f) +[2026-11-21T03:47:18Z] tokenizing the fixture on cuda +/usr/local/lib/python3.12/dist-packages/torch/jit/_script.py:1491: FutureWarning: `torch.jit.script` is deprecated. Please switch to `torch.compile` or `torch.export`. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/torch/jit/_script.py:1491: FutureWarning: `torch.jit.script` is deprecated. Please switch to `torch.compile` or `torch.export`. + warnings.warn( +/usr/local/lib/python3.12/dist-packages/graphein/protein/tensor/angles.py:475: UserWarning: Using torch.cross without specifying the dim arg is deprecated. +Please either pass the dim explicitly or simply use torch.linalg.cross. +The default value of dim will change to agree with that of linalg.cross in a future release. (Triggered internally at /__w/pytorch/pytorch/aten/src/ATen/native/Cross.cpp:63.) + n1 = torch.cross(F.normalize(a - b, dim=-1), bc) +Wrote 2 shards (skipped 0), 2 rows, 1 rejected, 2 null labels. Output: /tmp/tmp.fu2aMEryDb/tokens +[2026-11-21T03:47:28Z] tokenization finished in 10s; checking the answer +/usr/local/lib/python3.12/dist-packages/torch/jit/_script.py:1491: FutureWarning: `torch.jit.script` is deprecated. Please switch to `torch.compile` or `torch.export`. + warnings.warn( +ok: AF-A0A0FXCOMPLETE-F1: 243 tokens, 0 null, 0 coordinate-null, decode RMSD 0.673 A +ok: AF-A0A0FXGAPPED-F1: 243 tokens, 2 null, 1 coordinate-null, decode RMSD 0.721 A +ok: identity preset large, codebook 4096, stok 0.1.4, checkpoint 7d1d43950a29..., codebook 29d21936c951... +ok: rejected AF-A0A0FXSHORT-F1 (chains_too_short) +[2026-11-21T03:47:35Z] all checks passed, 17s from tokenize start +Job 1401: SUNK job dashboard: https://cks-grafana.coreweave.com/d/slurm-job-metrics/slurm-job-metrics?var-slurm_cluster=slurm&var-job_id=1401 diff --git a/tools/stok/Containerfile b/tools/stok/Containerfile new file mode 100644 index 0000000..fbbd227 --- /dev/null +++ b/tools/stok/Containerfile @@ -0,0 +1,67 @@ +# STok (github.com/briney/stok, MIT) at commit 5dcecfed: protein structure +# tokenization for paired sequence-structure training. Its GCP-VQVAE +# "Large" preset (huggingface.co/Mahdip72/gcp-vqvae-large, upstream MIT -- +# LICENSE-gcp-vqvae.md, stok's own MIT in LICENSE-stok.md) encodes a protein +# backbone as one of 4096 discrete structure tokens per residue under the +# fixed training-native policy: full chains of 25-1280 residues, native +# sequence inputs, reference imputation, FP32 singleton inference, so token +# identity never depends on batching or sharding. Input is mmCIF text in +# Parquet shards (afdb-rep's shape) or a directory of structure files; output +# is plain Parquet shards carrying stok.tokenizer identity metadata and a +# rejected/ record for every row the fixed policy refuses. A GPU tool with +# its weights baked in, so a tokenization run reaches no network. +FROM docker.io/pytorch/pytorch:2.14.1-cuda13.0-cudnn9-runtime +ENV NVIDIA_VISIBLE_DEVICES=all NVIDIA_DRIVER_CAPABILITIES=compute,utility +# git and ca-certificates: pip installs stok from GitHub at the pinned +# commit (the PyPI "stok" is an unrelated older package). gcc and g++: stok +# requires graphein, whose cpdb-protein 0.2.0 ships only an sdist and +# compiles a C extension at install time, and the runtime base carries no +# compiler -- the first build attempt died on exactly that (the shed's +# lesson: ablm-eval, e1 and boltz-2 all carry gcc g++ for it). The compile +# runs once, here, at build time. +RUN apt-get update && apt-get install -y --no-install-recommends git ca-certificates gcc g++ \ + && rm -rf /var/lib/apt/lists/* +# The pinned set this campaign preflighted on CPU (a venv with exactly these +# versions: stok's own tests for the parser, policy, format, CLI and loaders +# passed, and the tool test's fixture tokenized end to end). torch stays at +# the base's 2.14.1: ==2.14.1 matches the base's 2.14.1+cu13.0 local label, +# so the base's matching torchaudio and torchvision stay consistent. stok's +# caps that bite here: hydra-core needs omegaconf<2.4; graphein pins numpy<2, +# which holds pyarrow under 26. +RUN pip install --no-cache-dir --break-system-packages \ + "stok @ git+https://github.com/briney/stok@5dcecfedd1040f30ca28de54ac6a6b22b9dc184b" \ + torch==2.14.1 \ + hydra-core==1.3.8 \ + omegaconf==2.3.0 \ + graphein==1.7.8 \ + torch-geometric==2.8.1 \ + click==8.5.1 \ + wandb==0.31.2 \ + accelerate==1.16.0 \ + tqdm==4.70.1 \ + numpy==1.26.4 \ + pyarrow==25.0.1 \ + biopython==1.88 \ + pyyaml==6.0.4 \ + x-transformers==2.8.0 \ + vector-quantize-pytorch==1.25.2 \ + einops==0.8.2 \ + einx==0.4.3 \ + loguru==0.8.0 +# The weights, fetched at build time through stok's own resolver: it names +# the Large checkpoint by its pinned sha256 +# (7d1d43950a29834e7f702409bf957e9ffb75eb3cb3952074ba4c545bb4130eaf), +# downloads it and verifies the +# digest, so the image carries exactly the release every shard's metadata +# names. The tokenizer and codebook read STOK_GCP_CACHE, the decoder +# STOK_DECODER_CACHE (stok's two loaders look at different variables) -- +# both point at the baked directory. The last two loads prove the build: +# strict state-dict loads of the full tokenizer and decoder from the pinned +# bytes. HF_HUB_OFFLINE is set although stok's resolver never reads it: a +# caller pushing a run's outputs to the Hub opts in explicitly. +ENV STOK_GCP_CACHE=/opt/stok-gcp STOK_DECODER_CACHE=/opt/stok-gcp HF_HUB_OFFLINE=1 +COPY LICENSE-stok.md LICENSE-gcp-vqvae.md /opt/stok-gcp/ +RUN python3 -I -c "from stok.utils.pretrained import resolve_gcp_artifact; print(resolve_gcp_artifact('large', progress=False))" \ + && python3 -I -c "from stok.models.gcp_vqvae import load_pretrained_tokenizer; t = load_pretrained_tokenizer('large'); print('tokenizer max_length', t.encoder.max_length, 'codebook', tuple(t.quantizer.codebook.shape))" \ + && python3 -I -c "from stok.models.decoder import load_pretrained_decoder; d = load_pretrained_decoder('large'); print('decoder codebook identity', d.codebook_sha256)" +COPY test /toolshed/test diff --git a/tools/stok/LICENSE-gcp-vqvae.md b/tools/stok/LICENSE-gcp-vqvae.md new file mode 100644 index 0000000..4b16d28 --- /dev/null +++ b/tools/stok/LICENSE-gcp-vqvae.md @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Mahdi Pourmirzaei + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/stok/LICENSE-stok.md b/tools/stok/LICENSE-stok.md new file mode 100644 index 0000000..138c30e --- /dev/null +++ b/tools/stok/LICENSE-stok.md @@ -0,0 +1,22 @@ +MIT License + +Copyright (c) 2025 Bryan Briney +Copyright (c) 2025 Mahdi Pourmirzaei (adapted GCP-VQVAE preparation code) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/tools/stok/SKILL.md b/tools/stok/SKILL.md new file mode 100644 index 0000000..6e97756 --- /dev/null +++ b/tools/stok/SKILL.md @@ -0,0 +1,75 @@ +--- +name: stok +description: "STok's GCP-VQVAE Large structure tokenizer: turns protein structures into discrete structure tokens (one of 4096 per residue) from mmCIF text in Parquet shards or from structure files, as sharded Parquet with tokenizer-identity metadata. Reach for it to build a structure-token dataset over structures you hold, or to tokenize structures alongside their sequences for paired sequence-structure training." +metadata: + version: 1.0.0 +--- +# stok + +Image: `ghcr.io/brineylab/toolshed/stok:1.0.0` + +STōk (github.com/briney/stok, MIT) at commit `5dcecfedd1040f30ca28de54ac6a6b22b9dc184b`, on PyTorch 2.14.1 (the base's CUDA 13.0 build), with the GCP-VQVAE **Large** preset's weights (huggingface.co/Mahdip72/gcp-vqvae-large, revision `64bb4ff628f7d2ccba587cdc9f0eaa97c1f3f9a1`, upstream MIT) baked into the image at `/opt/stok-gcp`, fetched at build time through STōk's own resolver, which verifies the checkpoint's pinned sha256 — so a run reaches no network and carries exactly the release every output shard's metadata names. The same code also decodes tokens back to backbone coordinates (`load_pretrained_decoder`), which is how the tool's test checks the tokens mean the structure. + +## Commands + +Parquet shards of mmCIF text — afdb-rep's shape: an `id` column, the mmCIF text in a `cif` column, and any columns to carry along: + +```bash +stok tokenize-structures input_dir output_dir --preset large --text-column cif \ + --id-column id --keep-column mean_plddt --keep-column uniprot_accession \ + --include-coordinates --device cuda +``` + +A directory of structure files (`.cif`/`.mmcif` or `.pdb` with SEQRES; PDBs without SEQRES are refused): + +```bash +stok tokenize-structures structures_dir output_dir --preset large --device cuda +``` + +Shard-parallel jobs — job `I` of `N` takes every N-th input shard, and finished output shards are skipped, so a rerun resumes and a requeued twin does no harm: + +```bash +stok tokenize-structures input_dir output_dir --preset large --text-column cif \ + --num-jobs 16 --job-index 3 --device cuda +``` + +`--chain` selects one chain from each structure (default: the single polymer chain); `--rows-per-shard` (file mode only) sets output shard size; `--device cpu` works, slowly. `stok tokenize-structures --help` lists every option. + +## Inputs and outputs + +- **In (Parquet mode):** shards of rows shaped like afdb-rep's: a non-null `id` (or `--id-column`) and mmCIF text in `cif` (or `--text-column`); `--keep-column` names columns to carry into the output. The mmCIF needs a contiguous `entity_poly_seq` for its polymer and `atom_site` rows — exactly what an AlphaFold DB or experimental mmCIF carries. +- **Out:** one Parquet shard per input shard, named for its stem (`00000.parquet` → `00000.parquet`), zstd, holding `sequence_id`, `sequence` (STōk's own read of the deposited sequence), `structure_tokens` (a list of int16, one per residue: `null` where the residue lacks a complete backbone), `coordinates` when `--include-coordinates` (N/CA/C per residue, `null` at unobserved atoms), then the kept columns. Every shard's footer carries a `stok.tokenizer` metadata key: `preset`, `checkpoint_sha256`, `codebook_sha256`, `codebook_size`, `stok_version` — the identity downstream training should pin. Rows the fixed policy refuses land in `rejected/.jsonl` as `{"sequence_id", "reason"}` (a chain under 25 or over 1280 residues, `chains_too_short`/`chains_too_long`; unparseable or ambiguous structures, `parse_failed`/`metadata_conflict`/...); a shard whose every row is rejected is still written, empty, so shard counts stay stable. + +## Compute + +One GPU, or a CPU at perhaps a hundredth the speed. The 2.5 GB checkpoint loads once per process in about a minute, and inference needs a few GB of GPU memory at any chain length. Inference is FP32, one structure at a time — deliberate, so token identity never depends on batching or sharding (see Gotchas) — which caps throughput near a second per structure on a single GPU: the reference experiment measured 0.76 s per structure with the smaller Lite preset on an AMD 8060S, and this tool's test tokenizes its two 243-residue fixtures inside a minute on one B200 after the load. A corpus of millions of structures is a many-GPU-hour run: split it with `--num-jobs`/`--job-index` and budget from the per-structure rate; the campaign that built this tool measured Large on B200s at scale in its `afdb-rep-stok-large` dataset skill. + +## Running it + +On a laptop, from the directory holding the input shards: + +```bash +podman run --rm --device nvidia.com/gpu=all -v "$PWD":/work -w /work \ + ghcr.io/brineylab/toolshed/stok:1.0.0 \ + stok tokenize-structures input_dir output_dir --preset large --text-column cif \ + --id-column id --include-coordinates --device cuda +``` + +On a Slurm cluster with pyxis: + +```bash +srun --gres=gpu:1 --container-image=ghcr.io#brineylab/toolshed/stok:1.0.0 \ + --container-mounts="$PWD":/work --container-workdir=/work \ + stok tokenize-structures input_dir output_dir --preset large --text-column cif \ + --id-column id --include-coordinates --device cuda +``` + +## Gotchas + +- The tokenization policy is fixed — full chains of 25–1280 residues, native sequence inputs, reference imputation, FP32 — and there is no pLDDT or length filter to configure: every structure that parses and fits is tokenized, and the rest are rejected with a reason in `rejected/`. Residues missing any backbone atom (N, CA, C or O) get a `null` token, not a dropped position: the token list stays aligned to the sequence. +- Inference runs one structure at a time in FP32 by design: STōk found mixed-length batches alter terminal angles, so token identity is kept independent of batching and sharding. Don't expect batch-throughput scaling; expect shard-parallel scaling. +- Token identity is deterministic per device (the CLI turns on `torch.use_deterministic_algorithms`): reruns on the same device reproduce exactly, but CPU and GPU may differ near ties, and GPU model generations may differ from each other. `codebook_sha256` and `checkpoint_sha256` in each shard's footer pin which tokenizer produced it; compare those, not raw token values, when checking provenance. +- Only the Large preset is in this image. `--preset lite` would need its own checkpoint, which is not baked in: with the image offline it fails rather than downloading. +- The tool's test fixture is two 243-residue chains — well past the policy's 25-residue floor — one with a whole residue unobserved and one with a single missing O, so it exercises the null-token and null-coordinate paths, plus a 20-residue chain that must be rejected `chains_too_short`, and decodes both accepted rows' tokens back to within 1.5 Å RMSD of the input coordinates. Chains over 1280 residues (rejected `chains_too_long`) are the one policy branch it does not cover. +- The PyPI package named `stok` is an unrelated older project; this image installs STōk from GitHub at the pinned commit above. +- Weights are MIT licensed (both licenses sit beside the checkpoint in `/opt/stok-gcp`); STōk itself is MIT. diff --git a/tools/stok/test/check.py b/tools/stok/test/check.py new file mode 100644 index 0000000..7535ced --- /dev/null +++ b/tools/stok/test/check.py @@ -0,0 +1,140 @@ +"""check.py : the fixture's known answer, plus the decode roundtrip. + +expected.json beside this script is the known answer, built with the fixture: +per-row sequence, token count, the exact null-token and null-coordinate +positions, the carried columns, the short row's rejection, and the tokenizer +identity (preset, checkpoint and codebook sha256, codebook size, stok +version). This checks the shard layout through stok's own read_source, +every field of expected.json against the Parquet actually written, and that +the tokens mean the structure: each accepted row's tokens are decoded with +the pretrained decoder and its codebook, superimposed (Kabsch) onto the +coordinates the tokenizer itself wrote, and must land within 1.5 A RMSD. +""" +import json +import os +import sys +from pathlib import Path + +RMSD_BOUND = 1.5 + + +def main() -> None: + if len(sys.argv) != 2: + sys.exit("usage: check.py TOKENS_DIR") + tokens = Path(sys.argv[1]) + here = Path(__file__).parent + expected = json.loads((here / "expected.json").read_text()) + + import numpy as np + import pyarrow.parquet as pq + import torch + from Bio.SVDSuperimposer import SVDSuperimposer + + from stok.data.format import read_source + from stok.models.decoder import load_pretrained_decoder + from stok.utils.codebook import load_codebook + from stok.utils.decoding import decode_structure_tokens + from stok.utils.pretrained import resolve_gcp_artifact + + source = read_source(tokens) + assert source["rows"] == 2, f"{source['rows']} rows" + assert [s["rows"] for s in source["shards"]] == [2, 0], source["shards"] + assert source["structure"] and source["coordinates"], "shards lack tokens or coordinates" + assert source["columns"] == [ + "sequence_id", "sequence", "structure_tokens", "coordinates", + "mean_plddt", "uniprot_accession", + ], source["columns"] + for field, value in expected["tokenizer"].items(): + actual = source["tokenizer"].get(field) + assert actual == value, f"tokenizer {field} is {actual!r}, expected {value!r}" + + rejected = [ + json.loads(line) + for line in (tokens / "rejected" / "00001.jsonl").read_text().splitlines() + ] + assert rejected == expected["rejected"], f"rejected rows {rejected}" + + table = pq.read_table(tokens / "00000.parquet").to_pylist() + assert [row["sequence_id"] for row in table] == [ + row["sequence_id"] for row in expected["rows"] + ], "accepted rows out of order" + + checkpoint = resolve_gcp_artifact("large", progress=False) + codebook = load_codebook(path=str(checkpoint)) + decoder = load_pretrained_decoder("large", progress=False) + identity = source["tokenizer"] + + record = {"identity": identity, "rejected": rejected, "rows": []} + for row, answer in zip(table, expected["rows"]): + sequence_id = row["sequence_id"] + tokens_row = row["structure_tokens"] + assert row["sequence"] == expected["source"]["sequence"], sequence_id + assert len(tokens_row) == answer["tokens"], sequence_id + nulls = [i for i, token in enumerate(tokens_row) if token is None] + assert nulls == answer["null_token_positions"], (sequence_id, nulls) + assert all(0 <= token < identity["codebook_size"] + for token in tokens_row if token is not None), sequence_id + assert (row["mean_plddt"] == answer["mean_plddt"] + and row["uniprot_accession"] == answer["uniprot_accession"]), sequence_id + coordinates = row["coordinates"] + assert len(coordinates) == answer["tokens"], sequence_id + coordinate_nulls = [ + i for i, residue in enumerate(coordinates) + if all(atom == [None, None, None] for atom in residue) + ] + assert coordinate_nulls == answer["coordinate_null_positions"], \ + (sequence_id, coordinate_nulls) + for i, residue in enumerate(coordinates): + if i not in coordinate_nulls: + assert all(atom is not None and len(atom) == 3 + for atom in residue), (sequence_id, i) + + indices = torch.tensor( + [token if token is not None else -1 for token in tokens_row], + dtype=torch.long, + ) + decoded = decode_structure_tokens( + decoder, codebook, indices.unsqueeze(0), + residue_mask=torch.ones((1, len(indices)), dtype=torch.bool), + )[0].numpy() + truth = np.full((len(coordinates), 3, 3), np.nan, dtype=np.float32) + for i, residue in enumerate(coordinates): + for j, atom in enumerate(residue): + if atom is not None: + truth[i, j] = atom + keep = (np.isfinite(decoded).all(axis=(1, 2)) + & np.isfinite(truth).all(axis=(1, 2))) + # A token exists exactly where the residue's whole backbone was + # observed, and the coordinates column keeps N/CA/C beyond that, so + # the superimposed set is the complete residues. + assert keep.sum() == answer["tokens"] - len(nulls), (sequence_id, keep.sum()) + superimposer = SVDSuperimposer() + superimposer.set(decoded[keep].reshape(-1, 3), truth[keep].reshape(-1, 3)) + superimposer.run() + rmsd = float(superimposer.get_rms()) + assert rmsd < RMSD_BOUND, f"{sequence_id}: decode RMSD {rmsd:.3f} A" + record["rows"].append({ + "sequence_id": sequence_id, + "tokens": answer["tokens"], + "null_token_positions": nulls, + "coordinate_null_positions": coordinate_nulls, + "decode_rmsd": round(rmsd, 3), + }) + print(f"ok: {sequence_id}: {answer['tokens']} tokens, " + f"{len(nulls)} null, {len(coordinate_nulls)} coordinate-null, " + f"decode RMSD {rmsd:.3f} A") + + print(f"ok: identity preset {identity['preset']}, codebook " + f"{identity['codebook_size']}, stok {identity['stok_version']}, " + f"checkpoint {identity['checkpoint_sha256'][:12]}..., " + f"codebook {identity['codebook_sha256'][:12]}...") + print(f"ok: rejected {rejected[0]['sequence_id']} ({rejected[0]['reason']})") + out = os.environ.get("FLYLOFT_OUT") + if out and Path(out).is_dir(): + (Path(out) / "stok-test-record.json").write_text( + json.dumps(record, indent=2) + "\n" + ) + + +if __name__ == "__main__": + main() diff --git a/tools/stok/test/expected.json b/tools/stok/test/expected.json new file mode 100644 index 0000000..adff67c --- /dev/null +++ b/tools/stok/test/expected.json @@ -0,0 +1,51 @@ +{ + "source": { + "pdb": "7YPD_B.pdb", + "sha256": "77034053136ddd53c58ab55bda43668bb8c2f5546549129264384f59eba4e537", + "residues": 243, + "sequence": "SATVYVISGATRGIGFALTSLLAQRDNVLIFAGARDPAKALPLQALAAATGKVIPVKLESANEEDAAALAKLVKEKAGKVDFLLANAGVCELNKPVLSTPSATFVDHFTVNTLGPLTLFQQFYSLLTESSSPRFFVTSSAGGSTTYVSMAPDMDLAPYGISKAAVNHLVAHIARKFGAKDGLVAAVVHPGLVATDTRPFLEAAGLEHISPDESAAALVKIFDEAKRETHGGKFLSYDGTEIPW", + "author_numbering": "4..256 (renumbered 1..243)", + "drop_residue": 100, + "drop_atom": [ + 150, + "O" + ], + "short_keep": 20 + }, + "rows": [ + { + "sequence_id": "AF-A0A0FXCOMPLETE-F1", + "mean_plddt": 91.3, + "uniprot_accession": "A0A0FXCOMPLETE", + "tokens": 243, + "null_token_positions": [], + "coordinate_null_positions": [] + }, + { + "sequence_id": "AF-A0A0FXGAPPED-F1", + "mean_plddt": 85.7, + "uniprot_accession": "A0A0FXGAPPED", + "tokens": 243, + "null_token_positions": [ + 96, + 146 + ], + "coordinate_null_positions": [ + 96 + ] + } + ], + "rejected": [ + { + "sequence_id": "AF-A0A0FXSHORT-F1", + "reason": "chains_too_short" + } + ], + "tokenizer": { + "preset": "large", + "checkpoint_sha256": "7d1d43950a29834e7f702409bf957e9ffb75eb3cb3952074ba4c545bb4130eaf", + "codebook_size": 4096, + "stok_version": "0.1.4", + "codebook_sha256": "29d21936c951fc1421ce9bc0bcb4ebb3d6dc2391ede165d03db0be05cb958141" + } +} diff --git a/tools/stok/test/fixture/00000.parquet b/tools/stok/test/fixture/00000.parquet new file mode 100644 index 0000000..dfd8062 Binary files /dev/null and b/tools/stok/test/fixture/00000.parquet differ diff --git a/tools/stok/test/fixture/00001.parquet b/tools/stok/test/fixture/00001.parquet new file mode 100644 index 0000000..54ca156 Binary files /dev/null and b/tools/stok/test/fixture/00001.parquet differ diff --git a/tools/stok/test/run.sh b/tools/stok/test/run.sh new file mode 100644 index 0000000..10094f4 --- /dev/null +++ b/tools/stok/test/run.sh @@ -0,0 +1,19 @@ +set -euo pipefail +cd "$(dirname "$0")" +stamp() { date -u +"%Y-%m-%dT%H:%M:%SZ"; } +echo "[$(stamp)] $(hostname): $(nvidia-smi -L 2>/dev/null || echo 'no gpu visible')" +out=$(mktemp -d) +# The fixture is this campaign's build of STok's own CAMEO test structure +# 7YPD_B: shard 00000 holds two accepted 243-residue rows (one complete, one +# with a residue unobserved and one missing O, exercising the null-token and +# null-coordinate paths), shard 00001 a 20-residue row the fixed policy must +# reject -- well past the policy's 25-residue floor, under its 1280 ceiling. +start=$(date +%s) +echo "[$(stamp)] tokenizing the fixture on cuda" +stok tokenize-structures fixture "$out/tokens" --preset large \ + --text-column cif --id-column id \ + --keep-column mean_plddt --keep-column uniprot_accession \ + --include-coordinates --device cuda +echo "[$(stamp)] tokenization finished in $(( $(date +%s) - start ))s; checking the answer" +python3 -I check.py "$out/tokens" +echo "[$(stamp)] all checks passed, $(( $(date +%s) - start ))s from tokenize start"