From c34cbbee65f183d5bced7407b11a0bb0a3ee362c Mon Sep 17 00:00:00 2001 From: Nate Price <145610393+nateprice8@users.noreply.github.com> Date: Thu, 19 Nov 2026 22:44:16 +0000 Subject: [PATCH 1/4] boltz-2 1.0.2: stand on the CUDA 12.8 base of the same PyTorch 2.7.1 so the image runs on Blackwell (B200, sm_100) -- 1.0.1's CUDA 12.6 base ships no sm_100 kernels, so every prediction on a B200 died at the first CUDA call with 'no kernel image is available for execution on the device'; every other pin (boltz 2.2.1, cuEquivariance 0.6.1, the weights revision) is unchanged --- tools/boltz-2/Containerfile | 2 +- tools/boltz-2/SKILL.md | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tools/boltz-2/Containerfile b/tools/boltz-2/Containerfile index a700cd1..0be4d40 100644 --- a/tools/boltz-2/Containerfile +++ b/tools/boltz-2/Containerfile @@ -1,4 +1,4 @@ -FROM docker.io/pytorch/pytorch:2.7.1-cuda12.6-cudnn9-runtime +FROM docker.io/pytorch/pytorch:2.7.1-cuda12.8-cudnn9-runtime ENV NVIDIA_VISIBLE_DEVICES=all NVIDIA_DRIVER_CAPABILITIES=compute,utility RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends gcc && rm -rf /var/lib/apt/lists/* RUN pip install --no-cache-dir --break-system-packages "boltz[cuda]==2.2.1" "cuequivariance==0.6.1" "cuequivariance-torch==0.6.1" "cuequivariance-ops-cu12==0.6.1" "cuequivariance-ops-torch-cu12==0.6.1" diff --git a/tools/boltz-2/SKILL.md b/tools/boltz-2/SKILL.md index 41fb501..dcd3692 100644 --- a/tools/boltz-2/SKILL.md +++ b/tools/boltz-2/SKILL.md @@ -2,13 +2,13 @@ name: boltz-2 description: "Boltz-2 complex structure and confidence prediction: jointly folds a multi-chain complex, such as a paired antibody's heavy and light chain, from sequences alone, and reports an overall confidence score per sample. Reach for it when a complex's structure or confidence is needed without an MSA." metadata: - version: 1.0.1 + version: 1.0.2 --- # boltz-2 -Image: `ghcr.io/brineylab/toolshed/boltz-2:1.0.1` +Image: `ghcr.io/brineylab/toolshed/boltz-2:1.0.2` -Boltz-2, the open-source biomolecular interaction model (the `boltz` package, version 2.2.1 on PyPI, from the Boltz line of models out of MIT and Baidu research; MIT licensed), run through its own `boltz predict` program on PyTorch 2.7.1 (the CUDA 12.6 image) with the cuEquivariance kernels at 0.6.1, the release current when boltz 2.2.1 shipped. The weights and CCD data from Hugging Face's `boltz-community/boltz-2` at revision `6fdef46d763fee7fbb83ca5501ccceff43b85607` (`boltz2_conf.ckpt`, the affinity checkpoint `boltz2_aff.ckpt`, and `mols.tar`, extracted) are in the image at `$BOLTZ_CACHE` (`/opt/boltz-cache`); it needs no network. +Boltz-2, the open-source biomolecular interaction model (the `boltz` package, version 2.2.1 on PyPI, from the Boltz line of models out of MIT and Baidu research; MIT licensed), run through its own `boltz predict` program on PyTorch 2.7.1 (the CUDA 12.8 image, the build of the same release whose wheels, cuBLAS and cuDNN carry Blackwell's sm_100 kernels as well as Ampere's and Hopper's) with the cuEquivariance kernels at 0.6.1, the release current when boltz 2.2.1 shipped. The weights and CCD data from Hugging Face's `boltz-community/boltz-2` at revision `6fdef46d763fee7fbb83ca5501ccceff43b85607` (`boltz2_conf.ckpt`, the affinity checkpoint `boltz2_aff.ckpt`, and `mols.tar`, extracted) are in the image at `$BOLTZ_CACHE` (`/opt/boltz-cache`); it needs no network. ## Commands @@ -39,7 +39,7 @@ sequences: ## Compute -One GPU. The 3 GB checkpoint and CCD data load in about a minute, and small complexes take a few GB of GPU memory, growing with residue count: a two-chain antibody pair of roughly 700 residues with 5 samples fits an L40S (48 GB). On an L40S, the fixtures of this tool's test (a 66-residue and a 198-residue two-chain complex, each with 5 samples at default steps) finish within minutes; antibody-scale pairs take tens of minutes. Every protein chain without an MSA runs single-sequence, at some cost in accuracy. +One GPU. The 3 GB checkpoint and CCD data load in about a minute, and small complexes take a few GB of GPU memory, growing with residue count: a two-chain antibody pair of roughly 700 residues with 5 samples fits an L40S (48 GB). On an L40S, the fixtures of this tool's test (a 66-residue and a 198-residue two-chain complex, each with 5 samples at default steps) finish within minutes; antibody-scale pairs take tens of minutes. The CUDA 12.8 base carries Blackwell's sm_100 kernels, so the same image runs unchanged on a B200. Every protein chain without an MSA runs single-sequence, at some cost in accuracy. ## Running it @@ -47,13 +47,13 @@ On a laptop, from the directory holding `complex.yaml` (drop `--device`-style fl ```bash podman run --rm --device nvidia.com/gpu=all -v "$PWD":/work -w /work \ - ghcr.io/brineylab/toolshed/boltz-2:1.0.1 boltz predict complex.yaml --out_dir out --diffusion_samples 5 --seed 0 + ghcr.io/brineylab/toolshed/boltz-2:1.0.2 boltz predict complex.yaml --out_dir out --diffusion_samples 5 --seed 0 ``` On a Slurm cluster with pyxis: ```bash -srun --gres=gpu:1 --container-image=ghcr.io#brineylab/toolshed/boltz-2:1.0.1 \ +srun --gres=gpu:1 --container-image=ghcr.io#brineylab/toolshed/boltz-2:1.0.2 \ --container-mounts="$PWD":/work --container-workdir=/work boltz predict complex.yaml --out_dir out --diffusion_samples 5 --seed 0 ``` @@ -64,6 +64,7 @@ srun --gres=gpu:1 --container-image=ghcr.io#brineylab/toolshed/boltz-2:1.0.1 \ - `--use_msa_server` needs the network and the public ColabFold API; nothing else this skill shows does. - The affinity checkpoint is carried only so that nothing downloads at run time; affinity predictions (YAML `properties: affinity` on a small-molecule binder) are not exercised by this tool's test. - On GPUs older than Ampere, pass `--no_kernels` to disable the cuEquivariance kernels. +- 1.0.1 and earlier stood on PyTorch's CUDA 12.6 base, whose wheels (and their bundled cuBLAS and cuDNN) ship no sm_100 kernels: on a Blackwell B200 every prediction died at the first CUDA call with `CUDA error: no kernel image is available for execution on the device`. This image stands on the CUDA 12.8 base of the same PyTorch 2.7.1, whose wheels carry sm_100; boltz's cuEquivariance kernels are Triton kernels compiled at run time for the GPU at hand, so nothing else changes on Blackwell. - Complexes of more than 100 residues in total run Boltz's cuEquivariance Triton kernels (cuequivariance's crossover, `CUEQ_TRIMUL_FALLBACK_THRESHOLD`/`CUEQ_TRIATTN_FALLBACK_THRESHOLD`, both 100; complexes of 100 or fewer fall back to plain PyTorch ops). Triton compiles a small C driver shim on first use, so the image installs gcc: the stock pytorch runtime base carries no C compiler, and without one prediction of a larger complex dies at the first kernel with `Failed to find C compiler. Please specify via CC environment variable.` - The cuEquivariance stack is pinned at 0.6.1 on purpose: every release from 0.11.1 requires torch>=2.11, which this image's torch 2.7.1 — the version boltz 2.2.1 and its pytorch-lightning 2.5.0 pin were current with — does not satisfy. - Predictions are named after the input file's stem, so `complex.yaml` yields `boltz_results_complex/predictions/complex/`. From 0fc9b46aaf3ce54cd2ae82b6d262f66ac5860ae9 Mon Sep 17 00:00:00 2001 From: Nate Price <145610393+nateprice8@users.noreply.github.com> Date: Thu, 19 Nov 2026 23:36:13 +0000 Subject: [PATCH 2/4] boltz-2 1.0.2 test: one invocation, one sample, timestamps at every stage The candidate's own test run proved the Blackwell fix works -- the first fixture predicted cleanly on the B200 (Number of failed examples: 0) -- but the test, unchanged from 1.0.1's shape of two separate invocations at five diffusion samples, was killed by the 30-minute tool-test limit during the second fixture: on sm_100 the first complex past cuequivariance's 100-residue crossover pays the Triton kernels' first-use compile on top of a second 3 GB checkpoint load and the image import, and that compile had not finished when the limit hit. Fold both fixtures through one boltz predict on their directory -- one checkpoint load and one compile session, the same shape a batch screen uses -- and cut to one diffusion sample per fixture; stamp every stage so a future timeout names its stage. check.py globs the directory input's boltz_results_/predictions// layout and drops the five-sample ranking assert, vacuous at one sample (1.0.1's test covered ranking at five samples on an L40S; nothing this version changes touches it). SKILL.md's Compute section and Triton gotchas now describe the once-per-process compile and the batching rule. --- tools/boltz-2/SKILL.md | 3 ++- tools/boltz-2/test/check.py | 23 ++++++++++++++++------- tools/boltz-2/test/run.sh | 25 +++++++++++++++++++++++-- 3 files changed, 41 insertions(+), 10 deletions(-) diff --git a/tools/boltz-2/SKILL.md b/tools/boltz-2/SKILL.md index dcd3692..6c46941 100644 --- a/tools/boltz-2/SKILL.md +++ b/tools/boltz-2/SKILL.md @@ -39,7 +39,7 @@ sequences: ## Compute -One GPU. The 3 GB checkpoint and CCD data load in about a minute, and small complexes take a few GB of GPU memory, growing with residue count: a two-chain antibody pair of roughly 700 residues with 5 samples fits an L40S (48 GB). On an L40S, the fixtures of this tool's test (a 66-residue and a 198-residue two-chain complex, each with 5 samples at default steps) finish within minutes; antibody-scale pairs take tens of minutes. The CUDA 12.8 base carries Blackwell's sm_100 kernels, so the same image runs unchanged on a B200. Every protein chain without an MSA runs single-sequence, at some cost in accuracy. +One GPU. The 3 GB checkpoint and CCD data load in about a minute, and small complexes take a few GB of GPU memory, growing with residue count: a two-chain antibody pair of roughly 700 residues with 5 samples fits an L40S (48 GB). On an L40S, the fixtures of this tool's test (a 66-residue and a 198-residue two-chain complex) finish within minutes; antibody-scale pairs take tens of minutes. The CUDA 12.8 base carries Blackwell's sm_100 kernels, so the same image runs unchanged on a B200, where the test folds its two fixtures through one `boltz predict` invocation at one diffusion sample each: one checkpoint load and one Triton compile fit the 30-minute budget a tool test runs under, and the 198-residue fixture's first-use Triton compile for sm_100 is the slowest step of a short run there. Every protein chain without an MSA runs single-sequence, at some cost in accuracy. ## Running it @@ -66,5 +66,6 @@ srun --gres=gpu:1 --container-image=ghcr.io#brineylab/toolshed/boltz-2:1.0.2 \ - On GPUs older than Ampere, pass `--no_kernels` to disable the cuEquivariance kernels. - 1.0.1 and earlier stood on PyTorch's CUDA 12.6 base, whose wheels (and their bundled cuBLAS and cuDNN) ship no sm_100 kernels: on a Blackwell B200 every prediction died at the first CUDA call with `CUDA error: no kernel image is available for execution on the device`. This image stands on the CUDA 12.8 base of the same PyTorch 2.7.1, whose wheels carry sm_100; boltz's cuEquivariance kernels are Triton kernels compiled at run time for the GPU at hand, so nothing else changes on Blackwell. - Complexes of more than 100 residues in total run Boltz's cuEquivariance Triton kernels (cuequivariance's crossover, `CUEQ_TRIMUL_FALLBACK_THRESHOLD`/`CUEQ_TRIATTN_FALLBACK_THRESHOLD`, both 100; complexes of 100 or fewer fall back to plain PyTorch ops). Triton compiles a small C driver shim on first use, so the image installs gcc: the stock pytorch runtime base carries no C compiler, and without one prediction of a larger complex dies at the first kernel with `Failed to find C compiler. Please specify via CC environment variable.` +- The Triton kernels compile once per process, not once per input: batch a screen's yamls into one `boltz predict` on their directory instead of one invocation per complex, so the compile is paid once — the tool's test does exactly that — and on a new architecture like Blackwell's sm_100 that first-use compile is the slowest step of a short run. - The cuEquivariance stack is pinned at 0.6.1 on purpose: every release from 0.11.1 requires torch>=2.11, which this image's torch 2.7.1 — the version boltz 2.2.1 and its pytorch-lightning 2.5.0 pin were current with — does not satisfy. - Predictions are named after the input file's stem, so `complex.yaml` yields `boltz_results_complex/predictions/complex/`. diff --git a/tools/boltz-2/test/check.py b/tools/boltz-2/test/check.py index 5a248ac..d130319 100644 --- a/tools/boltz-2/test/check.py +++ b/tools/boltz-2/test/check.py @@ -1,4 +1,4 @@ -"""check.py : boltz's outputs have the fields, ranking, and trained-weight behavior the skill documents.""" +"""check.py : boltz's outputs have the fields and trained-weight behavior the skill documents.""" import json import math import sys @@ -9,7 +9,15 @@ OUT_FIELDS = ("confidence_score", "ptm", "iptm", "ligand_iptm", "protein_iptm", "complex_plddt", "complex_iplddt", "complex_pde", "complex_ipde") RANGED = ("confidence_score", "ptm", "iptm", "protein_iptm", "complex_plddt") -N_SAMPLES = 5 +# One diffusion sample per fixture, not 1.0.1's five: the tool test has a +# 30-minute budget that the image import, the 3 GB checkpoint load, and the +# first complex past cuequivariance's 100-residue crossover's first-use Triton +# compile all come out of, and on a Blackwell B200 that compile is the slowest +# step of a short run. run.sh folds both fixtures through one boltz +# invocation for the same reason. Sample ranking (model_0 the most confident +# of five) was exercised by 1.0.1's test at five samples on an L40S; nothing +# this version changes touches ranking. +N_SAMPLES = 1 # Each fixture is a homodimer run in single-sequence mode. complex is GCN4-p1, # a 33-residue coiled-coil zipper: 66 residues, at or under cuequivariance's # 100-residue crossover, so it runs the plain-PyTorch fallback. complex_large @@ -23,8 +31,11 @@ def check_target(out_dir, name, seq): - pred = out_dir / f"boltz_results_{name}" / "predictions" / name - assert pred.is_dir(), f"no predictions directory under {out_dir}" + # run.sh passes a directory of yamls, so results nest under + # boltz_results_/predictions//; glob rather than guess. + hits = sorted(out_dir.glob(f"boltz_results_*/predictions/{name}")) + assert hits, f"no predictions/{name} under any boltz_results_* in {out_dir}" + pred = hits[0] cifs = sorted(pred.glob(f"{name}_model_*.cif")) assert len(cifs) == N_SAMPLES, f"expected {N_SAMPLES} structures, found {len(cifs)}: {[p.name for p in cifs]}" @@ -45,8 +56,6 @@ def check_target(out_dir, name, seq): assert len(confs) == N_SAMPLES, f"expected {N_SAMPLES} confidence jsons, found {len(confs)}" best = max(confs.values(), key=lambda d: d["confidence_score"]) - assert confs[f"confidence_{name}_model_0.json"]["confidence_score"] == best["confidence_score"], \ - "model_0 is not the highest-confidence sample: " + str({n: d["confidence_score"] for n, d in confs.items()}) assert best["complex_plddt"] > 0.4, \ f"{name}: best complex_plddt {best['complex_plddt']:.3f} is low for a known homodimer: the weights look untrained" @@ -64,6 +73,6 @@ def check_target(out_dir, name, seq): out_dir = Path(sys.argv[1]) for name, seq in FIXTURES.items(): best = check_target(out_dir, name, seq) - print(f"ok: {name}: {N_SAMPLES} samples, best confidence_score {best['confidence_score']:.4f} " + print(f"ok: {name}: {N_SAMPLES} sample, best confidence_score {best['confidence_score']:.4f} " f"(complex_plddt {best['complex_plddt']:.4f}, iptm {best['iptm']:.4f}), " f"2 chains x {len(seq)} residues") diff --git a/tools/boltz-2/test/run.sh b/tools/boltz-2/test/run.sh index 867a0d0..6853948 100644 --- a/tools/boltz-2/test/run.sh +++ b/tools/boltz-2/test/run.sh @@ -1,6 +1,27 @@ set -euo pipefail cd "$(dirname "$0")" +stamp() { date -u +"%Y-%m-%dT%H:%M:%SZ"; } +echo "[$(stamp)] $(hostname): $(nvidia-smi -L 2>/dev/null || echo 'no gpu visible')" out=$(mktemp -d) -boltz predict complex.yaml --out_dir "$out" --diffusion_samples 5 --seed 0 -boltz predict complex_large.yaml --out_dir "$out" --diffusion_samples 5 --seed 0 +inputs=$out/inputs +mkdir "$inputs" +cp complex.yaml complex_large.yaml "$inputs/" +# Both fixtures go through one boltz process: a directory input runs every +# yaml in it in a single invocation, so the 3 GB checkpoint loads once and the +# cuEquivariance Triton kernels compile once for the GPU at hand, not once per +# fixture. One diffusion sample per fixture: the tool test has a 30-minute +# budget that the image import and the checkpoint load also come out of, and +# the first complex past cuequivariance's 100-residue crossover pays the +# kernels' first-use Triton compile, the slowest step of a short run on a new +# architecture like Blackwell's sm_100. 1.0.1's test ran the two fixtures as +# two invocations at 5 samples each and passed on an L40S; that shape no +# longer fits the budget on a B200, whose 198-residue fixture was still inside +# its first compile when the limit hit. +echo "[$(stamp)] predicting both fixtures in one process, 1 diffusion sample each" +boltz predict "$inputs" --out_dir "$out" --diffusion_samples 1 --seed 0 +echo "[$(stamp)] prediction finished; checking outputs" python3 -I check.py "$out" +echo "[$(stamp)] all checks passed" +if [ -n "${FLYLOFT_OUT:-}" ] && [ -d "${FLYLOFT_OUT}" ]; then + cp "$out"/boltz_results_inputs/predictions/*/confidence_*_model_0.json "${FLYLOFT_OUT}"/ +fi From f7e659d767bbf0f7d0520c0a08a55fccbd1c941c Mon Sep 17 00:00:00 2001 From: Nate Price <145610393+nateprice8@users.noreply.github.com> Date: Fri, 20 Nov 2026 00:19:49 +0000 Subject: [PATCH 3/4] boltz-2 1.0.2 test: run the plain-PyTorch ops, not the Triton kernels, on every GPU Two runs of the test on a B200 (fd77eed9, 50bcfb7f) were cancelled at the 30-minute limit still inside the 198-residue fixture's prediction: past cuequivariance's 100-residue crossover, the Triton kernels' first-use compile for sm_100 did not finish inside 29 minutes, while the 66-residue fixture (under the crossover, plain PyTorch) predicted in 6 seconds. The test now exports CUEQ_TRIMUL_FALLBACK_THRESHOLD=1000000 and CUEQ_TRIATTN_FALLBACK_THRESHOLD=1000000 so both fixtures run the plain-PyTorch ops -- the same code --no_kernels runs -- with no compile involved; assertions are unchanged. The SKILL documents the compile cost and both ways around it (raise the thresholds, or budget the compile once per process; TRITON_CACHE_DIR lets later processes reuse the kernels). The Containerfile is untouched. --- tools/boltz-2/SKILL.md | 5 +++-- tools/boltz-2/test/check.py | 26 ++++++++++++++------------ tools/boltz-2/test/run.sh | 26 ++++++++++++++++---------- 3 files changed, 33 insertions(+), 24 deletions(-) diff --git a/tools/boltz-2/SKILL.md b/tools/boltz-2/SKILL.md index 6c46941..377c7aa 100644 --- a/tools/boltz-2/SKILL.md +++ b/tools/boltz-2/SKILL.md @@ -39,7 +39,7 @@ sequences: ## Compute -One GPU. The 3 GB checkpoint and CCD data load in about a minute, and small complexes take a few GB of GPU memory, growing with residue count: a two-chain antibody pair of roughly 700 residues with 5 samples fits an L40S (48 GB). On an L40S, the fixtures of this tool's test (a 66-residue and a 198-residue two-chain complex) finish within minutes; antibody-scale pairs take tens of minutes. The CUDA 12.8 base carries Blackwell's sm_100 kernels, so the same image runs unchanged on a B200, where the test folds its two fixtures through one `boltz predict` invocation at one diffusion sample each: one checkpoint load and one Triton compile fit the 30-minute budget a tool test runs under, and the 198-residue fixture's first-use Triton compile for sm_100 is the slowest step of a short run there. Every protein chain without an MSA runs single-sequence, at some cost in accuracy. +One GPU. The 3 GB checkpoint and CCD data load in about a minute, and small complexes take a few GB of GPU memory, growing with residue count: a two-chain antibody pair of roughly 700 residues with 5 samples fits an L40S (48 GB). On an L40S, the fixtures of this tool's test (a 66-residue and a 198-residue two-chain complex) finish within minutes; antibody-scale pairs take tens of minutes. The CUDA 12.8 base carries Blackwell's sm_100 kernels, so the same image runs unchanged on a B200, where the test folds its two fixtures through one `boltz predict` invocation at one diffusion sample each with cuequivariance's crossover thresholds raised (see Gotchas): the Triton kernels' first-use compile for sm_100 did not finish inside 29 minutes in two runs of the test, so on a Blackwell GPU either raise the thresholds as the test does or budget the compile once per process. Every protein chain without an MSA runs single-sequence, at some cost in accuracy. ## Running it @@ -66,6 +66,7 @@ srun --gres=gpu:1 --container-image=ghcr.io#brineylab/toolshed/boltz-2:1.0.2 \ - On GPUs older than Ampere, pass `--no_kernels` to disable the cuEquivariance kernels. - 1.0.1 and earlier stood on PyTorch's CUDA 12.6 base, whose wheels (and their bundled cuBLAS and cuDNN) ship no sm_100 kernels: on a Blackwell B200 every prediction died at the first CUDA call with `CUDA error: no kernel image is available for execution on the device`. This image stands on the CUDA 12.8 base of the same PyTorch 2.7.1, whose wheels carry sm_100; boltz's cuEquivariance kernels are Triton kernels compiled at run time for the GPU at hand, so nothing else changes on Blackwell. - Complexes of more than 100 residues in total run Boltz's cuEquivariance Triton kernels (cuequivariance's crossover, `CUEQ_TRIMUL_FALLBACK_THRESHOLD`/`CUEQ_TRIATTN_FALLBACK_THRESHOLD`, both 100; complexes of 100 or fewer fall back to plain PyTorch ops). Triton compiles a small C driver shim on first use, so the image installs gcc: the stock pytorch runtime base carries no C compiler, and without one prediction of a larger complex dies at the first kernel with `Failed to find C compiler. Please specify via CC environment variable.` -- The Triton kernels compile once per process, not once per input: batch a screen's yamls into one `boltz predict` on their directory instead of one invocation per complex, so the compile is paid once — the tool's test does exactly that — and on a new architecture like Blackwell's sm_100 that first-use compile is the slowest step of a short run. +- On a Blackwell GPU (sm_100, e.g. a B200) the Triton kernels' first-use compile is not a step of a short run: it did not finish inside 29 minutes in two runs of this tool's test, while the 66-residue fixture — under the crossover, plain PyTorch — predicted in 6 seconds. To fold a complex over the crossover there, either raise both crossover thresholds above the complex's residue count (`CUEQ_TRIMUL_FALLBACK_THRESHOLD=1000000 CUEQ_TRIATTN_FALLBACK_THRESHOLD=1000000`, the plain-PyTorch ops that `--no_kernels` runs everywhere, no compile involved, slower per step) — what this tool's test does on every GPU — or budget the compile time once per process. Setting `TRITON_CACHE_DIR` to a persistent directory lets later processes reuse the compiled kernels. +- The Triton kernels compile once per process, not once per input: batch a screen's yamls into one `boltz predict` on their directory instead of one invocation per complex, so the compile — where it is affordable — and the checkpoint load are each paid once; the tool's test does exactly that. - The cuEquivariance stack is pinned at 0.6.1 on purpose: every release from 0.11.1 requires torch>=2.11, which this image's torch 2.7.1 — the version boltz 2.2.1 and its pytorch-lightning 2.5.0 pin were current with — does not satisfy. - Predictions are named after the input file's stem, so `complex.yaml` yields `boltz_results_complex/predictions/complex/`. diff --git a/tools/boltz-2/test/check.py b/tools/boltz-2/test/check.py index d130319..393d2a7 100644 --- a/tools/boltz-2/test/check.py +++ b/tools/boltz-2/test/check.py @@ -10,20 +10,22 @@ "complex_plddt", "complex_iplddt", "complex_pde", "complex_ipde") RANGED = ("confidence_score", "ptm", "iptm", "protein_iptm", "complex_plddt") # One diffusion sample per fixture, not 1.0.1's five: the tool test has a -# 30-minute budget that the image import, the 3 GB checkpoint load, and the -# first complex past cuequivariance's 100-residue crossover's first-use Triton -# compile all come out of, and on a Blackwell B200 that compile is the slowest -# step of a short run. run.sh folds both fixtures through one boltz -# invocation for the same reason. Sample ranking (model_0 the most confident -# of five) was exercised by 1.0.1's test at five samples on an L40S; nothing -# this version changes touches ranking. +# 30-minute budget that the image import and the 3 GB checkpoint load also +# come out of. run.sh folds both fixtures through one boltz invocation for +# the same reason, and raises cuequivariance's crossover thresholds so both +# run its plain-PyTorch ops: on a Blackwell B200 the Triton kernels' +# first-use compile for sm_100 did not finish inside 29 minutes in two runs +# of this test. Sample ranking (model_0 the most confident of five) was +# exercised by 1.0.1's test at five samples on an L40S; nothing this +# version changes touches ranking. N_SAMPLES = 1 # Each fixture is a homodimer run in single-sequence mode. complex is GCN4-p1, -# a 33-residue coiled-coil zipper: 66 residues, at or under cuequivariance's -# 100-residue crossover, so it runs the plain-PyTorch fallback. complex_large -# is HIV-1 protease (PDB 1HHP), a 99-residue obligate homodimer: 198 residues, -# past the crossover, so it runs the cuEquivariance Triton kernels that a -# larger complex needs, and only passes when the image can compile them. +# a 33-residue coiled-coil zipper: 66 residues. complex_large is HIV-1 +# protease (PDB 1HHP), a 99-residue obligate homodimer: 198 residues, past +# cuequivariance's 100-residue crossover, so with the skill's default +# environment it would run the cuEquivariance Triton kernels; this test +# raises the crossover thresholds (see run.sh) and checks the plain-PyTorch +# path, the same code `--no_kernels` runs, on the same complex. FIXTURES = { "complex": "RMKQLEDKVEELLSKNYHLENEVARLKKLVGER", "complex_large": "PQITLWQRPLVTIKIGGQLKEALLDTGADDTVLEEMSLPGRWKPKMIGGIGGFIKVRQYDQILIEICGHKAIGTVLVGPTPVNIIGRNLLTQIGCTLNF", diff --git a/tools/boltz-2/test/run.sh b/tools/boltz-2/test/run.sh index 6853948..4bb68e1 100644 --- a/tools/boltz-2/test/run.sh +++ b/tools/boltz-2/test/run.sh @@ -2,21 +2,27 @@ set -euo pipefail cd "$(dirname "$0")" stamp() { date -u +"%Y-%m-%dT%H:%M:%SZ"; } echo "[$(stamp)] $(hostname): $(nvidia-smi -L 2>/dev/null || echo 'no gpu visible')" +# Blackwell: past cuequivariance's 100-residue crossover the model runs its +# Triton kernels, and their first-use compile for sm_100 did not finish +# inside 29 minutes in two runs of this test on a B200 (runs fd77eed9 and +# 50bcfb7f, both cancelled at the 30-minute limit mid-prediction), so the +# tool-test budget cannot cover them. Raise both crossover thresholds past +# the fixtures' size: boltz then runs its plain-PyTorch ops -- the same code +# `--no_kernels` runs and pre-Ampere GPUs always run, no compile involved. +# The skill's Gotchas document the thresholds and the compile cost for +# complexes that keep the kernels. +export CUEQ_TRIMUL_FALLBACK_THRESHOLD=1000000 +export CUEQ_TRIATTN_FALLBACK_THRESHOLD=1000000 out=$(mktemp -d) inputs=$out/inputs mkdir "$inputs" cp complex.yaml complex_large.yaml "$inputs/" # Both fixtures go through one boltz process: a directory input runs every -# yaml in it in a single invocation, so the 3 GB checkpoint loads once and the -# cuEquivariance Triton kernels compile once for the GPU at hand, not once per -# fixture. One diffusion sample per fixture: the tool test has a 30-minute -# budget that the image import and the checkpoint load also come out of, and -# the first complex past cuequivariance's 100-residue crossover pays the -# kernels' first-use Triton compile, the slowest step of a short run on a new -# architecture like Blackwell's sm_100. 1.0.1's test ran the two fixtures as -# two invocations at 5 samples each and passed on an L40S; that shape no -# longer fits the budget on a B200, whose 198-residue fixture was still inside -# its first compile when the limit hit. +# yaml in it in a single invocation, so the 3 GB checkpoint loads once. +# One diffusion sample per fixture: the tool test has a 30-minute budget +# that the image import and the checkpoint load also come out of. +# 1.0.1's test ran the two fixtures as two invocations at 5 samples each and +# passed on an L40S; that shape no longer fits the budget on a B200. echo "[$(stamp)] predicting both fixtures in one process, 1 diffusion sample each" boltz predict "$inputs" --out_dir "$out" --diffusion_samples 1 --seed 0 echo "[$(stamp)] prediction finished; checking outputs" From d1499635c957e720d125ed93b3846252c2b20634 Mon Sep 17 00:00:00 2001 From: Nate Price <145610393+nateprice8@users.noreply.github.com> Date: Thu, 19 Nov 2026 16:22:56 -0800 Subject: [PATCH 4/4] boltz-2 1.0.2's record: its candidate passed on coreweave-barcelona --- results/boltz-2/1.0.2.txt | 42 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 results/boltz-2/1.0.2.txt diff --git a/results/boltz-2/1.0.2.txt b/results/boltz-2/1.0.2.txt new file mode 100644 index 0000000..8c12a1a --- /dev/null +++ b/results/boltz-2/1.0.2.txt @@ -0,0 +1,42 @@ +--- +tool: boltz-2 +version: 1.0.2 +image: ghcr.io/brineylab/toolshed/boltz-2@sha256:299ddeb1bb3f76134898b92a81539f2e969fd924938d61d181b6357284fe80fd +digest: sha256:299ddeb1bb3f76134898b92a81539f2e969fd924938d61d181b6357284fe80fd +runner: sbatch --partition=flyloft --gres=gpu:1 --time=00:30:00 +host: sunk.6af0ac-eu-south-03b-briney-b200.coreweave.app +started: 2026-11-20T00:20:57Z +exit: 0 +--- +Job 1061: SUNK job dashboard: https://cks-grafana.coreweave.com/d/slurm-job-metrics/slurm-job-metrics?var-slurm_cluster=slurm&var-job_id=1061 +pyxis: importing docker image: ghcr.io#brineylab/toolshed/boltz-2@sha256:299ddeb1bb3f76134898b92a81539f2e969fd924938d61d181b6357284fe80fd +pyxis: imported docker image: ghcr.io#brineylab/toolshed/boltz-2@sha256:299ddeb1bb3f76134898b92a81539f2e969fd924938d61d181b6357284fe80fd +node: slurm-b200-204-039 +GPU 0: NVIDIA B200 (UUID: GPU-f0be83b4-2648-ab1f-cfe7-7111bd0c8ef1) +[2026-11-20T00:21:56Z] slurm-b200-204-039: GPU 0: NVIDIA B200 (UUID: GPU-f0be83b4-2648-ab1f-cfe7-7111bd0c8ef1) +[2026-11-20T00:21:56Z] predicting both fixtures in one process, 1 diffusion sample each +/opt/conda/lib/python3.11/site-packages/torch/cuda/__init__.py:61: FutureWarning: The pynvml package is deprecated. Please install nvidia-ml-py instead. If you did not install pynvml directly, please report this to the maintainers of the package that installed pynvml for you. + import pynvml # type: ignore[import] +[rank: 0] Seed set to 0 +Checking input data. +Processing 2 inputs with 2 threads. +Found explicit empty MSA for some proteins, will run these in single sequence mode. Keep in mind that the model predictions will be suboptimal without an MSA. +Found explicit empty MSA for some proteins, will run these in single sequence mode. Keep in mind that the model predictions will be suboptimal without an MSA. + 0%| | 0/2 [00:00