From 986037cfcb555dafae181169bb19f066ca9ef709 Mon Sep 17 00:00:00 2001 From: SkyeAv Date: Wed, 14 Oct 2026 11:32:34 -0700 Subject: [PATCH] feat!: switch the QC embedding stage to the Babel-fine-tuned SapBERT Stage 4 of the QC cascade now scores residual pairs with RenaissanceComputing/SapBERT-PubMedBERT-BabelSynonyms-v2025-01-23, RENCI's fine-tune of cambridgeltl/SapBERT-from-PubMedBERT-fulltext on synonym pairs sampled from NCATS Translator Babel identifier cliques (release 2025-01-23), instead of the base checkpoint. Drop-in for sentence-transformers: same PubMedBERT architecture, 768-dim embeddings. The model cache moves from .tablassert/sapbert/ to .tablassert/sapbert-babel/ so installs holding the base checkpoint re-download rather than silently serving it as the new model; stale .tablassert/sapbert/ directories can be deleted. The 0.5 pass threshold is unchanged and remains un-tuned for the new score distribution. docs/api/qc.md tracks the new checkpoint, provenance, and cache location; tests/test_cover_qc.py pins the new HF repo id. --- docs/api/qc.md | 8 ++++---- src/tablassert/qc.py | 14 ++++++++------ tests/test_cover_qc.py | 2 +- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/docs/api/qc.md b/docs/api/qc.md index 4a717ecf..670e8eb7 100644 --- a/docs/api/qc.md +++ b/docs/api/qc.md @@ -138,17 +138,17 @@ return similarity >= qc.SIMILARITY_THRESHOLD # 0.5 ### SapBERT Model -**Model:** `cambridgeltl/SapBERT-from-PubMedBERT-fulltext` +**Model:** `RenaissanceComputing/SapBERT-PubMedBERT-BabelSynonyms-v2025-01-23` -**Backend:** [sentence-transformers](https://www.sbert.net/) (PyTorch). Embeddings are compared with scikit-learn's `cosine_similarity`. SapBERT's self-alignment pretraining pulls UMLS synonym pairs together in embedding space, which fits this stage's task, deciding whether two names denote the same entity, better than the NLI/STS-trained BioBERT it replaced. The 0.5 threshold was carried over from that BioBERT gate and has not been re-tuned for SapBERT's score distribution. +**Backend:** [sentence-transformers](https://www.sbert.net/) (PyTorch). Embeddings are compared with scikit-learn's `cosine_similarity`. SapBERT's self-alignment pretraining pulls synonym pairs together in embedding space; this checkpoint is RENCI's fine-tune of `cambridgeltl/SapBERT-from-PubMedBERT-fulltext` on synonym pairs sampled from NCATS Translator Babel identifier cliques, so its training signal matches this stage's task, deciding whether two names denote the same entity. The 0.5 threshold was carried over from the earlier BioBERT gate and has not been re-tuned for SapBERT's score distribution. **Lazy-loaded** on the first `fullmap_audit()` call that reaches the embedding stage via `get_sapbert()`, then cached globally for the lifetime of the process. ### Model Caching -`get_sapbert()` loads the model from the local cache when present; otherwise it downloads `cambridgeltl/SapBERT-from-PubMedBERT-fulltext` and saves it for future runs. +`get_sapbert()` loads the model from the local cache when present; otherwise it downloads `RenaissanceComputing/SapBERT-PubMedBERT-BabelSynonyms-v2025-01-23` and saves it for future runs. -**Cache location:** `.tablassert/sapbert/` on disk (`qc.MODEL`); the loaded model object is also cached in memory for the lifetime of the process. +**Cache location:** `.tablassert/sapbert-babel/` on disk (`qc.MODEL`); the loaded model object is also cached in memory for the lifetime of the process. Existing `.tablassert/sapbert/` directories hold the previous base checkpoint and are stale; they can be deleted. ### Example Usage diff --git a/src/tablassert/qc.py b/src/tablassert/qc.py index 5b0472a2..6d65e8b9 100644 --- a/src/tablassert/qc.py +++ b/src/tablassert/qc.py @@ -22,11 +22,13 @@ logger = cat("QC") -MODEL: Path = BASE / "sapbert" +# Cache dir is keyed to the model: swapping HF repos must move the dir, or +# existing installs would silently keep serving the previous checkpoint. +MODEL: Path = BASE / "sapbert-babel" # Pass threshold for the SapBERT cosine-similarity stage. Carried over from the -# BioBERT gate; SapBERT's UMLS synonym-alignment training shifts true synonym -# pairs upward, so a tuning pass against real reject logs is the natural follow-up. +# BioBERT gate; synonym-alignment fine-tuning shifts true synonym pairs upward, +# so a tuning pass against real reject logs is the natural follow-up. SIMILARITY_THRESHOLD: float = 0.5 FUZZ_RATIO_THRESHOLD: float = 70.0 FUZZ_PARTIAL_THRESHOLD: float = 80.0 @@ -103,8 +105,8 @@ def get_sapbert() -> object: """Lazy-load and memoize the SapBERT sentence-transformer (``functools.cache``). Loads from the local cache at ``MODEL`` when present; otherwise downloads - ``cambridgeltl/SapBERT-from-PubMedBERT-fulltext`` and saves it for future - runs. + ``RenaissanceComputing/SapBERT-PubMedBERT-BabelSynonyms-v2025-01-23`` and + saves it for future runs. Returns: The memoized ``SentenceTransformer`` instance. @@ -117,7 +119,7 @@ def get_sapbert() -> object: model: object = sentence_transformers.SentenceTransformer(str(MODEL)) # pyright: ignore else: model = sentence_transformers.SentenceTransformer( # pyright: ignore - "cambridgeltl/SapBERT-from-PubMedBERT-fulltext" + "RenaissanceComputing/SapBERT-PubMedBERT-BabelSynonyms-v2025-01-23" ) MODEL.mkdir(parents=True, exist_ok=True) model.save(MODEL) # pyright: ignore diff --git a/tests/test_cover_qc.py b/tests/test_cover_qc.py index 6fa8aa1a..06dafe2a 100644 --- a/tests/test_cover_qc.py +++ b/tests/test_cover_qc.py @@ -17,7 +17,7 @@ import tablassert.qc as qc from tablassert.errors import QcRuntimeMissingError -HF_REPO: str = "cambridgeltl/SapBERT-from-PubMedBERT-fulltext" +HF_REPO: str = "RenaissanceComputing/SapBERT-PubMedBERT-BabelSynonyms-v2025-01-23" class FakeModel: