From 18409f7e17c219e111b16e42eb7b9f71f642ebb9 Mon Sep 17 00:00:00 2001 From: Alex Rubinsteyn Date: Tue, 29 Sep 2026 13:36:01 -0400 Subject: [PATCH 1/4] Reconcile normalized ORF hypotheses and scoped RNA evidence --- CHANGELOG.md | 10 + docs/combined-sources.md | 81 +++++++- tests/test_consumer_workflows.py | 63 ++++++ tests/test_reconciliation.py | 176 ++++++++++++++++ tests/test_twin_conformance.py | 34 ++++ topiary/__init__.py | 9 +- topiary/candidates.py | 4 +- topiary/reconciliation.py | 332 +++++++++++++++++++++++++++++++ 8 files changed, 698 insertions(+), 11 deletions(-) create mode 100644 tests/test_reconciliation.py create mode 100644 topiary/reconciliation.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 6f3c8f3..65d5252 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,16 @@ between published tags; older pre-5.0 notes are retained below. For current interfaces, see the [consumer guide](docs/consumer-guide.md). +## 5.87.0 + +- Link normalized events, nucleotide ORF hypotheses, full protein products and + peptide occurrences without collapsing alternative translations or conflicting + source measurements. Add relational evidence views and explicit RNA count union + that requires known membership and measurement scope (#370). +- Preserve those identities through long/wide CSV and TSV, original/additive + ranking and supporting-hypothesis selection. Distinguish known terminal flanks + from missing flanks in source observation IDs (#435). + ## 5.86.2 - Make dtype inference explicit when stacking sparse prediction frames and diff --git a/docs/combined-sources.md b/docs/combined-sources.md index a8f2379..233a2b3 100644 --- a/docs/combined-sources.md +++ b/docs/combined-sources.md @@ -207,8 +207,8 @@ tool's RNA measurements; abundance is never automatically averaged or summed. Equal protein products do not establish equal nucleotide ORFs. Distinct `orf_id`, `coding_sequence`, transcript, frame or completeness annotations remain independent source observations even when the translated sequence is -identical. Full ORF reconciliation is tracked in -[#370](https://github.com/openvax/topiary/issues/370). +identical. Use `reconcile_evidence` below to establish explicit ORF relationships +while retaining those observations. `event_id` must be explicitly harmonized, including assembly/variant identity where appropriate. Native labels from different tools are not assumed equal. @@ -216,15 +216,15 @@ Without a common event ID, observations stay separate while sequence IDs still show exact sequence agreement. Local `pep_context` or `sequence` is not promoted to a full ORF. A peptide-only table does not establish its generating full ORF. -To use another source's RNA measurement for an explicitly matching ORF, join it -as a named feature. `join_annotations` refuses ambiguous annotation keys: +To use another source's RNA measurement for an explicitly matching ORF, first +reconcile the input and join the selected measurement as a named feature. `join_annotations` refuses ambiguous annotation keys: ```python -from topiary import join_annotations +from topiary import join_annotations, reconcile_evidence -# combined contains both ORF/RNA rows and peptide rows with explicit -# protein_sequence and event_id identifying their generating protein. -keys = ["candidate_sample", "event_id", "protein_sequence_id"] +combined = reconcile_evidence(combined) +# Require an explicit ORF match; equal proteins alone are insufficient. +keys = ["candidate_sample", "orf_hypothesis_id"] rna = combined.df.loc[ combined.df.source_label.eq("exacto_normalized"), [*keys, "transcript_expression"], @@ -232,7 +232,7 @@ rna = combined.df.loc[ enriched = join_annotations( combined, rna, on=keys, prefix="exacto", provenance={"source": "exacto_normalized", "unit": "TPM", - "policy": "same sample, event and exact full sequence"}, + "policy": "same sample and reconciled ORF; retain transcript-level unit"}, ) ranked = rank_candidates( enriched, "exacto_transcript_expression / affinity['netmhcpan'].value", @@ -244,6 +244,69 @@ This expression illustrates a policy, not a calibrated biological model. The alternative ORF's abundance cannot attach just because its variant or gene agrees. Missing or ambiguous identity requires explicit resolution. +## Reconcile ORFs and RNA observations (5.87.0+) + +`reconcile_evidence(combined)` returns a copy with stable links between biological +entities. It neither chooses a caller nor changes the candidate universe. +`evidence_views(combined)` exposes `events`, `orfs`, `proteins`, `occurrences`, +`candidates`, `rna_observations` and `links` as DataFrames; supply +`source_labels=["lens"]` for a source-stratified view. + +| Input assertion | Identity rule | +| --- | --- | +| `event_id` or list-valued `event_ids` | Same sample, explicit `reference_name` and normalized event name; absent reference remains source-local | +| `orf_id` | Caller-local ID; contradictory descriptors raise, absent descriptors can be supplied by another row with the same local ID | +| `coding_sequence` and `transcript_path`, or `transcript_id` plus `orf_start`/`orf_end` | Cross-caller ORF agreement requires the same sample, reference and all supplied descriptors | +| `reading_frame`, `orf_completeness`, start/stop flags, `linked_variants` | Retained in ORF identity; differing assertions remain alternative hypotheses | +| `protein_sequence` | Exact full product, separate from nucleotide ORF identity | +| `protein_hypothesis_sequence` | May be partial; never promoted to a full protein | +| `peptide_start`/`peptide_end` | Zero-based half-open occurrence in the supplied protein; validated against its sequence | +| Missing peptide coordinates | Source-local occurrence; no guessed equivalence from a shared peptide | + +ORF bounds are also zero-based half-open. `transcript_path` is an explicit ordered +JSON path in the caller's normalized reference convention. Topiary does not lift +over assemblies, normalize native genomic variant strings or infer a path from +gene names. Incomplete records remain useful without establishing equivalence. +Reference scope follows the distinction between sequence and annotation identity +in [NCBI's feature documentation](https://www.ncbi.nlm.nih.gov/genbank/genomes_gff/). + +`rank_candidates` retains the chosen row's `orf_hypothesis_id` and +`peptide_occurrence_id`. `candidate_observations` and the relational `links` +retain alternative support. Rediscovery does not multiply candidate scores. +Use an explicit duplicate policy or source filter when measurements disagree. + +RNA observations can be supplied as a list of records in `rna_observations`: + +```python +measurement = { + "sample_name": "patient-01", "entity_type": "transcript", + "entity_id": "ENST-example", "quantity": "count", "unit": "reads", + "value": 2, "library_id": "rna-library-1", "read_set_id": "alignment-1", + "evidence_unit_ids": ["read-a", "read-b"], "method": "caller", "version": "1", +} +``` + +`normalize_rna_observation` validates this shape. `entity_type` may be `gene`, +`transcript`, `orf` or `variant`. A TPM measurement uses `quantity="abundance"`, +`unit="TPM"` and no evidence-unit list. Unknown values remain null. Original +quantifier metadata and extra fields survive; gene/transcript expression never +becomes ORF expression automatically. Each caller's observations remain distinct. + +`union_rna_observations([measurement, other_measurement])` unions identified +read, fragment, UMI or cell memberships only within one sample, library, read-set +namespace, measured entity and unit. Shared members count once. Missing membership, +different namespaces and TPM quantities raise: neither different file names nor +different callers establish independent evidence. An explicitly empty membership +is measured zero; an absent membership is unknown. Scalar expression/count +columns already present in source tables stay untouched and are never implicitly +converted into identified evidence sets. + +All identity columns, descriptors, alternative observations and metadata survive +Topiary CSV/TSV save/reload in long and wide forms. Reconciliation can be repeated +after reloading. The composed workflow test covers original and additive scoring, +retained hypothesis selection, and count union without running a predictor during +assembly. + ## Add prediction features on demand ```python diff --git a/tests/test_consumer_workflows.py b/tests/test_consumer_workflows.py index 5b0d1a5..e3595e6 100644 --- a/tests/test_consumer_workflows.py +++ b/tests/test_consumer_workflows.py @@ -3153,3 +3153,66 @@ def test_cli_output_can_be_read_back_by_topiary(cli_output_request, tmp_path, ca affinity = reloaded[reloaded.kind == "pMHC_affinity"].set_index("peptide") assert affinity.value.dtype == "float64" assert affinity.value.to_dict() == {"SIINFEKL": 50.0, "GILGFVFTL": 500.0} + + +@pytest.mark.parametrize("wide", [False, True]) +@pytest.mark.parametrize("extension", ["tsv", "csv"]) +def test_reconciled_hypotheses_survive_files_and_explicit_scoring(tmp_path, monkeypatch, wide, extension): + from topiary import ( + combine_sources, evidence_views, reconcile_evidence, rank_candidates, + rescore_candidates, read_tsv, read_csv, union_rna_observations, + ) + from .test_candidate_tables import Model, source + from .test_reconciliation import rna + + def forbidden(*args, **kwargs): + raise AssertionError("Table-only reconciliation must not instantiate a predictor") + + monkeypatch.setattr(TopiaryPredictor, "__init__", forbidden) + protein = "MAAASIINFEKL" + coding = "ATGGCTGCTGCTTCTATTATTAATTTTGAAAAACTG" + common = dict(reference_name="GRCh38", event_ids=["event-one", "event-two"], + orf_id="caller-one-orf", coding_sequence=coding, transcript_id="t", + orf_start=0, orf_end=len(coding), orf_completeness="start_to_stop") + first = source(**common) + first.df["event_ids"] = [common["event_ids"], common["event_ids"]] + first.df["protein_sequence"] = [protein, "MAAAGILGFVFTL"] + first.df["orf_id"] = ["orf-one", "orf-two"] + first.df["coding_sequence"] = [coding, "ATGGCTGCTGCTGGTATTCTGGGTTTTGTTTTTACTCTG"] + first.df["orf_end"] = first.df.coding_sequence.str.len() + first.df["peptide_start"] = [4, 4] + first.df["peptide_end"] = [12, 13] + first.df["rna_observations"] = [[rna()], [rna(entity_id="other-transcript")]] + other = first.df.iloc[:1].copy() + other["orf_id"] = "other-caller-name" + other["value"] = 75. + other["rna_observations"] = [[rna(evidence_unit_ids=["r2", "r3"], method="other")]] + orphan = pd.DataFrame([dict(common, protein_sequence=protein, transcript_expression=99.)]) + result = reconcile_evidence(combine_sources({"one": first, "two": other, "orf-only": orphan}, sample_name="p")) + assert len(evidence_views(result)["orfs"]) == 2 + assert len(evidence_views(result)["events"]) == 2 + ranked = rank_candidates(result, "affinity.value", ascending=True, duplicates="best") + assert ranked.peptide.tolist() == ["SIINFEKL", "GILGFVFTL"] + assert ranked.iloc[0].orf_hypothesis_id == result.df.iloc[0].orf_hypothesis_id + assert ranked.iloc[0].peptide_occurrence_id == result.df.iloc[0].peptide_occurrence_id + assert len(rank_candidates(result, "affinity.value", ascending=True, duplicates="best", + strata=["source_label", "candidate_mhc_class"])) == 3 + # The same read is not independent support merely because two callers used it. + assert union_rna_observations([result.df.iloc[0].rna_observations[0], + result.df.iloc[2].rna_observations[0]])["value"] == 3 + model = Model() + scored = rescore_candidates(result, model, prefix="fresh") + assert len(model.calls) == 2 + assert scored.df.candidate_id.nunique() == 2 + saved = scored.to_wide() if wide else scored + path = tmp_path / ("reconciled." + extension) + getattr(saved, "to_" + extension)(path) + restored = (read_tsv if extension == "tsv" else read_csv)(path).to_long() + after = reconcile_evidence(restored) + for name, before in evidence_views(result).items(): + pd.testing.assert_frame_equal(evidence_views(after)[name], before, check_dtype=False) + assert rank_candidates(after, "affinity.value", ascending=True, duplicates="best").peptide.tolist() == ranked.peptide.tolist() + assert rank_candidates(after, "fresh__testmodel__pMHC_affinity__value", ascending=True, + duplicates="best").peptide.tolist() == ["GILGFVFTL", "SIINFEKL"] + assert after.df.loc[after.df.source_label.eq("orf-only"), "candidate_id"].isna().all() + assert after.df.loc[after.df.source_label.eq("orf-only"), "value"].isna().all() diff --git a/tests/test_reconciliation.py b/tests/test_reconciliation.py new file mode 100644 index 0000000..772455d --- /dev/null +++ b/tests/test_reconciliation.py @@ -0,0 +1,176 @@ +"""Biological identities must not collapse alternative hypotheses or evidence.""" + +from copy import deepcopy + +import pandas as pd +import pytest + +from topiary import ( + combine_sources, evidence_views, normalize_rna_observation, + reconcile_evidence, union_rna_observations, +) + + +def rna(**updates): + return dict(dict(sample_name="p", entity_type="transcript", entity_id="t", + quantity="count", unit="reads", value=2, + library_id="lib", read_set_id="alignment-1", + evidence_unit_ids=["r1", "r2"], method="caller"), **updates) + + +def biological_rows(): + base = dict(sample_name="p", reference_name="GRCh38", event_id="chr1:1:A:T", + orf_id="local-1", coding_sequence="ATGGCT", transcript_id="t", + orf_start=0, orf_end=6, reading_frame=0, orf_completeness="start_to_stop", + protein_sequence="MA", rna_observations=[rna()], + transcript_expression=10., starts_at_start_codon=True, ends_with_stop_codon=True) + return pd.DataFrame([ + base, + dict(base, orf_id="synonymous", coding_sequence="ATGGCC"), + dict(base, orf_id="alternative", coding_sequence="ATGGGT", protein_sequence="MG"), + dict(base, orf_id="partial", coding_sequence=None, protein_sequence=None, + protein_hypothesis_sequence="MA", orf_completeness="partial", ends_with_stop_codon=None), + ]) + + +def test_cross_caller_orf_agreement_preserves_synonymous_and_alternative_hypotheses(): + original = biological_rows() + other = original.iloc[:1].copy() + other["orf_id"] = "different-local-name" + other["transcript_expression"] = 99. + combined = combine_sources({"one": original, "two": other}) + result = reconcile_evidence(combined) + assert result.df.orf_hypothesis_id.nunique() == 4 + assert result.df.protein_sequence_id.nunique() == 2 + assert result.df.iloc[0].orf_hypothesis_id == result.df.iloc[-1].orf_hypothesis_id + assert result.df.iloc[0].orf_hypothesis_id != result.df.iloc[1].orf_hypothesis_id + assert result.df.transcript_expression.tolist() == [10., 10., 10., 10., 99.] + assert result.df.iloc[3].protein_sequence_id is None + assert result.df.candidate_id.isna().all() + assert "orf_hypothesis_id" not in combined.df + pd.testing.assert_frame_equal(reconcile_evidence(result).df, result.df) + views = evidence_views(result) + assert len(views["events"]) == 1 + assert len(views["orfs"]) == 4 + assert len(views["proteins"]) == 2 + assert len(views["rna_observations"]) == 2 + assert len(views["links"]) == 5 + assert len(evidence_views(result, source_labels=["two"])["orfs"]) == 1 + assert evidence_views(result, source_labels=[])["orfs"].empty + with pytest.raises(ValueError, match="Unknown source"): + evidence_views(result, source_labels=["missing"]) + + +@pytest.mark.parametrize("field,value", [ + ("reference_name", "GRCh37"), ("reference_name", None), + ("sample_name", "other-patient"), ("transcript_id", "other-locus"), + ("reading_frame", 1), ("orf_completeness", "partial"), + ("linked_variants", ["second-variant"]), +]) +def test_reference_sample_path_frame_and_completeness_boundaries(field, value): + first = biological_rows().iloc[:1].drop(columns="rna_observations") + second = first.copy() + second[field] = pd.Series([value], dtype=object) + result = reconcile_evidence(combine_sources({"one": first, "two": second})) + assert result.df.orf_hypothesis_id.nunique() == 2 + if field in {"reference_name", "sample_name"}: + assert result.df.biological_event_ids.iloc[0] != result.df.biological_event_ids.iloc[1] + + +def test_missing_reference_or_coding_path_cannot_prove_cross_source_identity(): + for missing in ("reference_name", "coding_sequence", "transcript_id", "orf_end"): + rows = biological_rows().iloc[:1].drop(columns=missing) + result = reconcile_evidence(combine_sources({"one": rows, "two": rows})) + assert result.df.orf_hypothesis_id.nunique() == 2 + assert result.df.protein_sequence_id.nunique() == 1 + + +def test_local_id_cannot_hide_conflicting_orfs_and_absent_assertions_can_be_filled(): + rows = biological_rows().iloc[:2].copy() + rows["orf_id"] = "shared" + with pytest.raises(ValueError, match="Conflicting coding_sequence"): + reconcile_evidence(combine_sources({"one": rows})) + rows.loc[1, "coding_sequence"] = None + result = reconcile_evidence(combine_sources({"one": rows})) + assert result.df.orf_hypothesis_id.nunique() == 1 + assert result.df.iloc[1].coding_sequence is None + assert result.df.iloc[1].orf_descriptor["coding_sequence"] == "ATGGCT" + + +def test_occurrences_keep_flanks_loci_and_unknown_geometry(): + row = dict(sample_name="p", peptide="MA", allele="HLA-A*02:01", protein_sequence="MAMA", + orf_id="one", peptide_start=0, peptide_end=2) + rows = pd.DataFrame([row, dict(row, peptide_start=2, peptide_end=4), + dict(row, orf_id="two"), dict(row, peptide_start=None, peptide_end=None), + dict(row, n_flank="", c_flank=""), dict(row, gene_id="other")]) + result = reconcile_evidence(combine_sources({"one": rows})) + assert result.df.candidate_id.nunique() == 1 + assert result.df.peptide_occurrence_id.nunique() == 6 + assert len(evidence_views(result)["candidates"].iloc[0].source_observations) == 6 + + +@pytest.mark.parametrize("updates,message", [ + ({"peptide_start": 1, "peptide_end": 3}, "supporting protein"), + ({"peptide_start": 0}, "supplied together"), + ({"peptide_start": -1, "peptide_end": 1}, "length"), + ({"peptide_start": .5, "peptide_end": 2.5}, "integers"), + ({"reading_frame": 3}, "reading_frame"), + ({"orf_start": 2, "orf_end": 1}, "orf_start"), + ({"starts_at_start_codon": "yes"}, "boolean"), + ({"event_ids": ["different"]}, "event_id disagrees"), + ({"rna_observations": {}}, "rna_observations"), +]) +def test_invalid_assertions_fail(updates, message): + row = biological_rows().iloc[0].to_dict() + row.update(peptide="MA", **updates) + with pytest.raises((ValueError, TypeError), match=message): + reconcile_evidence(combine_sources({"one": pd.DataFrame([row])})) + + +def test_rna_union_deduplicates_only_known_membership(): + one, two = rna(), rna(evidence_unit_ids=["r2", "r3"], method="second") + result = union_rna_observations([one, two, one]) + assert result["value"] == 3 + assert result["evidence_unit_ids"] == ["r1", "r2", "r3"] + assert len(result["observations"]) == 3 + assert union_rna_observations([rna(value=0, evidence_unit_ids=[])])["value"] == 0 + assert one == rna() + + +@pytest.mark.parametrize("changes", [ + {"quantity": "abundance", "unit": "TPM", "evidence_unit_ids": None}, + {"evidence_unit_ids": None}, {"library_id": "other"}, {"read_set_id": "other"}, + {"sample_name": "other"}, {"entity_type": "gene"}, {"entity_id": "other"}, {"unit": "fragments"}, +]) +def test_rna_union_refuses_unknown_overlap_or_different_subject(changes): + with pytest.raises(ValueError): + union_rna_observations([rna(), rna(**changes)]) + + +@pytest.mark.parametrize("changes", [ + {"value": -1}, {"value": True}, {"value": float("inf")}, {"value": .5}, + {"value": 3}, {"entity_type": "protein"}, {"entity_id": ""}, {"library_id": None}, + {"evidence_unit_ids": ["r1", "r1"]}, +]) +def test_invalid_rna_measurement(changes): + with pytest.raises(ValueError): + normalize_rna_observation(rna(**changes)) + + +def test_rna_unknown_measurement_is_not_zero_and_original_annotations_survive(): + record = rna(value=None, evidence_unit_ids=None, extra={"keep": True}) + copy = deepcopy(record) + result = normalize_rna_observation(record) + assert result["value"] is None + result["extra"]["keep"] = False + assert record == copy + with pytest.raises(ValueError, match="sample_name"): + normalize_rna_observation(record, sample_name="other") + with pytest.raises(ValueError): + union_rna_observations([]) + + +def test_empty_reconciliation(): + result = reconcile_evidence(combine_sources({})) + assert result.empty + assert all(table.empty for table in evidence_views(result).values()) diff --git a/tests/test_twin_conformance.py b/tests/test_twin_conformance.py index 2a9b6f9..49ab84f 100644 --- a/tests/test_twin_conformance.py +++ b/tests/test_twin_conformance.py @@ -1333,3 +1333,37 @@ def test_source_tracking_preserves_scoped_prediction_aggregation(scope, expressi expected = evaluate_scores(plain.df, parse(expr)) assert expected.notna().all() pd.testing.assert_series_equal(evaluate_scores(combined.df, parse(expr)), expected) + + +# Normalization must agree at the explicit API and relational-view entry point. +def _reconciled_links(result): + from topiary import evidence_views, reconcile_evidence + return evidence_views(reconcile_evidence(result)) + + +def _direct_links(result): + from topiary import evidence_views + return evidence_views(result) + + +RECONCILIATION_TWINS = (_reconciled_links, _direct_links) + + +@pytest.mark.parametrize("conflict", [False, True]) +def test_reconciliation_entry_points_agree(conflict): + from .test_reconciliation import biological_rows + rows = biological_rows() + if conflict: + rows["orf_id"] = "conflicting" + result = combine_sources({"source": rows}) + outcomes = [] + for door in RECONCILIATION_TWINS: + try: + outcomes.append(door(result)) + except ValueError: + outcomes.append(None) + if conflict: + assert outcomes == [None, None] + else: + for name in outcomes[0]: + pd.testing.assert_frame_equal(outcomes[0][name], outcomes[1][name]) diff --git a/topiary/__init__.py b/topiary/__init__.py index ad75db8..8ace2c8 100644 --- a/topiary/__init__.py +++ b/topiary/__init__.py @@ -3,6 +3,9 @@ from .report_geometry import map_peptide_intervals, mutation_intervals_from_positions from .annotations import join_annotations from .candidates import combine_sources, protein_evidence_view, rank_candidates, rescore_candidates +from .reconciliation import ( + reconcile_evidence, evidence_views, normalize_rna_observation, union_rna_observations, +) from .prediction_batch import predict_with_cache_miss_report, PartialPredictionWarning from .predictor import ( TopiaryPredictor, @@ -187,7 +190,7 @@ encode_amino_acids, ) -__version__ = "5.86.2" +__version__ = "5.87.0" __all__ = [ "normalize_isovar_rna_support", @@ -198,6 +201,10 @@ "map_peptide_intervals", "mutation_intervals_from_positions", "combine_sources", + "reconcile_evidence", + "evidence_views", + "normalize_rna_observation", + "union_rna_observations", "protein_evidence_view", "rank_candidates", "rescore_candidates", diff --git a/topiary/candidates.py b/topiary/candidates.py index 70f209c..e82b34e 100644 --- a/topiary/candidates.py +++ b/topiary/candidates.py @@ -154,7 +154,9 @@ def combine_sources(sources, *, sample_name=None): # remain axes within an observation, so peptide_view still composes. fields = sorted(set(native.columns) - _MEASUREMENT_FIELDS) frame["source_observation_id"] = [ - _identity([label, sample, fields, *values]) + _identity([label, sample, fields, *values, + {field: isinstance(value, str) for field, value in zip(fields, values) + if field in {"n_flank", "c_flank"}}]) for sample, values in zip(samples, frame[fields].itertuples(index=False, name=None)) ] # The ordinary DSL reads one value per observation/model. Refuse diff --git a/topiary/reconciliation.py b/topiary/reconciliation.py new file mode 100644 index 0000000..d6b53ec --- /dev/null +++ b/topiary/reconciliation.py @@ -0,0 +1,332 @@ +"""Conservative identity and relational views for combined biological evidence.""" + +from collections.abc import Mapping +from copy import deepcopy +import math + +import pandas as pd + +from .candidates import _combined_frame, _identity +from .ranking import is_stated +from .result import TopiaryResult +from .serialization import normalize_python_types + + +_ORF_FIELDS = ( + "coding_sequence", "nucleotide_sequence", "transcript_path", "transcript_id", + "orf_start", "orf_end", "reading_frame", "orf_completeness", + "starts_at_start_codon", "ends_with_stop_codon", "linked_variants", + "protein_sequence", "protein_hypothesis_sequence", +) +_ID_COLUMNS = ("biological_event_ids", "orf_hypothesis_id", "peptide_occurrence_id", "rna_observation_ids") + + +def _value(value): + if isinstance(value, (Mapping, list, tuple)): + return normalize_python_types(value) + return normalize_python_types(value) if is_stated(value) else None + + +def _names(value, field): + if not isinstance(value, (list, tuple)) and not is_stated(value): + return [] + if not isinstance(value, (list, tuple)) or any(not isinstance(v, str) or not v.strip() for v in value): + raise ValueError(f"{field} must be a list of nonempty strings") + return sorted(set(value)) + + +def normalize_rna_observation(observation, *, sample_name=None): + """Validate an RNA measurement without changing its biological subject. + + Parameters + ---------- + observation : mapping + Required fields are ``entity_type`` (gene, transcript, ORF or variant), + ``entity_id``, ``quantity``, ``unit`` and ``value`` (nonnegative or null). + ``sample_name`` is required here or as an argument. Optional library, + read-set, method, version and other provenance fields are preserved. + ``evidence_unit_ids`` supplies exact members of a count, if known; + an empty list means measured zero, whereas absent/null means unknown. + sample_name : str, optional + Fill an absent sample; a different stated sample raises. + + Returns + ------- + dict + Independent copy, with sorted unique evidence members. No expression + is promoted from gene/transcript to ORF, and no counts are inferred. + A missing observation is invalid: omit it from the observation list. + """ + if not isinstance(observation, Mapping): + raise ValueError("RNA observation must be a mapping") + record = deepcopy(normalize_python_types(dict(observation))) + sample = record.get("sample_name") or sample_name + if sample_name is not None and sample != sample_name: + raise ValueError("RNA observation sample_name disagrees with its source row") + record["sample_name"] = sample + for key in ("sample_name", "entity_type", "entity_id", "quantity", "unit"): + if not isinstance(record.get(key), str) or not record[key].strip(): + raise ValueError(f"RNA observation requires {key}") + if record["entity_type"] not in {"gene", "transcript", "orf", "variant"}: + raise ValueError("RNA entity_type must be gene, transcript, orf or variant") + value = record.setdefault("value", None) + if value is not None: + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0: + raise ValueError("RNA value must be a finite nonnegative number or null") + if record["quantity"] == "count": + if int(value) != value: + raise ValueError("RNA count must be an integer") + record["value"] = int(value) + members = record.get("evidence_unit_ids") + if members is not None: + members = _names(members, "evidence_unit_ids") + if record["quantity"] != "count" or value != len(members): + raise ValueError("RNA evidence membership must agree with a stated count") + for field in ("library_id", "read_set_id"): + if not isinstance(record.get(field), str) or not record[field].strip(): + raise ValueError(f"Identified RNA evidence requires {field}") + record["evidence_unit_ids"] = members + return record + + +def union_rna_observations(observations): + """Count the union of identified RNA evidence in one measurement scope. + + Parameters + ---------- + observations : iterable of mapping + Records accepted by :func:`normalize_rna_observation`. Every record + must measure the same sample, library, read-set namespace, entity, + count quantity and unit, with explicit evidence-unit membership. + + Returns + ------- + dict + Union count and sorted member IDs, plus original ``observations``. + Shared reads count once even if several callers report them. Empty + input raises; unknown memberships, differing entities/namespaces and + expression quantities such as TPM raise rather than implying disjoint + evidence. Different read-set IDs do not prove independence. + """ + records = [normalize_rna_observation(r) for r in observations] + if not records: + raise ValueError("At least one RNA observation is required") + fields = ("sample_name", "library_id", "read_set_id", "entity_type", "entity_id", "quantity", "unit") + first = records[0] + if any(r["quantity"] != "count" or r["unit"] not in {"reads", "fragments", "umis", "cells"} + or r.get("evidence_unit_ids") is None for r in records): + raise ValueError("RNA union requires counts with known evidence-unit membership") + if any(any(r.get(k) != first.get(k) for k in fields) for r in records): + raise ValueError("RNA union requires the same sample/library/read-set/entity/quantity/unit scope") + members = sorted({member for r in records for member in r["evidence_unit_ids"]}) + return dict({k: first[k] for k in fields}, value=len(members), evidence_unit_ids=members, + method="evidence_union", observations=records) + + +def reconcile_evidence(result): + """Link events, ORFs and occurrences without selecting biological evidence. + + Parameters + ---------- + result : TopiaryResult + Output of ``combine_sources``, optionally rescored. Each row may name + one ``orf_id`` and several ``event_ids`` (or one ``event_id``). Equal + event names link across callers only with the same explicit + ``reference_name`` and sample; missing references remain source-local. + ORF descriptors use coding/nucleotide sequence, transcript path or + transcript ID, zero-based half-open ``orf_start``/``orf_end``, reading + frame, completeness, start/stop flags and linked variants. A local ORF + ID may recur across peptide rows: supplied descriptors must agree. + Cross-source ORF agreement requires a reference, coding sequence and + explicit path (or transcript ID plus ORF bounds); every supplied ORF + descriptor must match. Equal proteins alone never establish ORF identity. + ``peptide_start``/``peptide_end`` are zero-based half-open coordinates in + the supplied protein/hypothesis sequence. Absent coordinates retain a + source-local occurrence rather than guessing from a shared short peptide. + Optional ``rna_observations`` is a list of explicit measurements accepted + by ``normalize_rna_observation``. Other original annotations survive. + + Returns + ------- + TopiaryResult + Copy with ``biological_event_ids``, ``orf_hypothesis_id``, + ``peptide_occurrence_id``, normalized ``rna_observations`` and + ``rna_observation_ids``. ``orf_descriptor`` records reconciled assertions + without overwriting the caller's cells. No measurements are combined or + selected and no peptides, alleles or predictions are generated. Partial + windows retain their own ORFs and never acquire full-protein identity. + Empty input yields the same enriched schema. Reapplying is idempotent. + Use ``evidence_views`` for relational tables and ``rank_candidates`` to + select an actual supporting observation with an explicit score policy. + + Notes + ----- + This reconciles explicit, normalized identities, not genome assemblies or + coordinate conventions. Native event names must first be normalized by the + caller; there is no implicit liftover, contig aliasing or variant alignment. + Unknown specificity and conflicting abundance remain unchanged. + """ + frame = _combined_frame(result).copy() + records = frame.to_dict("records") + descriptors, local_keys, events_by_row = {}, [], [] + for row in records: + sample, source = row["candidate_sample"], row["source_label"] + reference = _value(row.get("reference_name")) + scope = [sample, reference] if reference is not None else [sample, None, source] + names = _names(row.get("event_ids"), "event_ids") + event = _value(row.get("event_id")) + if event is not None: + if names and str(event) not in names: + raise ValueError("event_id disagrees with event_ids") + names = sorted(set([*names, str(event)])) + events = [_identity(["event", scope, name]) for name in names] + events_by_row.append(events) + descriptor = {key: _value(row.get(key)) for key in _ORF_FIELDS} + for key in ("orf_start", "orf_end", "reading_frame"): + value = descriptor[key] + if value is not None: + if isinstance(value, bool) or not isinstance(value, (int, float)) or int(value) != value or value < 0: + raise ValueError(f"{key} must be a nonnegative integer") + descriptor[key] = int(value) + if descriptor["reading_frame"] not in (None, 0, 1, 2): + raise ValueError("reading_frame must be 0, 1 or 2") + if descriptor["orf_start"] is not None and descriptor["orf_end"] is not None: + if descriptor["orf_start"] >= descriptor["orf_end"]: + raise ValueError("orf_start must be smaller than orf_end") + for key in ("starts_at_start_codon", "ends_with_stop_codon"): + if descriptor[key] is not None and type(descriptor[key]) is not bool: + raise ValueError(f"{key} must be boolean or null") + local_id = _value(row.get("orf_id")) + has_orf = local_id is not None or any(descriptor[k] is not None for k in + ("coding_sequence", "nucleotide_sequence", "protein_sequence", "protein_hypothesis_sequence")) + local = _identity(["local_orf", scope, source, local_id or row["source_observation_id"]]) if has_orf else None + local_keys.append(local) + if local is not None: + previous = descriptors.setdefault(local, dict(descriptor, sample_name=sample, reference_name=reference)) + for key, value in descriptor.items(): + if value is not None and previous[key] is not None and _identity([value]) != _identity([previous[key]]): + raise ValueError(f"Conflicting {key} for source ORF {local_id!r}") + if value is not None: + previous[key] = value + enriched = [] + for row, local, events in zip(records, local_keys, events_by_row): + descriptor = deepcopy(descriptors.get(local)) + orf = local + if descriptor is not None: + located = descriptor["transcript_path"] is not None or ( + descriptor["transcript_id"] is not None and descriptor["orf_start"] is not None + and descriptor["orf_end"] is not None) + if descriptor["reference_name"] is not None and descriptor["coding_sequence"] is not None and located: + orf = _identity(["orf", descriptor]) + peptide, occurrence = _value(row.get("peptide")), None + if peptide is not None: + start, end = _value(row.get("peptide_start")), _value(row.get("peptide_end")) + if (start is None) != (end is None): + raise ValueError("peptide_start and peptide_end must be supplied together") + if start is not None: + if any(isinstance(v, bool) or not isinstance(v, (int, float)) or int(v) != v for v in (start, end)): + raise ValueError("Peptide coordinates must be integers") + start, end = int(start), int(end) + if start < 0 or end - start != len(peptide): + raise ValueError("Peptide coordinates disagree with peptide length") + sequence = (descriptor or {}).get("protein_sequence") or (descriptor or {}).get("protein_hypothesis_sequence") + if sequence is not None and sequence[start:end] != peptide: + raise ValueError("Peptide coordinates disagree with the supporting protein") + occurrence = _identity([ + "occurrence", row["candidate_sample"], orf, peptide, start, end, + _value(row.get("gene_id")), _value(row.get("pep_context")), + [isinstance(row.get("n_flank"), str), _value(row.get("n_flank"))], + [isinstance(row.get("c_flank"), str), _value(row.get("c_flank"))], + None if orf is not None and start is not None else row["source_observation_id"]]) + rna = row.get("rna_observations") + if not isinstance(rna, list) and not is_stated(rna): + rna = [] + if not isinstance(rna, list): + raise ValueError("rna_observations must be a list of measurement mappings") + rna = [normalize_rna_observation(r, sample_name=row["candidate_sample"]) for r in rna] + enriched.append(dict(biological_event_ids=events, orf_hypothesis_id=orf, + orf_descriptor=descriptor, peptide_occurrence_id=occurrence, + rna_observations=rna, + rna_observation_ids=[_identity(["rna", row["source_label"], r]) for r in rna])) + for column in (*_ID_COLUMNS, "orf_descriptor", "rna_observations"): + frame[column] = pd.Series([r[column] for r in enriched], index=frame.index, dtype=object) + metadata = deepcopy(result.metadata) + metadata.extra["evidence_reconciliation"] = {"schema": "topiary.evidence.v1", "interval_convention": "zero_based_half_open"} + return TopiaryResult(frame, metadata=metadata, form="long") + + +def evidence_views(result, *, source_labels=None): + """Return relational event/ORF/protein/occurrence/RNA and observation tables. + + Parameters + ---------- + result : TopiaryResult + Combined or reconciled evidence. Uses ``reconcile_evidence`` for the + same identity rules before materializing any view. + source_labels : iterable of str, optional + Explicit source-stratified view; None includes all sources. Unknown + labels raise. No caller, ORF or RNA observation is preferred implicitly. + + Returns + ------- + dict of pandas.DataFrame + ``events``, ``orfs``, ``proteins``, ``occurrences``, ``candidates``, + ``rna_observations`` and ``links``. Node tables have ``id`` and + ``source_observations``; links retain one distinct combination of source + observation, event, ORF, protein, occurrence, candidate and RNA IDs. + A link's list-valued event/RNA columns express many-to-many support. + Empty views retain their columns. These views never aggregate abundance; + traverse links to retain the supporting hypothesis of a selected window. + """ + frame = reconcile_evidence(result).df + if source_labels is not None: + if isinstance(source_labels, str): + raise TypeError("source_labels must be an iterable of labels") + labels = set(source_labels) + if labels - set(frame.source_label): + raise ValueError("Unknown source_labels") + frame = frame[frame.source_label.isin(labels)] + tables = {name: {} for name in ("events", "orfs", "proteins", "occurrences", "candidates", "rna_observations")} + links = {} + + def add(table, identity, payload, observation): + if identity is not None: + node = tables[table].setdefault(identity, dict(id=identity, **payload, source_observations=[])) + node["source_observations"] = sorted(set([*node["source_observations"], observation])) + + for row in frame.to_dict("records"): + observation = row["source_observation_id"] + sample = row["candidate_sample"] + names = _names(row.get("event_ids"), "event_ids") + if is_stated(row.get("event_id")): + names = sorted(set([*names, str(row["event_id"])])) + for identity, name in zip(row["biological_event_ids"], names): + add("events", identity, dict(sample_name=sample, reference_name=_value(row.get("reference_name")), + event_id=name), observation) + descriptor = row["orf_descriptor"] + sequence = (descriptor or {}).get("protein_sequence") + protein_id = _identity([sequence]) if sequence is not None else None + add("orfs", row["orf_hypothesis_id"], dict(descriptor=descriptor, protein_sequence_id=protein_id), observation) + add("proteins", protein_id, dict(sequence=sequence), observation) + add("proteins", _value(row["protein_sequence_id"]), dict(sequence=_value(row.get("protein_sequence"))), observation) + add("occurrences", row["peptide_occurrence_id"], dict(sample_name=sample, peptide=_value(row.get("peptide")), + orf_hypothesis_id=row["orf_hypothesis_id"], peptide_start=_value(row.get("peptide_start")), + peptide_end=_value(row.get("peptide_end"))), observation) + add("candidates", _value(row["candidate_id"]), dict(sample_name=sample, peptide=_value(row.get("peptide")), + allele=row["candidate_allele"]), observation) + for identity, rna in zip(row["rna_observation_ids"], row["rna_observations"]): + add("rna_observations", identity, dict(measurement=rna), observation) + link = {key: _value(row.get(key)) for key in + ("source_label", "source_observation_id", "candidate_id", "protein_sequence_id", *_ID_COLUMNS)} + links[_identity([link])] = link + columns = { + "events": ["sample_name", "reference_name", "event_id"], + "orfs": ["descriptor", "protein_sequence_id"], "proteins": ["sequence"], + "occurrences": ["sample_name", "peptide", "orf_hypothesis_id", "peptide_start", "peptide_end"], + "candidates": ["sample_name", "peptide", "allele"], "rna_observations": ["measurement"], + } + views = {name: pd.DataFrame(list(nodes.values()), columns=["id", *columns[name], "source_observations"]) + for name, nodes in tables.items()} + views["links"] = pd.DataFrame(list(links.values()), columns=[ + "source_label", "source_observation_id", "candidate_id", "protein_sequence_id", *_ID_COLUMNS]) + return views From becd21af7fc68768bed20b173317e10a4d2421f1 Mon Sep 17 00:00:00 2001 From: Alex Rubinsteyn Date: Tue, 29 Sep 2026 14:38:40 -0400 Subject: [PATCH 2/4] Keep source validation and warning checks compatible with pandas 3 --- CHANGELOG.md | 2 ++ tests/test_args_outputs.py | 8 ++++++-- tests/test_reconciliation.py | 4 ++-- topiary/candidates.py | 8 ++++---- 4 files changed, 14 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 65d5252..3ce0556 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,8 @@ For current interfaces, see the [consumer guide](docs/consumer-guide.md). - Preserve those identities through long/wide CSV and TSV, original/additive ranking and supporting-hypothesis selection. Distinguish known terminal flanks from missing flanks in source observation IDs (#435). +- Keep empty-source validation and output-warning tests compatible with pandas 3 + (#437). ## 5.86.2 diff --git a/tests/test_args_outputs.py b/tests/test_args_outputs.py index e96367b..554579e 100644 --- a/tests/test_args_outputs.py +++ b/tests/test_args_outputs.py @@ -4,7 +4,6 @@ import warnings import pandas as pd import pytest -from pandas.errors import SettingWithCopyWarning from .common import eq_ @@ -42,10 +41,15 @@ def test_write_outputs(): print(df_from_file) eq_(len(df_expected), len(df_from_file)) assert (df_expected == df_from_file).all().all() + assignment_warnings = tuple( + getattr(pd.errors, name) for name in ("SettingWithCopyWarning", "ChainedAssignmentError") + if hasattr(pd.errors, name) + ) assert not any( - issubclass(warning.category, SettingWithCopyWarning) + issubclass(warning.category, assignment_warnings) for warning in caught_warnings ) + pd.testing.assert_frame_equal(df, pd.DataFrame({"x": [1, 2, 3], "y": [10, 20, 30]})) def test_preview_shows_selected_renamed_columns_and_reports_omitted_rows(capsys): diff --git a/tests/test_reconciliation.py b/tests/test_reconciliation.py index 772455d..13b57d0 100644 --- a/tests/test_reconciliation.py +++ b/tests/test_reconciliation.py @@ -45,7 +45,7 @@ def test_cross_caller_orf_agreement_preserves_synonymous_and_alternative_hypothe assert result.df.iloc[0].orf_hypothesis_id == result.df.iloc[-1].orf_hypothesis_id assert result.df.iloc[0].orf_hypothesis_id != result.df.iloc[1].orf_hypothesis_id assert result.df.transcript_expression.tolist() == [10., 10., 10., 10., 99.] - assert result.df.iloc[3].protein_sequence_id is None + assert pd.isna(result.df.iloc[3].protein_sequence_id) assert result.df.candidate_id.isna().all() assert "orf_hypothesis_id" not in combined.df pd.testing.assert_frame_equal(reconcile_evidence(result).df, result.df) @@ -93,7 +93,7 @@ def test_local_id_cannot_hide_conflicting_orfs_and_absent_assertions_can_be_fill rows.loc[1, "coding_sequence"] = None result = reconcile_evidence(combine_sources({"one": rows})) assert result.df.orf_hypothesis_id.nunique() == 1 - assert result.df.iloc[1].coding_sequence is None + assert pd.isna(result.df.iloc[1].coding_sequence) assert result.df.iloc[1].orf_descriptor["coding_sequence"] == "ATGGCT" diff --git a/topiary/candidates.py b/topiary/candidates.py index e82b34e..6d22c4a 100644 --- a/topiary/candidates.py +++ b/topiary/candidates.py @@ -16,7 +16,7 @@ from .predictor import from_predictions from .ranking import ( apply_filter, as_dsl_node, evaluate_scores, format_allele_set, is_stated, - mhc_dependence, split_allele_set, + mhc_dependence, split_allele_set, stated_values, ) from .result import TopiaryResult, stack_results from .serialization import normalize_python_types @@ -130,12 +130,12 @@ def combine_sources(sources, *, sample_name=None): frame[column] = None proteins = frame.get("protein_sequence", pd.Series(None, index=frame.index, dtype=object)) hypotheses = frame.get("protein_hypothesis_sequence", pd.Series(None, index=frame.index, dtype=object)) - if not (frame.peptide.map(is_stated) | proteins.map(is_stated) | hypotheses.map(is_stated)).all(): + if not (stated_values(frame.peptide) | stated_values(proteins) | stated_values(hypotheses)).all(): raise ValueError(f"Source {label!r} needs a peptide or explicit protein_sequence " "or protein_hypothesis_sequence on every row") samples = frame.get("sample_name", pd.Series(None, index=frame.index, dtype=object)) - samples = samples.where(samples.map(is_stated), sample_name) - if not samples.map(is_stated).all(): + samples = samples.where(stated_values(samples), sample_name) + if not stated_values(samples).all(): raise ValueError(f"Source {label!r} needs sample_name for unlabelled rows") frame["source_label"] = label frame["source_row"] = np.arange(len(frame)) From 6f9a4d0dc81a91d403bb4e53360f84cb3d576c0a Mon Sep 17 00:00:00 2001 From: Alex Rubinsteyn Date: Tue, 29 Sep 2026 14:44:27 -0400 Subject: [PATCH 3/4] Link allele-free peptide evidence to existing candidate queries --- docs/combined-sources.md | 7 ++++++- tests/test_reconciliation.py | 18 ++++++++++++++++++ topiary/reconciliation.py | 22 +++++++++++++++++++++- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/docs/combined-sources.md b/docs/combined-sources.md index 233a2b3..c8bac92 100644 --- a/docs/combined-sources.md +++ b/docs/combined-sources.md @@ -249,7 +249,7 @@ agrees. Missing or ambiguous identity requires explicit resolution. `reconcile_evidence(combined)` returns a copy with stable links between biological entities. It neither chooses a caller nor changes the candidate universe. `evidence_views(combined)` exposes `events`, `orfs`, `proteins`, `occurrences`, -`candidates`, `rna_observations` and `links` as DataFrames; supply +`candidates`, `rna_observations`, `links` and `candidate_occurrences` as DataFrames; supply `source_labels=["lens"]` for a source-stratified view. | Input assertion | Identity rule | @@ -274,6 +274,11 @@ in [NCBI's feature documentation](https://www.ncbi.nlm.nih.gov/genbank/genomes_g `peptide_occurrence_id`. `candidate_observations` and the relational `links` retain alternative support. Rediscovery does not multiply candidate scores. Use an explicit duplicate policy or source filter when measurements disagree. +`candidate_occurrences` links all same-sample peptide occurrences to existing +pMHC queries, including sequence reports with no HLA assignment. Its +`reported_candidate` flag distinguishes a reported peptide-HLA pair from a +sequence match. These links transfer no scores, expression, specificity or +eligibility between observations, and create no new candidates. RNA observations can be supplied as a list of records in `rna_observations`: diff --git a/tests/test_reconciliation.py b/tests/test_reconciliation.py index 13b57d0..85cf5d0 100644 --- a/tests/test_reconciliation.py +++ b/tests/test_reconciliation.py @@ -174,3 +174,21 @@ def test_empty_reconciliation(): result = reconcile_evidence(combine_sources({})) assert result.empty assert all(table.empty for table in evidence_views(result).values()) + + +def test_allele_free_occurrences_link_to_queries_without_fabricating_predictions(): + rows = pd.DataFrame([ + dict(sample_name="p", peptide="SIINFEKL", allele="HLA-A*02:01", value=50.), + dict(sample_name="p", peptide="SIINFEKL", allele="HLA-B*07:02", value=90.), + dict(sample_name="p", peptide="SIINFEKL", protein_hypothesis_sequence="AAASIINFEKL", orf_id="local"), + dict(sample_name="other", peptide="SIINFEKL", protein_hypothesis_sequence="AAASIINFEKL", orf_id="local"), + ]) + result = reconcile_evidence(combine_sources({"one": rows})) + relations = evidence_views(result)["candidate_occurrences"] + # HLA is a query axis: the first two rows share one occurrence. Both + # queries also link to the allele-free sequence report from this patient. + assert len(relations) == 4 + assert relations.reported_candidate.sum() == 2 + assert set(relations.peptide_occurrence_id) == set(result.df.iloc[:3].peptide_occurrence_id) + assert result.df.iloc[2:].candidate_id.isna().all() + assert result.df.iloc[2:].value.isna().all() diff --git a/topiary/reconciliation.py b/topiary/reconciliation.py index d6b53ec..a17ecb9 100644 --- a/topiary/reconciliation.py +++ b/topiary/reconciliation.py @@ -271,7 +271,12 @@ def evidence_views(result, *, source_labels=None): ------- dict of pandas.DataFrame ``events``, ``orfs``, ``proteins``, ``occurrences``, ``candidates``, - ``rna_observations`` and ``links``. Node tables have ``id`` and + ``rna_observations``, ``links`` and ``candidate_occurrences``. The latter + links same-sample peptide occurrences to existing pMHC queries, including + allele-free sequence evidence; ``reported_candidate`` distinguishes an + actual peptide-HLA report from a sequence match. It never transfers + predictions, RNA values, tumor specificity or admission between rows. + Node tables have ``id`` and ``source_observations``; links retain one distinct combination of source observation, event, ORF, protein, occurrence, candidate and RNA IDs. A link's list-valued event/RNA columns express many-to-many support. @@ -329,4 +334,19 @@ def add(table, identity, payload, observation): for name, nodes in tables.items()} views["links"] = pd.DataFrame(list(links.values()), columns=[ "source_label", "source_observation_id", "candidate_id", "protein_sequence_id", *_ID_COLUMNS]) + occurrences = {} + for row in tables["occurrences"].values(): + occurrences.setdefault((row["sample_name"], row["peptide"]), []).append(row) + reported = {(row["candidate_id"], row["peptide_occurrence_id"]) for row in links.values() + if row["candidate_id"] is not None} + candidate_occurrences = [] + for candidate in tables["candidates"].values(): + for occurrence in occurrences.get((candidate["sample_name"], candidate["peptide"]), []): + candidate_occurrences.append(dict( + candidate_id=candidate["id"], peptide_occurrence_id=occurrence["id"], + orf_hypothesis_id=occurrence["orf_hypothesis_id"], + source_observations=occurrence["source_observations"], + reported_candidate=(candidate["id"], occurrence["id"]) in reported)) + views["candidate_occurrences"] = pd.DataFrame(candidate_occurrences, columns=[ + "candidate_id", "peptide_occurrence_id", "orf_hypothesis_id", "source_observations", "reported_candidate"]) return views From 8115d5e0d59cd4808fedd5993513676f8d9fb81e Mon Sep 17 00:00:00 2001 From: Alex Rubinsteyn Date: Tue, 29 Sep 2026 14:56:18 -0400 Subject: [PATCH 4/4] Avoid mutating pandas 3 genotype masks --- CHANGELOG.md | 4 ++-- tests/test_allele_set.py | 2 ++ topiary/ranking/apply.py | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3ce0556..53bf369 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,8 +16,8 @@ For current interfaces, see the [consumer guide](docs/consumer-guide.md). - Preserve those identities through long/wide CSV and TSV, original/additive ranking and supporting-hypothesis selection. Distinguish known terminal flanks from missing flanks in source observation IDs (#435). -- Keep empty-source validation and output-warning tests compatible with pandas 3 - (#437). +- Keep empty-source validation, genotype-aware filtering and output-warning + tests compatible with pandas 3 (#437). ## 5.86.2 diff --git a/tests/test_allele_set.py b/tests/test_allele_set.py index fde717a..2a212d5 100644 --- a/tests/test_allele_set.py +++ b/tests/test_allele_set.py @@ -159,12 +159,14 @@ def test_dependence_is_read_from_the_set(): def test_an_allele_scoped_filter_keeps_the_genotype_row(): """Same peptide-level rule as allele-free evidence (#183).""" df = _genotype_df() + original = df.copy(deep=True) kept = apply_filter(df, parse("affinity.value <= 500")) assert sorted(kept["kind"].unique()) == [ "pMHC_affinity", "pMHC_presentation", ] + pd.testing.assert_frame_equal(df, original) def test_a_peptide_filtered_out_entirely_takes_its_genotype_row(): diff --git a/topiary/ranking/apply.py b/topiary/ranking/apply.py index 8457eed..d29a0ca 100644 --- a/topiary/ranking/apply.py +++ b/topiary/ranking/apply.py @@ -114,7 +114,7 @@ def _peptide_level_groups(ctx): """ peptide_level = _blank(ctx.group_index.get_level_values("allele")) if ALLELE_SET_COLUMN in ctx.group_keys: - peptide_level |= ~_blank( + peptide_level = peptide_level | ~_blank( ctx.group_index.get_level_values(ALLELE_SET_COLUMN) ) return peptide_level