From 53f2662d714e351d6c3ae7d0bedb3cc54813dd3a Mon Sep 17 00:00:00 2001 From: Longping Fu Date: Wed, 14 Oct 2026 10:38:52 -0400 Subject: [PATCH 1/5] calibrated MS1 patch: skip the smoothness check on scans the cap did not cut Nothing is interpolated there, and short end-of-run scans (80-150 peaks) can have a held-out p99 just over 1 ppm, which refused NGlycoMM NG_12_R for no reason. Co-Authored-By: Claude Opus 5.5 --- src/mzml_utils/spectrum_cache.py | 10 +++++----- tests/test_ms1_patch.py | 8 ++++++++ 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/mzml_utils/spectrum_cache.py b/src/mzml_utils/spectrum_cache.py index b9f8bb9..51599f2 100644 --- a/src/mzml_utils/spectrum_cache.py +++ b/src/mzml_utils/spectrum_cache.py @@ -766,9 +766,9 @@ def merge_calibrated_ms1(cached_mz, cached_int, raw_mz, raw_int, max_shift_ppm: RAW scan's most intense. The calibration shift (ppm) is then a function of m/z inside the scan and is interpolated from the pairs onto the peaks the cap removed (constant beyond the ends). Kept peaks keep their calibrated values exactly; added peaks get RAW intensity and - interpolated calibrated m/z. Accuracy is measured on the scan itself: shifts at odd-indexed - pairs are predicted from the even-indexed ones, and the 99th percentile must stay under - ``max_heldout_ppm``. + interpolated calibrated m/z. Accuracy is measured on the scan itself, when peaks are added: + shifts at odd-indexed pairs are predicted from the even-indexed ones, and the 99th percentile + must stay under ``max_heldout_ppm``. A scan the cap did not cut is returned as cached. Returns (mz, intensity, info) or raises ValueError with the reason. """ @@ -806,6 +806,8 @@ def merge_calibrated_ms1(cached_mz, cached_int, raw_mz, raw_int, max_shift_ppm: raise ValueError("recalibration reorders peaks; not a smooth m/z function") shift = (b - a) / a * 1e6 info["max_abs_shift_ppm"] = float(np.abs(shift).max()) + if len(pair) == len(rmz): # nothing was cut, nothing to interpolate + return cmz, cit.astype(np.float64), info if len(a) >= 10: tr_i, te_i = np.arange(0, len(a), 2), np.arange(1, len(a), 2) res = np.abs(np.interp(a[te_i], a[tr_i], shift[tr_i]) - shift[te_i]) @@ -813,8 +815,6 @@ def merge_calibrated_ms1(cached_mz, cached_int, raw_mz, raw_int, max_shift_ppm: info["heldout_max_ppm"] = float(res.max()) if info["heldout_p99_ppm"] > max_heldout_ppm: raise ValueError(f"calibration shift is not smooth in m/z (held-out p99 {info['heldout_p99_ppm']:.3f} ppm)") - if len(pair) == len(rmz): - return cmz, cit.astype(np.float64), info add = np.setdiff1d(np.arange(len(rmz)), pair) add_mz = rmz[add] * (1 + np.interp(rmz[add], a, shift) / 1e6) mz = np.concatenate([cmz, add_mz]) diff --git a/tests/test_ms1_patch.py b/tests/test_ms1_patch.py index 8579594..29da2ef 100644 --- a/tests/test_ms1_patch.py +++ b/tests/test_ms1_patch.py @@ -293,3 +293,11 @@ def test_find_pwiz_dir_copies_the_dlls_locally(tmp_path, monkeypatch): assert tr.find_pwiz_dir(required=False) is None with pytest.raises(FileNotFoundError): tr.find_pwiz_dir() + + +def test_uncut_scan_is_not_held_to_the_smoothness_check(): + rng = np.random.default_rng(3) + rmz = np.sort(rng.uniform(400, 1800, 40)); rit = rng.uniform(1e3, 1e6, 40).astype(np.float32).astype(float) + cmz = rmz * (1 + rng.uniform(-8, 8, 40) / 1e6) # rough shift, but every peak kept + mz, it, info = sc.merge_calibrated_ms1(cmz.astype(np.float32), rit.astype(np.float32), rmz, rit) + assert info["n_added"] == 0 and len(mz) == 40 From 8842424e6e580c71a500e516e5692aefd401aac6 Mon Sep 17 00:00:00 2001 From: Longping Fu Date: Wed, 14 Oct 2026 10:40:10 -0400 Subject: [PATCH 2/5] ms1_cap_status: a cap needs at least 3 scans at the maximum peak count With few MS1 scans the single largest one already reached 5%, so an uncapped source read as capped. Co-Authored-By: Claude Opus 5.5 --- src/mzml_utils/spectrum_cache.py | 6 +++--- tests/test_ms1_patch.py | 11 +++++++++++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/src/mzml_utils/spectrum_cache.py b/src/mzml_utils/spectrum_cache.py index 51599f2..6f55e0f 100644 --- a/src/mzml_utils/spectrum_cache.py +++ b/src/mzml_utils/spectrum_cache.py @@ -1023,8 +1023,8 @@ def ms1_cap_status(reader) -> dict: """Whether a reader's MS1 peaks look capped (a per-scan most-intense limit) or were patched. For a SpectrumCache: patched when its meta records ``ms1_patch``; otherwise capped when at - least 5% of MS1 scans share exactly the maximum peak count (a natural peak-count distribution - has no such pile-up). For an MzMLReader: read from the mzML header's dataProcessing + least 5% of MS1 scans, and at least 3, share exactly the maximum peak count (a natural + peak-count distribution has no such pile-up: its maximum is one scan). For an MzMLReader: read from the mzML header's dataProcessing ("most intense count"). Returns {capped, cap, patched, frac_at_cap, source}. """ if isinstance(reader, SpectrumCache): @@ -1041,7 +1041,7 @@ def ms1_cap_status(reader) -> dict: if not tot or not top: return dict(capped=False, cap=None, patched=False, frac_at_cap=0.0, source="peak-counts") frac = top[1] / tot - capped = frac >= 0.05 and top[0] >= 50 + capped = top[1] >= max(3, 0.05 * tot) and top[0] >= 50 return dict(capped=capped, cap=int(top[0]) if capped else None, patched=False, frac_at_cap=round(frac, 4), source="peak-counts") path = getattr(reader, "path", None) or getattr(reader, "filepath", None) or getattr(reader, "filename", None) diff --git a/tests/test_ms1_patch.py b/tests/test_ms1_patch.py index 29da2ef..ae48842 100644 --- a/tests/test_ms1_patch.py +++ b/tests/test_ms1_patch.py @@ -301,3 +301,14 @@ def test_uncut_scan_is_not_held_to_the_smoothness_check(): cmz = rmz * (1 + rng.uniform(-8, 8, 40) / 1e6) # rough shift, but every peak kept mz, it, info = sc.merge_calibrated_ms1(cmz.astype(np.float32), rit.astype(np.float32), rmz, rit) assert info["n_added"] == 0 and len(mz) == 40 + + +def test_a_natural_peak_count_maximum_is_not_a_cap(tmp_path): + db = os.path.join(str(tmp_path), "x.db") + conn = sc.open_db(db) + for k in range(10): + n = 100 + 7 * k # distinct counts: the max is one scan + sc.insert_scan(conn, _spec(k + 1, 1, np.linspace(400, 1800, n), np.full(n, 1e5), float(k))) + conn.commit(); conn.close() + with sc.SpectrumCache(db, mirror_local=False) as c: + assert sc.ms1_cap_status(c)["capped"] is False From 9c1f72630eb7959664df23c163d456335d6d7124 Mon Sep 17 00:00:00 2001 From: Longping Fu Date: Wed, 14 Oct 2026 10:54:22 -0400 Subject: [PATCH 3/5] canonical_match records each match's origin: deisotoped, raw or raw_fallback MatchedIon gains an origin field (default empty, so positional construction is unchanged). Callers that keep evidence provenance can now tell a raw-fallback match from a deisotoped one. Co-Authored-By: Claude Opus 5.5 --- src/mzml_utils/fragments.py | 1 + src/mzml_utils/match_context.py | 14 ++++++++++++++ tests/test_fragments.py | 16 ++++++++++++++++ 3 files changed, 31 insertions(+) diff --git a/src/mzml_utils/fragments.py b/src/mzml_utils/fragments.py index 8d98ec8..3e2fc0a 100644 --- a/src/mzml_utils/fragments.py +++ b/src/mzml_utils/fragments.py @@ -57,6 +57,7 @@ class MatchedIon: exp_mz: float = 0.0 exp_intensity: float = 0.0 mass_error_ppm: float = 0.0 + origin: str = "" # set by match_context.canonical_match: deisotoped / raw / raw_fallback @dataclass diff --git a/src/mzml_utils/match_context.py b/src/mzml_utils/match_context.py index 03d51db..ef11eaa 100644 --- a/src/mzml_utils/match_context.py +++ b/src/mzml_utils/match_context.py @@ -90,6 +90,15 @@ def from_arrays( ) +def _tag_origin(matches, origin: str) -> None: + """Record which spectrum a match came from (caller subclasses without the field are skipped).""" + for m in matches: + try: + m.origin = origin + except AttributeError: + pass + + def canonical_match( ions: List[TheoreticalIon], ctx: MatchContext, @@ -113,6 +122,9 @@ def canonical_match( 3. If ``exclude_mzs`` is provided, any match whose observed m/z is within ``exclude_tol_ppm`` of an exclusion target is dropped. + Each match's ``origin`` says where it was found: ``deisotoped`` (step 1 on a deisotoped + spectrum), ``raw`` (step 1 without deisotoping) or ``raw_fallback`` (step 2). + ``match_fn`` lets callers plug in a project-local ``match_peaks`` implementation (e.g., the annotator's extended ``MatchedIon`` subclass with ``has_modification``). The function must accept the same signature @@ -131,6 +143,7 @@ def canonical_match( tolerance_ppm=ctx.tolerance_ppm, match_isotopes=not ctx.deisotope_applied, ) + _tag_origin(primary, "deisotoped" if ctx.deisotope_applied else "raw") if raw_fallback and ctx.deisotope_applied: matched_keys = {(m.ion_type, m.ion_number, m.charge) for m in primary} @@ -144,6 +157,7 @@ def canonical_match( tolerance_ppm=ctx.tolerance_ppm, match_isotopes=False, ) + _tag_origin(raw_hits, "raw_fallback") primary.extend(raw_hits) if exclude_mzs: diff --git a/tests/test_fragments.py b/tests/test_fragments.py index 1b099fb..4e8d11f 100644 --- a/tests/test_fragments.py +++ b/tests/test_fragments.py @@ -187,3 +187,19 @@ def test_y0_minus_water_is_below_y0(): ions = FragmentCalculator('SAAAAAK', mods, precursor_charge=3).calculate_Y_ions(charges=[z]) by = {i.annotation.split()[0]: i.mz for i in ions if i.annotation and i.annotation.startswith('Y0')} assert abs((by['Y0'] - by['Y0-H2O']) - 18.0106 / z) < 1e-6, by + + +def test_canonical_match_records_origin(): + import numpy as np + from mzml_utils import FragmentCalculator, canonical_match, from_spectrum + from mzml_utils.reader import Spectrum + calc = FragmentCalculator("PEPTIDEK", [], 2) + ions = calc.calculate_by_ions() if hasattr(calc, "calculate_by_ions") else calc.calculate_b_ions() + calc.calculate_y_ions() + peaks = sorted({round(i.mz, 5) for i in ions if i.charge == 1})[:4] + sp = Spectrum(scan_num=1, mz=np.array(peaks), intensity=np.full(len(peaks), 1e4), ms_level=2, + rt=1.0, precursor_mz=500.0, precursor_charge=2) + raw = canonical_match(ions, from_spectrum(sp, tolerance_ppm=10, deisotope_spectrum=False, max_charge=2)) + assert raw and {m.origin for m in raw} == {"raw"} + deiso = canonical_match(ions, from_spectrum(sp, tolerance_ppm=10, deisotope_spectrum=True, max_charge=2), + raw_fallback=True) + assert deiso and {m.origin for m in deiso} <= {"deisotoped", "raw_fallback"} From 28123a2eda65365b23266f93e06d913b31eb20b6 Mon Sep 17 00:00:00 2001 From: Longping Fu Date: Wed, 14 Oct 2026 11:06:16 -0400 Subject: [PATCH 4/5] match_peaks(exclusive=False): report every identity a peak fits The default stays exclusive: a peak explains one ion, the first in priority order. exclusive=False (also through canonical_match with the default matcher) lets each ion take its nearest peak, so ambiguous peaks are visible as such. Co-Authored-By: Claude Opus 5.5 --- src/mzml_utils/fragments.py | 9 +++++++-- src/mzml_utils/match_context.py | 10 ++++++++++ tests/test_fragments.py | 11 +++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) diff --git a/src/mzml_utils/fragments.py b/src/mzml_utils/fragments.py index 3e2fc0a..711f226 100644 --- a/src/mzml_utils/fragments.py +++ b/src/mzml_utils/fragments.py @@ -377,7 +377,8 @@ def match_peaks(theoretical_ions: List[TheoreticalIon], exp_intensity: np.ndarray, tolerance_ppm: float = 20.0, match_isotopes: bool = True, - max_isotope: int = 2) -> List[MatchedIon]: + max_isotope: int = 2, + exclusive: bool = True) -> List[MatchedIon]: """ Match experimental peaks to theoretical ions. @@ -388,6 +389,10 @@ def match_peaks(theoretical_ions: List[TheoreticalIon], tolerance_ppm: Mass tolerance in ppm. match_isotopes: Also try M+1, M+2 isotope peaks. max_isotope: Maximum isotope offset. + exclusive: Each peak explains at most one ion (default): ions are taken in priority + order (Y, then no-loss, then neutral-loss ions, by m/z) and a used peak is skipped. + ``False`` lets every ion take its nearest peak, so a peak that fits several + identities is reported once per identity -- the evidence before any choice. Returns: List of :class:`MatchedIon` objects. @@ -416,7 +421,7 @@ def _sort_key(ion: TheoreticalIon): if len(indices) == 0: continue for idx in indices[np.argsort(np.abs(exp_mz[indices] - target))]: - if idx in used_peaks: + if exclusive and idx in used_peaks: continue err = (exp_mz[idx] - target) / target * 1e6 ann = f"{ion.annotation}+{iso}" if iso > 0 else ion.annotation diff --git a/src/mzml_utils/match_context.py b/src/mzml_utils/match_context.py index ef11eaa..0bf95b7 100644 --- a/src/mzml_utils/match_context.py +++ b/src/mzml_utils/match_context.py @@ -14,6 +14,8 @@ """ from __future__ import annotations +import functools + from dataclasses import dataclass from typing import Callable, List, Optional @@ -107,6 +109,7 @@ def canonical_match( exclude_mzs: Optional[List[float]] = None, exclude_tol_ppm: float = 15.0, match_fn: Optional[Callable] = None, + exclusive: bool = True, ) -> List[MatchedIon]: """Canonical theoretical→experimental match. @@ -122,6 +125,9 @@ def canonical_match( 3. If ``exclude_mzs`` is provided, any match whose observed m/z is within ``exclude_tol_ppm`` of an exclusion target is dropped. + ``exclusive=False`` (default matcher only) lets one peak be reported for every identity it + fits, instead of the first in priority order. + Each match's ``origin`` says where it was found: ``deisotoped`` (step 1 on a deisotoped spectrum), ``raw`` (step 1 without deisotoping) or ``raw_fallback`` (step 2). @@ -137,6 +143,10 @@ def canonical_match( return [] mp = match_fn or _default_match_peaks + if not exclusive: + if match_fn is not None: + raise ValueError("exclusive=False needs the default matcher") + mp = functools.partial(_default_match_peaks, exclusive=False) primary = mp( ions, ctx.exp_mz, ctx.exp_intensity, diff --git a/tests/test_fragments.py b/tests/test_fragments.py index 4e8d11f..afda9ef 100644 --- a/tests/test_fragments.py +++ b/tests/test_fragments.py @@ -203,3 +203,14 @@ def test_canonical_match_records_origin(): deiso = canonical_match(ions, from_spectrum(sp, tolerance_ppm=10, deisotope_spectrum=True, max_charge=2), raw_fallback=True) assert deiso and {m.origin for m in deiso} <= {"deisotoped", "raw_fallback"} + + +def test_non_exclusive_matching_reports_every_identity_of_a_peak(): + import numpy as np + from mzml_utils.fragments import TheoreticalIon, match_peaks + ions = [TheoreticalIon(ion_type='b', ion_number=2, charge=1, mz=300.1, sequence='AB', annotation='a'), + TheoreticalIon(ion_type='y', ion_number=2, charge=1, mz=300.1, sequence='CD', annotation='b')] + mz, it = np.array([300.1]), np.array([1e4]) + assert len(match_peaks(ions, mz, it, match_isotopes=False)) == 1 + both = match_peaks(ions, mz, it, match_isotopes=False, exclusive=False) + assert {m.ion_type for m in both} == {'b', 'y'} From d5ed66bf8083b1a8de4d79f93d3bec7aa26e8a83 Mon Sep 17 00:00:00 2001 From: Longping Fu Date: Wed, 14 Oct 2026 11:06:16 -0400 Subject: [PATCH 5/5] changelog: match provenance Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14efde2..361bc1f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,10 @@ All notable changes to this project are documented here. The format follows `cap_peaks()` (msconvert's rule, ties at the threshold excluded), so one tool can run on capped and uncapped MS1 side by side. `ms1_cap_status()` / `warn_if_ms1_capped()` tell a tool when its MS1 is capped and unpatched. +- Match provenance: `MatchedIon.origin` (`deisotoped` / `raw` / `raw_fallback`, set by + `canonical_match`) and `match_peaks(exclusive=False)` / `canonical_match(exclusive=False)`, + which report a peak once per identity it fits instead of giving it to the first in priority + order. Defaults are unchanged. - `mzml_utils.ms1quant`: precursor MS1 intensity, IonQuant-like (sum of the top-3 isotope-cluster scans within rttol; nearest peak per isotope within mztol; minscans, minisotopes), parameters from `fragpipe.workflow`. Ported from NGlycoMM's `ms1_xic_extractor.py` and identical to it on