From 93231ce98f25e29c1176ef118c6832bf70b3920a Mon Sep 17 00:00:00 2001 From: Zach Vorhies Date: Fri, 23 Oct 2026 16:37:42 -0700 Subject: [PATCH] ci: perf-ab sanity gate judges head against max(base, null arm) (#620) A single bimodal base sample (drain RSS 48.3 vs null 208.0 MiB in run 39896572843) aborted a campaign the null arm reproduced. With --null-arm, screen head against the per-metric worse of the two base runs. Local-Gate: v1 tree=2513449976a263b4ffdbcad5abf733bd38e8f940 secs=62 lanes=lint:run Ci-Attestation: {"at":1792798726,"gate":"python/all/lint","host":"linux-x86_64","key":"9241646da94324de4e0c5f44e6050bef2db7f18250ef225a4226a804f195ca1b","lane":"lint","parents":["aaef948845f03eecd7e80b29196a4e891bf55444"],"secs":62,"stamp":"ea51a21011d8b9a78e7a952ebddb4aab","tree":"2513449976a263b4ffdbcad5abf733bd38e8f940","v":1,"via":"run"} --- ci/perf_ab.py | 18 +++++++++++++++++- ci/tests/test_perf_ab.py | 13 +++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/ci/perf_ab.py b/ci/perf_ab.py index b7c656cc..830474b2 100644 --- a/ci/perf_ab.py +++ b/ci/perf_ab.py @@ -400,6 +400,15 @@ def build( SANITY_SLACK_MIB = 8.0 +def sanity_reference(base: list[float], null: list[float] | None) -> list[float]: + """The first-repetition sample head is screened against (#620). + + With a null arm, base's own spread is measured: take the worse of the two base runs per + metric, so a bimodal base sample cannot abort a campaign the null arm reproduces. + """ + return base if null is None else [max(b, n) for b, n in zip(base, null)] + + def sanity_failures( first: dict[tuple[str, str], list[float]], workloads: Iterable[str], @@ -959,8 +968,15 @@ def run_rep(rep: int, rows: Iterable[str]) -> None: run_rep(rep, workloads) if rep == 0 and not args.no_sanity_gate: broken: list[str] = [] + reference = { + w: sanity_reference( + samples[(w, "base")][0], + samples[(w, NULL_ARM)][0] if args.null_arm else None, + ) + for w in workloads + } for arm in head_arms: - first = {(w, "base"): samples[(w, "base")][0] for w in workloads} + first = {(w, "base"): reference[w] for w in workloads} first |= {(w, "head"): samples[(w, arm)][0] for w in workloads} broken += [f"[{arm}] {m}" for m in sanity_failures(first, workloads)] if broken: # fail in minutes, not after the whole run (#573) diff --git a/ci/tests/test_perf_ab.py b/ci/tests/test_perf_ab.py index 0f490403..28c02e1b 100644 --- a/ci/tests/test_perf_ab.py +++ b/ci/tests/test_perf_ab.py @@ -373,6 +373,19 @@ def test_sanity_gate_ignores_cpu_and_throughput() -> None: assert perf_ab.sanity_failures(_first(base, head), ["row"]) == [] +def test_sanity_gate_reference_absorbs_null_arm_spread() -> None: + # #620: run 39896572843 -- base 48.3, head 208.5, null (same base binary) 208.0 MiB + drain = "RSS_0.5_s_after_drain_MiB" + base = _metrics(**{drain: 48.3}) + head = _metrics(**{drain: 208.5}) + null = _metrics(**{drain: 208.0}) + assert perf_ab.sanity_failures(_first(base, head), ["row"]) + reference = perf_ab.sanity_reference(base, null) + assert perf_ab.sanity_failures(_first(reference, head), ["row"]) == [] + # without a null arm the reference is base itself, so the gate keeps its teeth + assert perf_ab.sanity_reference(base, None) is base + + def test_replay_build_counts_events_and_prints_them() -> None: # #573 A1: the untimed replay carries MI_STAT=1 and asks for the statistics at exit assert "MI_STAT=1" in perf_ab.KIND_CPPDEFS[perf_ab.HOLES_KIND]