ci: perf-ab sanity gate judges head against max(base, null arm) (#620)

ci/perf_ab.py

@@ -400,6 +400,15 @@ def build(
 SANITY_SLACK_MIB = 8.0
 
 
+def sanity_reference(base: list[float], null: list[float] | None) -> list[float]:
+    """The first-repetition sample head is screened against (#620).
+
+    With a null arm, base's own spread is measured: take the worse of the two base runs per
+    metric, so a bimodal base sample cannot abort a campaign the null arm reproduces.
+    """
+    return base if null is None else [max(b, n) for b, n in zip(base, null)]
+
+
 def sanity_failures(
     first: dict[tuple[str, str], list[float]],
     workloads: Iterable[str],
@@ -959,8 +968,15 @@ def run_rep(rep: int, rows: Iterable[str]) -> None:
                 run_rep(rep, workloads)
                 if rep == 0 and not args.no_sanity_gate:
                     broken: list[str] = []
+                    reference = {
+                        w: sanity_reference(
+                            samples[(w, "base")][0],
+                            samples[(w, NULL_ARM)][0] if args.null_arm else None,
+                        )
+                        for w in workloads
+                    }
                     for arm in head_arms:
-                        first = {(w, "base"): samples[(w, "base")][0] for w in workloads}
+                        first = {(w, "base"): reference[w] for w in workloads}
                         first |= {(w, "head"): samples[(w, arm)][0] for w in workloads}
                         broken += [f"[{arm}] {m}" for m in sanity_failures(first, workloads)]
                     if broken:  # fail in minutes, not after the whole run (#573)

ci/tests/test_perf_ab.py

@@ -373,6 +373,19 @@ def test_sanity_gate_ignores_cpu_and_throughput() -> None:
     assert perf_ab.sanity_failures(_first(base, head), ["row"]) == []
 
 
+def test_sanity_gate_reference_absorbs_null_arm_spread() -> None:
+    # #620: run 39896572843 -- base 48.3, head 208.5, null (same base binary) 208.0 MiB
+    drain = "RSS_0.5_s_after_drain_MiB"
+    base = _metrics(**{drain: 48.3})
+    head = _metrics(**{drain: 208.5})
+    null = _metrics(**{drain: 208.0})
+    assert perf_ab.sanity_failures(_first(base, head), ["row"])
+    reference = perf_ab.sanity_reference(base, null)
+    assert perf_ab.sanity_failures(_first(reference, head), ["row"]) == []
+    # without a null arm the reference is base itself, so the gate keeps its teeth
+    assert perf_ab.sanity_reference(base, None) is base
+
+
 def test_replay_build_counts_events_and_prints_them() -> None:
     # #573 A1: the untimed replay carries MI_STAT=1 and asks for the statistics at exit
     assert "MI_STAT=1" in perf_ab.KIND_CPPDEFS[perf_ab.HOLES_KIND]