Compare benchmark proxy rankings against submission order

benchmark/README.md

@@ -58,6 +58,8 @@ helper delivery neither launches trials nor changes frozen historical arms.
 
 ## Current work
 
+- [Compare a local proxy with released scores](../docs/reference/benchmark-behavior-findings.md#compare-a-local-proxy-with-released-scores) covers source matching, missing feedback and the later-submission control for an offline study.
+
 - [FrontierCS-2 compatibility repair v2](frontiercs2/README.md) provides an opt-in
   patch and synthetic HTTP/client/Docker checks for the frozen single-host v1
   sources. It preserves historical attribution and does not launch a model or

benchmark/proxy_order.py

@@ -0,0 +1,140 @@
+"""Compare recorded proxy rankings with released native scores in one fixed cohort."""
+
+import argparse
+from dataclasses import dataclass
+import json
+import re
+from decimal import Decimal
+from itertools import combinations
+from pathlib import Path
+
+
+def is_finite_number(value):
+    # JSON floats are decoded as Decimal so close scores keep their ordering.
+    return type(value) is int or (isinstance(value, Decimal) and value.is_finite())
+
+
+def load_panel(path, *, clock, allow_unscored):
+    panel = json.loads(path.read_bytes(), parse_float=Decimal)
+    if not isinstance(panel, dict) or not isinstance(panel.get("rows"), list):
+        raise ValueError("expected a panel with a rows array")
+    if not isinstance(panel.get("cohort"), str) or not panel["cohort"].strip():
+        raise ValueError("each panel needs a cohort identifier")
+    if panel.get("direction") not in ("minimize", "maximize"):
+        raise ValueError("each panel needs its metric direction")
+    by_source = {}
+    for row in panel["rows"]:
+        if not isinstance(row, dict):
+            raise ValueError("expected a row object")
+        source = row.get("source_sha256")
+        if not isinstance(source, str) or re.fullmatch(r"[0-9a-f]{64}", source) is None:
+            raise ValueError("expected a lowercase candidate SHA-256")
+        if source in by_source:
+            raise ValueError(
+                "multiple observations for one source: select a repeat policy first"
+            )
+        if not is_finite_number(row.get(clock)):
+            raise ValueError(f"expected finite {clock} on a common clock")
+        if "score" not in row or not (
+            is_finite_number(row["score"]) or (allow_unscored and row["score"] is None)
+        ):
+            raise ValueError(
+                "expected a finite score or an explicitly unscored submission"
+            )
+        by_source[source] = row
+    return panel["cohort"], panel["direction"], by_source
+
+
+@dataclass(frozen=True)
+class MatchedScore:
+    proxy: Decimal | int
+    native: Decimal | int
+    accepted_at: Decimal | int
+
+
+def sign(left, right):
+    return (left > right) - (left < right)
+
+
+def order(left, right, direction):
+    return sign(left, right) if direction == "maximize" else sign(right, left)
+
+
+def compare(proxy_path, native_path):
+    cohort, proxy_direction, predictions = load_panel(
+        proxy_path, clock="observed_at", allow_unscored=False
+    )
+    native_cohort, native_direction, submissions = load_panel(
+        native_path, clock="accepted_at", allow_unscored=True
+    )
+    if cohort != native_cohort:
+        raise ValueError("proxy and native panels must describe the same cohort")
+    coverage = dict(
+        native_candidates=len(submissions),
+        proxy_only=len(predictions.keys() - submissions.keys()),
+        native_unscored=0,
+        missing_proxy=0,
+        late_or_simultaneous_proxy=0,
+        matched=0,
+    )
+    matched = []
+    for source, native in submissions.items():
+        proxy = predictions.get(source)
+        if native["score"] is None:
+            coverage["native_unscored"] += 1
+        elif proxy is None:
+            coverage["missing_proxy"] += 1
+        elif proxy["observed_at"] >= native["accepted_at"]:
+            coverage["late_or_simultaneous_proxy"] += 1
+        else:
+            coverage["matched"] += 1
+            matched.append(
+                MatchedScore(proxy["score"], native["score"], native["accepted_at"])
+            )
+
+    # Both rules see exactly the same pairs. A flat proxy abstains on that pair.
+    rules = {
+        rule: dict(agrees=0, reverses=0, abstains=0)
+        for rule in ("proxy", "later_submission")
+    }
+    native_ties = 0
+    for left, right in combinations(matched, 2):
+        actual = order(left.native, right.native, native_direction)
+        if actual == 0:
+            native_ties += 1
+            continue
+        decisions = {
+            "proxy": order(left.proxy, right.proxy, proxy_direction),
+            "later_submission": sign(left.accepted_at, right.accepted_at),
+        }
+        for rule, prediction in decisions.items():
+            if prediction == 0:
+                rules[rule]["abstains"] += 1
+            elif prediction == actual:
+                rules[rule]["agrees"] += 1
+            else:
+                rules[rule]["reverses"] += 1
+    return dict(
+        cohort=cohort, coverage=coverage, native_tied_pairs=native_ties, rules=rules
+    )
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument(
+        "proxy", type=Path, help="local scores on one common proxy panel"
+    )
+    parser.add_argument(
+        "native",
+        type=Path,
+        help="all accepted candidates in the selected native window",
+    )
+    args = parser.parse_args()
+    try:
+        print(json.dumps(compare(args.proxy, args.native), indent=2))
+    except (ValueError, OSError) as exc:
+        parser.exit(2, f"Cannot compare panels: {exc}\n")
+
+
+if __name__ == "__main__":
+    main()

benchmark/tests/test_proxy_order.py

@@ -0,0 +1,96 @@
+import json
+
+import pytest
+
+from benchmark.proxy_order import compare
+
+
+def panel(tmp_path, name, rows, direction, cohort="fixed"):
+    clock = "observed_at" if name == "proxy" else "accepted_at"
+    data = [
+        dict(source_sha256=f"{source:064x}", score=score, **{clock: timestamp})
+        for source, score, timestamp in rows
+    ]
+    path = tmp_path / name
+    path.write_text(json.dumps(dict(cohort=cohort, direction=direction, rows=data)))
+    return path
+
+
+def test_compare_with_later_wins_on_the_same_pairs(tmp_path):
+    proxy = panel(
+        tmp_path,
+        "proxy",
+        [(1, 2, 1), (2, 1, 1), (3, 1, 1), (4, 2, 6), (6, 0, 1)],
+        "minimize",
+    )
+    native = panel(
+        tmp_path,
+        "native",
+        [(3, 11, 5), (2, 12, 3), (1, 11, 2), (4, 15, 6), (5, 16, 8), (7, None, 9)],
+        "maximize",
+    )
+    result = compare(proxy, native)
+    assert result["coverage"] == dict(
+        native_candidates=6,
+        matched=3,
+        proxy_only=1,
+        missing_proxy=1,
+        late_or_simultaneous_proxy=1,
+        native_unscored=1,
+    )
+    assert result["native_tied_pairs"] == 1
+    assert result["rules"]["proxy"] == dict(agrees=1, reverses=0, abstains=1)
+    assert result["rules"]["later_submission"] == dict(agrees=1, reverses=1, abstains=0)
+
+
+def test_reversal_and_simultaneous_submissions(tmp_path):
+    proxy = panel(tmp_path, "proxy", [(1, 0, 0), (2, 1, 0)], "maximize")
+    native = panel(tmp_path, "native", [(1, 0, 1), (2, -1, 1)], "maximize")
+    result = compare(proxy, native)
+    assert result["rules"]["proxy"]["reverses"] == 1
+    assert result["rules"]["later_submission"]["abstains"] == 1
+
+
+def test_only_one_matching_forecast(tmp_path):
+    proxy = panel(tmp_path, "proxy", [(1, 0, 0), (2, 1, 0)], "maximize")
+    native = panel(tmp_path, "native", [(1, 0, 1)], "maximize")
+    result = compare(proxy, native)
+    assert result["coverage"]["proxy_only"] == 1
+    assert all(sum(rule.values()) == 0 for rule in result["rules"].values())
+
+
+@pytest.mark.parametrize(
+    "rows",
+    [
+        [(1, 0, 0), (1, 1, 1)],
+        [(1, float("nan"), 0)],
+        [(1, False, 0)],
+        [(1, 1, None)],
+        [(1, None, 0)],
+    ],
+)
+def test_ambiguous_or_invalid_local_evidence(tmp_path, rows):
+    with pytest.raises(ValueError):
+        compare(
+            panel(tmp_path, "proxy", rows, "maximize"),
+            panel(tmp_path, "native", [], "maximize"),
+        )
+
+
+def test_reject_mixed_cohorts(tmp_path):
+    with pytest.raises(ValueError, match="same cohort"):
+        compare(
+            panel(tmp_path, "proxy", [], "maximize"),
+            panel(tmp_path, "native", [], "maximize", "other"),
+        )
+
+
+def test_scores_below_float_precision_are_ordered(tmp_path):
+    proxy = panel(tmp_path, "proxy", [(1, 1, 1), (2, 2, 1)], "maximize")
+    native = panel(tmp_path, "native", [(1, 0, 2), (2, 0, 3)], "maximize")
+    native.write_text(
+        native.read_text()
+        .replace('"score": 0', '"score": 1.0000000000000000002', 1)
+        .replace('"score": 0', '"score": 1.0000000000000000001', 1)
+    )
+    assert compare(proxy, native)["rules"]["proxy"]["reverses"] == 1

docs/reference/benchmark-behavior-findings.md

@@ -77,6 +77,47 @@ do not infer permission for outcome statistics from permission to share effort.
 The local provider performs no network upload. Remote publication requires an
 independently authorized provider and an explicit content review.
 
+## Compare a local proxy with released scores
+
+For one run, compare the proxy with a simple control that always picks the
+later submission. A search that improves over time can give both rules a high
+agreement count. The offline [analysis script](../../benchmark/proxy_order.py)
+compares the two rules on exactly the same pairs.
+
+Run `python -m benchmark.proxy_order proxy.json native.json` from the checkout.
+Both files have a common `cohort` string, their own metric `direction`
+(`minimize` or `maximize`) and a `rows` array. For example:
+
+```json
+{"cohort":"example","direction":"minimize","rows":[
+  {"source_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","observed_at":10,"score":2},
+  {"source_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","observed_at":20,"score":1}]}
+```
+
+In the native file, use the accepted source digest, `accepted_at` on the
+same clock, and the released score (`null` until a valid score is available).
+Select one case, protocol, scoring version, budget and fixed local panel first.
+Hash the same candidate content on both sides and include every accepted
+candidate in the window, even when its local score is missing. For repeat
+measurements choose a rule before inspecting the scores. The script rejects
+duplicate source rows. A matching cohort name records the analyst's selection;
+the analyst must verify comparable inputs, source digests and timestamps.
+
+The output distinguishes missing coverage, native ties, abstentions, agreements
+and reversals. Each local result must precede acceptance of its own candidate.
+Pairs share candidates and the solver may already have learned from older
+feedback. Counts describe the selected sequence. They give neither independent
+trials nor a population accuracy estimate. Close scores can change order under
+timing noise. Check same-source repeats where available and keep the ordering
+provisional when uncertainty is unknown. To test a prospective choice, freeze
+the alternatives and the choice before their respective outcomes are known.
+If only one alternative has a label, absolute error can still be measured
+but the ranking between alternatives remains unknown.
+
+Use the coverage and counts in a redacted behavior finding when authorized;
+retain the study selection, uncertainty and counterexamples alongside the
+measurements. This analysis reads caller-supplied local files only.
+
 ## 中文说明
 
 行为发现用于分享局部 pattern 与案例解释；无需先上传完整实验数据。它复用