diff --git a/benchmark/README.md b/benchmark/README.md index b13235342a..d219238a4f 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -58,6 +58,8 @@ helper delivery neither launches trials nor changes frozen historical arms. ## Current work +- [Compare a local proxy with released scores](../docs/reference/benchmark-behavior-findings.md#compare-a-local-proxy-with-released-scores) covers source matching, missing feedback and the later-submission control for an offline study. + - [FrontierCS-2 compatibility repair v2](frontiercs2/README.md) provides an opt-in patch and synthetic HTTP/client/Docker checks for the frozen single-host v1 sources. It preserves historical attribution and does not launch a model or diff --git a/benchmark/proxy_order.py b/benchmark/proxy_order.py new file mode 100644 index 0000000000..1d94ed2a5f --- /dev/null +++ b/benchmark/proxy_order.py @@ -0,0 +1,140 @@ +"""Compare recorded proxy rankings with released native scores in one fixed cohort.""" + +import argparse +from dataclasses import dataclass +import json +import re +from decimal import Decimal +from itertools import combinations +from pathlib import Path + + +def is_finite_number(value): + # JSON floats are decoded as Decimal so close scores keep their ordering. + return type(value) is int or (isinstance(value, Decimal) and value.is_finite()) + + +def load_panel(path, *, clock, allow_unscored): + panel = json.loads(path.read_bytes(), parse_float=Decimal) + if not isinstance(panel, dict) or not isinstance(panel.get("rows"), list): + raise ValueError("expected a panel with a rows array") + if not isinstance(panel.get("cohort"), str) or not panel["cohort"].strip(): + raise ValueError("each panel needs a cohort identifier") + if panel.get("direction") not in ("minimize", "maximize"): + raise ValueError("each panel needs its metric direction") + by_source = {} + for row in panel["rows"]: + if not isinstance(row, dict): + raise ValueError("expected a row object") + source = row.get("source_sha256") + if not isinstance(source, str) or re.fullmatch(r"[0-9a-f]{64}", source) is None: + raise ValueError("expected a lowercase candidate SHA-256") + if source in by_source: + raise ValueError( + "multiple observations for one source: select a repeat policy first" + ) + if not is_finite_number(row.get(clock)): + raise ValueError(f"expected finite {clock} on a common clock") + if "score" not in row or not ( + is_finite_number(row["score"]) or (allow_unscored and row["score"] is None) + ): + raise ValueError( + "expected a finite score or an explicitly unscored submission" + ) + by_source[source] = row + return panel["cohort"], panel["direction"], by_source + + +@dataclass(frozen=True) +class MatchedScore: + proxy: Decimal | int + native: Decimal | int + accepted_at: Decimal | int + + +def sign(left, right): + return (left > right) - (left < right) + + +def order(left, right, direction): + return sign(left, right) if direction == "maximize" else sign(right, left) + + +def compare(proxy_path, native_path): + cohort, proxy_direction, predictions = load_panel( + proxy_path, clock="observed_at", allow_unscored=False + ) + native_cohort, native_direction, submissions = load_panel( + native_path, clock="accepted_at", allow_unscored=True + ) + if cohort != native_cohort: + raise ValueError("proxy and native panels must describe the same cohort") + coverage = dict( + native_candidates=len(submissions), + proxy_only=len(predictions.keys() - submissions.keys()), + native_unscored=0, + missing_proxy=0, + late_or_simultaneous_proxy=0, + matched=0, + ) + matched = [] + for source, native in submissions.items(): + proxy = predictions.get(source) + if native["score"] is None: + coverage["native_unscored"] += 1 + elif proxy is None: + coverage["missing_proxy"] += 1 + elif proxy["observed_at"] >= native["accepted_at"]: + coverage["late_or_simultaneous_proxy"] += 1 + else: + coverage["matched"] += 1 + matched.append( + MatchedScore(proxy["score"], native["score"], native["accepted_at"]) + ) + + # Both rules see exactly the same pairs. A flat proxy abstains on that pair. + rules = { + rule: dict(agrees=0, reverses=0, abstains=0) + for rule in ("proxy", "later_submission") + } + native_ties = 0 + for left, right in combinations(matched, 2): + actual = order(left.native, right.native, native_direction) + if actual == 0: + native_ties += 1 + continue + decisions = { + "proxy": order(left.proxy, right.proxy, proxy_direction), + "later_submission": sign(left.accepted_at, right.accepted_at), + } + for rule, prediction in decisions.items(): + if prediction == 0: + rules[rule]["abstains"] += 1 + elif prediction == actual: + rules[rule]["agrees"] += 1 + else: + rules[rule]["reverses"] += 1 + return dict( + cohort=cohort, coverage=coverage, native_tied_pairs=native_ties, rules=rules + ) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "proxy", type=Path, help="local scores on one common proxy panel" + ) + parser.add_argument( + "native", + type=Path, + help="all accepted candidates in the selected native window", + ) + args = parser.parse_args() + try: + print(json.dumps(compare(args.proxy, args.native), indent=2)) + except (ValueError, OSError) as exc: + parser.exit(2, f"Cannot compare panels: {exc}\n") + + +if __name__ == "__main__": + main() diff --git a/benchmark/tests/test_proxy_order.py b/benchmark/tests/test_proxy_order.py new file mode 100644 index 0000000000..c347436b7d --- /dev/null +++ b/benchmark/tests/test_proxy_order.py @@ -0,0 +1,96 @@ +import json + +import pytest + +from benchmark.proxy_order import compare + + +def panel(tmp_path, name, rows, direction, cohort="fixed"): + clock = "observed_at" if name == "proxy" else "accepted_at" + data = [ + dict(source_sha256=f"{source:064x}", score=score, **{clock: timestamp}) + for source, score, timestamp in rows + ] + path = tmp_path / name + path.write_text(json.dumps(dict(cohort=cohort, direction=direction, rows=data))) + return path + + +def test_compare_with_later_wins_on_the_same_pairs(tmp_path): + proxy = panel( + tmp_path, + "proxy", + [(1, 2, 1), (2, 1, 1), (3, 1, 1), (4, 2, 6), (6, 0, 1)], + "minimize", + ) + native = panel( + tmp_path, + "native", + [(3, 11, 5), (2, 12, 3), (1, 11, 2), (4, 15, 6), (5, 16, 8), (7, None, 9)], + "maximize", + ) + result = compare(proxy, native) + assert result["coverage"] == dict( + native_candidates=6, + matched=3, + proxy_only=1, + missing_proxy=1, + late_or_simultaneous_proxy=1, + native_unscored=1, + ) + assert result["native_tied_pairs"] == 1 + assert result["rules"]["proxy"] == dict(agrees=1, reverses=0, abstains=1) + assert result["rules"]["later_submission"] == dict(agrees=1, reverses=1, abstains=0) + + +def test_reversal_and_simultaneous_submissions(tmp_path): + proxy = panel(tmp_path, "proxy", [(1, 0, 0), (2, 1, 0)], "maximize") + native = panel(tmp_path, "native", [(1, 0, 1), (2, -1, 1)], "maximize") + result = compare(proxy, native) + assert result["rules"]["proxy"]["reverses"] == 1 + assert result["rules"]["later_submission"]["abstains"] == 1 + + +def test_only_one_matching_forecast(tmp_path): + proxy = panel(tmp_path, "proxy", [(1, 0, 0), (2, 1, 0)], "maximize") + native = panel(tmp_path, "native", [(1, 0, 1)], "maximize") + result = compare(proxy, native) + assert result["coverage"]["proxy_only"] == 1 + assert all(sum(rule.values()) == 0 for rule in result["rules"].values()) + + +@pytest.mark.parametrize( + "rows", + [ + [(1, 0, 0), (1, 1, 1)], + [(1, float("nan"), 0)], + [(1, False, 0)], + [(1, 1, None)], + [(1, None, 0)], + ], +) +def test_ambiguous_or_invalid_local_evidence(tmp_path, rows): + with pytest.raises(ValueError): + compare( + panel(tmp_path, "proxy", rows, "maximize"), + panel(tmp_path, "native", [], "maximize"), + ) + + +def test_reject_mixed_cohorts(tmp_path): + with pytest.raises(ValueError, match="same cohort"): + compare( + panel(tmp_path, "proxy", [], "maximize"), + panel(tmp_path, "native", [], "maximize", "other"), + ) + + +def test_scores_below_float_precision_are_ordered(tmp_path): + proxy = panel(tmp_path, "proxy", [(1, 1, 1), (2, 2, 1)], "maximize") + native = panel(tmp_path, "native", [(1, 0, 2), (2, 0, 3)], "maximize") + native.write_text( + native.read_text() + .replace('"score": 0', '"score": 1.0000000000000000002', 1) + .replace('"score": 0', '"score": 1.0000000000000000001', 1) + ) + assert compare(proxy, native)["rules"]["proxy"]["reverses"] == 1 diff --git a/docs/reference/benchmark-behavior-findings.md b/docs/reference/benchmark-behavior-findings.md index f7db24906e..87df154950 100644 --- a/docs/reference/benchmark-behavior-findings.md +++ b/docs/reference/benchmark-behavior-findings.md @@ -77,6 +77,47 @@ do not infer permission for outcome statistics from permission to share effort. The local provider performs no network upload. Remote publication requires an independently authorized provider and an explicit content review. +## Compare a local proxy with released scores + +For one run, compare the proxy with a simple control that always picks the +later submission. A search that improves over time can give both rules a high +agreement count. The offline [analysis script](../../benchmark/proxy_order.py) +compares the two rules on exactly the same pairs. + +Run `python -m benchmark.proxy_order proxy.json native.json` from the checkout. +Both files have a common `cohort` string, their own metric `direction` +(`minimize` or `maximize`) and a `rows` array. For example: + +```json +{"cohort":"example","direction":"minimize","rows":[ + {"source_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa","observed_at":10,"score":2}, + {"source_sha256":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb","observed_at":20,"score":1}]} +``` + +In the native file, use the accepted source digest, `accepted_at` on the +same clock, and the released score (`null` until a valid score is available). +Select one case, protocol, scoring version, budget and fixed local panel first. +Hash the same candidate content on both sides and include every accepted +candidate in the window, even when its local score is missing. For repeat +measurements choose a rule before inspecting the scores. The script rejects +duplicate source rows. A matching cohort name records the analyst's selection; +the analyst must verify comparable inputs, source digests and timestamps. + +The output distinguishes missing coverage, native ties, abstentions, agreements +and reversals. Each local result must precede acceptance of its own candidate. +Pairs share candidates and the solver may already have learned from older +feedback. Counts describe the selected sequence. They give neither independent +trials nor a population accuracy estimate. Close scores can change order under +timing noise. Check same-source repeats where available and keep the ordering +provisional when uncertainty is unknown. To test a prospective choice, freeze +the alternatives and the choice before their respective outcomes are known. +If only one alternative has a label, absolute error can still be measured +but the ranking between alternatives remains unknown. + +Use the coverage and counts in a redacted behavior finding when authorized; +retain the study selection, uncertainty and counterexamples alongside the +measurements. This analysis reads caller-supplied local files only. + ## 中文说明 行为发现用于分享局部 pattern 与案例解释；无需先上传完整实验数据。它复用