From 60cbba2f5bb02e52d0cdf591e2c182a70b4f1c02 Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 29 Sep 2026 11:29:25 +0000 Subject: [PATCH] feat(package): Let the sequential test take its error rates match-estimate takes --alpha and --beta, the chances of a wrong pass and a wrong fail the test accepts. Both are five percent unless given, as they were before, and must lie between nought and a half. They set Wald's bounds and nothing else: the ratio is the same whatever they are. A verdict at the default rates reads as it did. One reached at other rates names them in the report, the line and the trailer (`SPRT [0, 10] alpha=0.01 beta=0.01 passed`), since a pass at one rate is not a pass at another. An inconclusive one asks for the same rates again. The json adds alpha and beta beside the bounds they gave, which the format allows. The rates were fixed until now, on the grounds that they are what a verdict means. Naming them wherever a verdict is quoted keeps that meaning while letting a caller choose them. actions/summarise-match takes them as alpha and beta. The reusable workflows pin the actions at a release, so strength.yml can pass them once a release holds this. Checked against fastchess 1.8.2 on one match at alpha 0.2 and beta 0.01. Both printed bounds of (-4.38, 1.60) and a final ratio of 1.60, and both passed it. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018ccJFsmZ4jAKYmFNgNBh9u --- README.md | 13 ++- actions/summarise-match/action.yml | 10 ++ actions/summarise-match/summarise.sh | 10 +- mache/match_estimate.py | 116 ++++++++++++++++++---- tests/test_action_output.py | 33 ++++++- tests/test_match_estimate.py | 143 +++++++++++++++++++++++++++ 6 files changed, 297 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 9413ff6..f2475f2 100644 --- a/README.md +++ b/README.md @@ -121,8 +121,8 @@ how many games were drawn. A sequential test runs until the pairs settle it, and the closer its bounds, the longer that takes. In the simulation below, halving the gap between the bounds took between three and four times the pairs. It ran 300 tests a row of -the normalized test with mache's own code, judged every 250 pairs as with the -workflow's default batch. The pair scores were drawn from a distribution in +the normalized test with mache's own code, at the default error rates of five +percent each, judged every 250 pairs as with the workflow's default batch. The pair scores were drawn from a distribution in which three pairs in five score one point, shifted to each true difference: | bounds (normalized elo) | true difference at a bound | true difference halfway between | @@ -221,6 +221,15 @@ match-estimate strength-1-1-shard-*/games.pgn \ SPRT [0, 10] inconclusive. The log likelihood ratio over the 150 pairs of the test (75 from this batch and 75 from the batches before it) is 2.64 against bounds of (-2.94, 2.94). Over all of them the difference is +56 ±26 elo, which is the figure the trailer carries. The games so far settle it neither way. Launch another batch with prior_pairs set to 4,18,72,38,18. ``` +`--alpha` and `--beta` are the chances of a wrong pass and a wrong fail the test +accepts, five percent each unless given. They set Wald's bounds, which are +(-2.94, 2.94) at the defaults and (-4.60, 4.60) at one percent each, so +stricter rates take more pairs to settle a test. A verdict reached at other +rates names them wherever it is quoted, as in +`SPRT [0, 10] alpha=0.01 beta=0.01 passed`. Give every batch of one test the +same rates, since they are what the bounds were drawn at. +`actions/summarise-match` takes them as `alpha` and `beta`. + Three flags each replace the whole report. `--line` prints what release notes carry: diff --git a/actions/summarise-match/action.yml b/actions/summarise-match/action.yml index a723e6e..37da801 100644 --- a/actions/summarise-match/action.yml +++ b/actions/summarise-match/action.yml @@ -48,6 +48,14 @@ inputs: What elo0 and elo1 are differences in: `logistic` elo, or `normalized` elo. Every batch of one test is judged under the same one. default: logistic + alpha: + description: > + The chance of a wrong pass the test accepts. Every batch of one test is + judged at the same rates, since they set where the bounds are. + default: "0.05" + beta: + description: The chance of a wrong fail the test accepts. + default: "0.05" provenance: description: > One paragraph from the caller saying where the games are kept and what @@ -90,6 +98,8 @@ runs: ELO1: ${{ inputs.elo1 }} PRIOR_PAIRS: ${{ inputs.prior_pairs }} SPRT_MODEL: ${{ inputs.sprt_model }} + SPRT_ALPHA: ${{ inputs.alpha }} + SPRT_BETA: ${{ inputs.beta }} PROVENANCE: ${{ inputs.provenance }} run: | # the report itself goes to the log, so this is not redirected. The diff --git a/actions/summarise-match/summarise.sh b/actions/summarise-match/summarise.sh index d38fb11..ae7fc98 100755 --- a/actions/summarise-match/summarise.sh +++ b/actions/summarise-match/summarise.sh @@ -7,9 +7,10 @@ # summarise.sh # # The caller sets CANDIDATE, CANDIDATE_SHA, BASELINE, BASELINE_SHA, SHARDS, -# TIME_CONTROL, SPRT, ELO0, ELO1, PRIOR_PAIRS, SPRT_MODEL and PROVENANCE. An -# unset SPRT_MODEL is the logistic model, which is what a caller got before -# there was a choice. +# TIME_CONTROL, SPRT, ELO0, ELO1, PRIOR_PAIRS, SPRT_MODEL, SPRT_ALPHA, +# SPRT_BETA and PROVENANCE. An unset SPRT_MODEL is the logistic model, and an +# unset rate is five percent, which is what a caller got before there was a +# choice. # # Writes report.md, and writes the `key=value` lines the caller wants into # . The report goes to stdout, so it is in the job's log as well @@ -47,7 +48,8 @@ estimate=(python3 -m mache.match_estimate "${games[@]}" --tc "$TIME_CONTROL") if [ "$SPRT" = "true" ]; then estimate+=(--elo0 "$ELO0" --elo1 "$ELO1" --prior-pairs "$PRIOR_PAIRS" - --model "${SPRT_MODEL:-logistic}") + --model "${SPRT_MODEL:-logistic}" + --alpha "${SPRT_ALPHA:-0.05}" --beta "${SPRT_BETA:-0.05}") fi # Named for what writes it. The caller redirects this script's own stderr into diff --git a/mache/match_estimate.py b/mache/match_estimate.py index 71bd047..ab5db6d 100644 --- a/mache/match_estimate.py +++ b/mache/match_estimate.py @@ -25,7 +25,8 @@ the batch are added to the pairs the earlier batches of the same test played, and the log likelihood ratio over all of them is judged against Wald's bounds. Looking only at batch boundaries is what leaves the bounds meaning what they -say. +say. `--alpha` and `--beta` set the error rates the bounds are drawn at, five +percent each by default. The report goes to stdout for the run summary. `--line` prints the one line the release notes carry and `--trailer` the trailer a commit does. `--json` prints @@ -79,13 +80,16 @@ # the sequential test weighs PAIR = tuple(score / 2 for score in PENTANOMIAL) -# The error rates the sprt verdicts are accepted at: a wrong "passed" one run -# in twenty, a wrong "failed" the same. They are what the verdicts mean, so -# they are fixed here rather than asked for. +# The error rates an sprt verdict is accepted at unless the caller asks for +# others: a wrong "passed" one test in twenty, a wrong "failed" the same. They +# are what a verdict means, so a verdict reached at any other rates names them. ALPHA = BETA = 0.05 # Wald's bounds on the log likelihood ratio at those rates, -2.94 and +2.94 LOWER = math.log(BETA / (1 - ALPHA)) UPPER = math.log((1 - BETA) / ALPHA) +# An error rate is the chance of a wrong verdict, so it is above nought, and at +# a half or more the test does no better than tossing a coin for it. +MAX_RATE = 0.5 # a count of nought has no logarithm, so it is nudged off nought first, which # is what fastchess does with the same counts REGULARISED = 1e-3 @@ -700,7 +704,11 @@ class Sprt: The model says what the hypotheses are differences in: logistic elo, or normalized elo. The counts are the same under either, so a test could be carried on under the other model, but the error rates hold only for a - test judged under the one it started with.""" + test judged under the one it started with. + + The same goes for alpha and beta, the chances of a wrong "passed" and a + wrong "failed". They set where the bounds are, so a test carried on at + other rates is being judged against other bounds part way through.""" def __init__( self, @@ -709,10 +717,19 @@ def __init__( elo1: float, prior: list[int] | None = None, model: str = "logistic", + alpha: float = ALPHA, + beta: float = BETA, ): if model not in MODELS: raise ValueError(f"the model is one of {', '.join(MODELS)}, not {model}") + for name, rate in (("alpha", alpha), ("beta", beta)): + if not 0 < rate < MAX_RATE: + raise ValueError(f"{name} is between 0 and {MAX_RATE:g}, not {rate:g}") self.elo0, self.elo1, self.model = elo0, elo1, model + self.alpha, self.beta = alpha, beta + # Wald's bounds at these rates + self.lower = math.log(beta / (1 - alpha)) + self.upper = math.log((1 - beta) / alpha) counted = Counter(pairs) self.batch = [counted[score] for score in PENTANOMIAL] self.prior = list(prior) if prior else [0] * len(PENTANOMIAL) @@ -726,9 +743,9 @@ def __init__( if sum(self.counts) else 0.0 ) - if self.llr >= UPPER: + if self.llr >= self.upper: self.verdict = "passed" - elif self.llr <= LOWER: + elif self.llr <= self.lower: self.verdict = "failed" else: self.verdict = "inconclusive" @@ -739,7 +756,16 @@ def hypotheses(self) -> str: # there was a choice, so it stays as it was and the normalized one # names itself unit = " nElo" if self.model == "normalized" else "" - return f"[{number(self.elo0)}, {number(self.elo1)}]{unit}" + return f"[{number(self.elo0)}, {number(self.elo1)}]{unit}{self.rates}" + + @property + def rates(self) -> str: + """The error rates, where they are not the defaults. A verdict at the + defaults reads as it always has, and one at any other rates says so + wherever it is quoted.""" + if (self.alpha, self.beta) == (ALPHA, BETA): + return "" + return f" alpha={self.alpha:g} beta={self.beta:g}" @property def unit(self) -> str: @@ -748,7 +774,7 @@ def unit(self) -> str: @property def bounds(self) -> str: - return f"({number(LOWER)}, {number(UPPER)})" + return f"({number(self.lower)}, {number(self.upper)})" @property def estimate(self) -> Estimate: @@ -778,28 +804,49 @@ def __str__(self) -> str: ) +def percent(rate: float) -> str: + # :g rather than a fixed precision, so that 0.05 reads as 5 and 0.025 as 2.5 + return f"{rate * 100:g} percent" + + def sequential(sprt: Sprt) -> str: """The sprt reading for the report, and what its verdict means.""" + if sprt.alpha == sprt.beta: + rates = f"at a {percent(sprt.alpha)} error rate each way" + else: + rates = ( + f"at error rates of {percent(sprt.alpha)} for a wrong pass and" + f" {percent(sprt.beta)} for a wrong fail" + ) + # what the next batch has to be given besides the counts, so that it is + # judged the way this one was + again = [] + if sprt.model == "normalized": + again.append("the normalized model") + if sprt.rates: + # named once already, where the paragraph opens with the hypotheses + again.append("the same alpha and beta") means = { "passed": ( f"The pairs favour a difference of about {number(sprt.elo1)}" - f" {sprt.unit} over one of about {number(sprt.elo0)}, at a five" - " percent error rate each way. That is the hypothesis the test prefers and not a" - " floor under the difference: the estimate above is what the games" - " measured." + f" {sprt.unit} over one of about {number(sprt.elo0)}, {rates}." + " That is the hypothesis the test prefers and not a floor under the" + " difference: the estimate above is what the games measured." ), "failed": ( f"The pairs favour a difference of about {number(sprt.elo0)}" - f" {sprt.unit} over one of about {number(sprt.elo1)}, at a five" - " percent error rate each way. That does not show the candidate is weaker, only" - " that the games did not favour the larger difference." + f" {sprt.unit} over one of about {number(sprt.elo1)}, {rates}." + " That does not show the candidate is weaker, only that the games did" + " not favour the larger difference." ), "inconclusive": ( "The games so far settle it neither way. Launch another batch with" f" prior_pairs set to {sprt.carried}" + ( - " and the normalized model again." - if sprt.model == "normalized" + f" and {again[0]} again." + if len(again) == 1 + else f", {' and '.join(again)} again." + if again else "." ) ), @@ -1101,8 +1148,10 @@ def as_json( "elo0": sprt.elo0, "elo1": sprt.elo1, "llr": sprt.llr, - "lower": LOWER, - "upper": UPPER, + "alpha": sprt.alpha, + "beta": sprt.beta, + "lower": sprt.lower, + "upper": sprt.upper, "verdict": sprt.verdict, "batch": sprt.batch, "prior": sprt.prior, @@ -1177,6 +1226,16 @@ def main() -> None: help="what --elo0 and --elo1 are differences in: logistic elo, the" " default, or normalized elo", ) + parser.add_argument( + "--alpha", + type=float, + help=f"the chance of a wrong pass the sprt accepts, {ALPHA:g} by default", + ) + parser.add_argument( + "--beta", + type=float, + help=f"the chance of a wrong fail the sprt accepts, {BETA:g} by default", + ) printed = parser.add_mutually_exclusive_group() printed.add_argument( "--line", @@ -1207,6 +1266,13 @@ def main() -> None: parser.error("--model says what an sprt's bounds are in, so it wants --elo0") if args.elo0 is None and any(prior): parser.error("--prior-pairs carries a test on, so it wants --elo0 and --elo1") + for name, rate in (("--alpha", args.alpha), ("--beta", args.beta)): + if rate is None: + continue + if args.elo0 is None: + parser.error(f"{name} is an error rate of an sprt, so it wants --elo0") + if not 0 < rate < MAX_RATE: + parser.error(f"{name} is between 0 and {MAX_RATE:g}, not {rate:g}") if args.elo0 is not None: cap = MAX_NORMALIZED if args.model == "normalized" else MAX_HYPOTHESIS for name, elo in (("--elo0", args.elo0), ("--elo1", args.elo1)): @@ -1229,7 +1295,15 @@ def main() -> None: estimate = Estimate(sum(games), len(games), pairs) try: sprt = ( - Sprt(pairs, args.elo0, args.elo1, prior, args.model) + Sprt( + pairs, + args.elo0, + args.elo1, + prior, + args.model, + ALPHA if args.alpha is None else args.alpha, + BETA if args.beta is None else args.beta, + ) if args.elo0 is not None else None ) diff --git a/tests/test_action_output.py b/tests/test_action_output.py index b578cad..ed668ff 100644 --- a/tests/test_action_output.py +++ b/tests/test_action_output.py @@ -159,7 +159,7 @@ def games(tmp_path): return tmp_path, stub -def summarise(root, stub, outputs, sprt="false", model=None): +def summarise(root, stub, outputs, sprt="false", model=None, rates=None): return subprocess.run( [str(SUMMARISE), "shards", str(outputs)], cwd=root, @@ -180,6 +180,9 @@ def summarise(root, stub, outputs, sprt="false", model=None): "ELO1": "5", "PRIOR_PAIRS": "", **({} if model is None else {"SPRT_MODEL": model}), + **( + {} if rates is None else {"SPRT_ALPHA": rates[0], "SPRT_BETA": rates[1]} + ), "PROVENANCE": "", "GITHUB_STEP_SUMMARY": str(root / "step_summary"), "GITHUB_RUN_ATTEMPT": "1", @@ -289,6 +292,34 @@ def test_the_model_reaches_the_estimate(self, games, model, expected): for asked in (root / "asked").read_text().splitlines(): assert f"--model {expected}" in asked, asked + @pytest.mark.parametrize( + "rates,expected", + [(None, ("0.05", "0.05")), (("0.01", "0.1"), ("0.01", "0.1"))], + ) + def test_the_error_rates_reach_the_estimate(self, games, rates, expected): + # unset rates are the five percent a caller had before the choice + root, stub = games + executable( + stub / "python3", + f""" + if [ "$1" = "-m" ] && [ "$2" = "mache.match_estimate" ]; then + echo "$@" >> {root / "asked"} + for argument in "$@"; do + [ "$argument" = "--json" ] && echo '{{"sprt": null}}' && exit 0 + done + echo "+3 ±2 Elo (200 games)" + exit 0 + fi + exec /usr/bin/python3 "$@" + """, + ) + outputs = root / "outputs" + result = summarise(root, stub, outputs, sprt="true", rates=rates) + assert result.returncode == 0, result.stderr + alpha, beta = expected + for asked in (root / "asked").read_text().splitlines(): + assert f"--alpha {alpha} --beta {beta}" in asked, asked + def test_no_shard_uploaded_any_games_is_an_error(self, games): root, stub = games (root / "shards" / "shard-0" / "games.pgn").unlink() diff --git a/tests/test_match_estimate.py b/tests/test_match_estimate.py index b166cd5..ed78efc 100644 --- a/tests/test_match_estimate.py +++ b/tests/test_match_estimate.py @@ -736,6 +736,149 @@ def test_a_test_that_failed_says_what_that_does_not_show(self, tmp_path): assert "That does not show the candidate is weaker" in printed +class TestErrorRates: + """Alpha and beta, the chances of a wrong pass and a wrong fail. + + They move Wald's bounds and nothing else: the ratio is the same whatever + they are, so the same pairs can pass at one pair of rates and settle + nothing at a stricter one. A verdict at the defaults reads as it did before + the rates could be asked for, and one at any other rates names them + wherever it is quoted, since a pass at one in five is not a pass at one in + twenty.""" + + # a ratio of about 3.40 over [0, 10], between the bounds at five percent + # (2.94) and at one percent (4.60) + COUNTS = (0, 0, 100, 0, 20) + + def test_the_bounds_are_walds_at_the_rates_asked_for(self): + for alpha, beta in ((0.05, 0.05), (0.01, 0.01), (0.2, 0.01), (0.01, 0.1)): + sprt = match_estimate.Sprt([], 0, 10, alpha=alpha, beta=beta) + assert math.isclose(sprt.lower, math.log(beta / (1 - alpha))) + assert math.isclose(sprt.upper, math.log((1 - beta) / alpha)) + + def test_the_defaults_are_the_bounds_the_verdicts_always_had(self): + sprt = match_estimate.Sprt([], 0, 10) + assert (sprt.alpha, sprt.beta) == (0.05, 0.05) + assert sprt.bounds == "(-2.94, 2.94)" + assert sprt.rates == "" + + def test_stricter_rates_can_leave_a_pass_unsettled(self): + default = match_estimate.Sprt([], 0, 10, self.COUNTS) + strict = match_estimate.Sprt([], 0, 10, self.COUNTS, alpha=0.01, beta=0.01) + assert default.llr == strict.llr + assert default.verdict == "passed" + assert strict.verdict == "inconclusive" + assert strict.bounds == "(-4.60, 4.60)" + + def test_the_rates_can_differ(self): + # a loose alpha brings the upper bound in and a tight beta pushes the + # lower one out, so the two are not mirror images + sprt = match_estimate.Sprt([], 0, 10, alpha=0.2, beta=0.01) + assert sprt.bounds == "(-4.38, 1.60)" + + @pytest.mark.parametrize("rate", [0, -0.1, 0.5, 0.9, math.nan]) + def test_a_rate_that_is_not_a_chance_of_a_wrong_verdict_is_refused(self, rate): + with pytest.raises(ValueError): + match_estimate.Sprt([], 0, 10, alpha=rate) + with pytest.raises(ValueError): + match_estimate.Sprt([], 0, 10, beta=rate) + + def test_every_form_names_rates_that_are_not_the_defaults(self, tmp_path): + arguments = ("--elo0", "0", "--elo1", "10", "--alpha", "0.01") + arguments += ("--beta", "0.01") + run = TestCommandLine().run + line = run(tmp_path / "a", [batch(self.COUNTS)], *arguments, "--line") + assert line.returncode == 0, line.stderr + assert "SPRT [0, 10] alpha=0.01 beta=0.01 inconclusive" in line.stdout + assert "(-4.60, 4.60)" in line.stdout + trailer = run(tmp_path / "b", [batch(self.COUNTS)], *arguments, "--trailer") + assert "sprt [0, 10] alpha=0.01 beta=0.01 inconclusive" in trailer.stdout + report = run(tmp_path / "c", [batch(self.COUNTS)], *arguments).stdout + assert "SPRT [0, 10] alpha=0.01 beta=0.01 inconclusive." in report + assert "against bounds of (-4.60, 4.60)" in report + assert "and the same alpha and beta again." in report + + def test_the_defaults_are_not_named(self, tmp_path): + result = TestCommandLine().run( + tmp_path, [batch(self.COUNTS)], "--elo0", "0", "--elo1", "10", "--line" + ) + assert "SPRT [0, 10] passed, LLR 3.40 (-2.94, 2.94)" in result.stdout + assert "alpha" not in result.stdout + + def test_asking_for_the_defaults_reads_as_leaving_them_out(self, tmp_path): + run = TestCommandLine().run + base = ("--elo0", "0", "--elo1", "10", "--line") + left_out = run(tmp_path / "a", [batch(self.COUNTS)], *base).stdout + asked = run( + tmp_path / "b", [batch(self.COUNTS)], *base, "--alpha", "0.05" + ).stdout + assert left_out == asked + + def test_a_verdict_says_the_rates_it_was_reached_at(self, tmp_path): + shards, estimate, text = pooled(tmp_path, [drawn(1) + pair(2)]) + pairs = [score for one in shards for score in one.pairs] + passed = match_estimate.report( + shards, + estimate, + text, + match_estimate.Sprt(pairs, 0, 10, self.COUNTS, alpha=0.2, beta=0.01), + ) + assert "SPRT [0, 10] alpha=0.2 beta=0.01 passed." in passed + assert ( + "at error rates of 20 percent for a wrong pass and 1 percent for a" + " wrong fail." in passed + ) + default = match_estimate.report( + shards, estimate, text, match_estimate.Sprt(pairs, 0, 10, self.COUNTS) + ) + assert "at a 5 percent error rate each way." in default + + def test_a_normalized_test_at_other_rates_asks_for_both_again(self, tmp_path): + shards, estimate, text = pooled(tmp_path, [drawn(1) + pair(2)]) + pairs = [score for one in shards for score in one.pairs] + printed = match_estimate.report( + shards, + estimate, + text, + match_estimate.Sprt(pairs, 0, 5, None, "normalized", 0.01, 0.01), + ) + assert ", the normalized model and the same alpha and beta again." in printed + + @pytest.mark.parametrize("flag", ["--alpha", "--beta"]) + def test_a_rate_wants_a_test(self, tmp_path, flag): + result = TestCommandLine().run(tmp_path, [drawn(1)], flag, "0.01") + assert result.returncode != 0 + assert "so it wants --elo0" in result.stderr + + @pytest.mark.parametrize("flag", ["--alpha", "--beta"]) + @pytest.mark.parametrize("rate", ["0", "0.5", "-0.01", "nan"]) + def test_a_rate_outside_nought_and_a_half_is_refused(self, tmp_path, flag, rate): + result = TestCommandLine().run( + tmp_path, [drawn(1)], "--elo0", "0", "--elo1", "10", flag, rate + ) + assert result.returncode != 0 + assert "is between 0 and 0.5" in result.stderr + assert "Traceback" not in result.stderr + + def test_the_json_carries_the_rates_and_the_bounds_they_gave(self, tmp_path): + written = TestJson().loaded( + tmp_path, + [batch(self.COUNTS)], + "--elo0", + "0", + "--elo1", + "10", + "--alpha", + "0.2", + "--beta", + "0.01", + ) + sprt = written["sprt"] + assert (sprt["alpha"], sprt["beta"]) == (0.2, 0.01) + assert math.isclose(sprt["lower"], math.log(0.01 / 0.8)) + assert math.isclose(sprt["upper"], math.log(0.99 / 0.2)) + + class TestNormalized: """The difference in normalized elo, pinned against what the pinned fastchess works out for the same pairs.