From ecee7c1e45fd7ecc937b440b2ab09014f7621d7f Mon Sep 17 00:00:00 2001 From: Claude Date: Mon, 28 Sep 2026 21:38:46 +0000 Subject: [PATCH] feat(ci): Let strength.yml take its sequential test in normalized elo v0.6.0 gave summarise-match an sprt_model input, and the reusable workflows could not pass it until a release held it. strength.yml now takes sprt_model and hands it through batch.yml to the summary, so a caller of the workflow can state its bounds in normalized elo. The model is checked before anything is built: it is logistic or normalized, and normalized bounds lie within 100 either side of nought. The summary refuses both too, but only after the batch has been played. batch.yml's manifest recorded model=logistic whatever the test was, and now records the model the summary used. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_018ccJFsmZ4jAKYmFNgNBh9u --- .github/workflows/README.md | 6 ++++++ .github/workflows/batch.yml | 8 +++++++- .github/workflows/strength.yml | 30 ++++++++++++++++++++++++++++++ README.md | 10 +++++----- 4 files changed, 48 insertions(+), 6 deletions(-) diff --git a/.github/workflows/README.md b/.github/workflows/README.md index 5f979a2..4607155 100644 --- a/.github/workflows/README.md +++ b/.github/workflows/README.md @@ -111,6 +111,12 @@ the ceiling; asking for more is refused rather than quietly truncated. A test that wants more carries on in a second run, the way every batch did before: take the `carried` output and hand it to the next run as `prior_pairs`. +`elo0` and `elo1` are logistic elo unless `sprt_model` says `normalized`. A +test carried on in a second run should be given the same model again. Nothing +checks that, and the error rates only hold for a test judged under one model +throughout. The main +README has how long a normalized test runs at two sets of bounds. + Three outputs come back rather than one. `verdict` is `passed`, `failed` or `inconclusive`, and is empty when the run was not a sequential test. `carried` is the pair counts of the whole test. `line` is the last batch that played. diff --git a/.github/workflows/batch.yml b/.github/workflows/batch.yml index a05a8ea..9158881 100644 --- a/.github/workflows/batch.yml +++ b/.github/workflows/batch.yml @@ -103,6 +103,10 @@ on: description: The alternative the test is against. type: number default: 10 + sprt_model: + description: What elo0 and elo1 are differences in, logistic or normalized. + type: string + default: logistic cache_paths: description: What to hand actions/cache, one path a line. type: string @@ -260,6 +264,7 @@ jobs: SPRT: ${{ inputs.sprt }} ELO0: ${{ inputs.elo0 }} ELO1: ${{ inputs.elo1 }} + SPRT_MODEL: ${{ inputs.sprt_model }} PRIOR_PAIRS: ${{ inputs.prior_pairs }} BUILD: ${{ inputs.build }} MANIFEST_EXTRA: ${{ inputs.manifest_extra }} @@ -291,7 +296,7 @@ jobs: echo "startup_ms: ${STARTUP_MS}" echo "games_asked: $((PAIRS * 2))" if [ "$SPRT" = "true" ]; then - echo "sprt: judged by the summary over the pooled pairs, not by fastchess: elo0=${ELO0} elo1=${ELO1} alpha=0.05 beta=0.05 model=logistic prior_pairs=${PRIOR_PAIRS:-none}" + echo "sprt: judged by the summary over the pooled pairs, not by fastchess: elo0=${ELO0} elo1=${ELO1} alpha=0.05 beta=0.05 model=${SPRT_MODEL} prior_pairs=${PRIOR_PAIRS:-none}" else echo "sprt: off" fi @@ -370,6 +375,7 @@ jobs: sprt: ${{ inputs.sprt }} elo0: ${{ inputs.elo0 }} elo1: ${{ inputs.elo1 }} + sprt_model: ${{ inputs.sprt_model }} prior_pairs: ${{ inputs.prior_pairs }} provenance: >- The games, the results and a manifest of what each shard played are diff --git a/.github/workflows/strength.yml b/.github/workflows/strength.yml index 6026932..16be600 100644 --- a/.github/workflows/strength.yml +++ b/.github/workflows/strength.yml @@ -69,6 +69,13 @@ on: description: The alternative the test is against. type: number default: 10 + sprt_model: + description: > + What elo0 and elo1 are differences in: `logistic` elo, or + `normalized` elo. Every batch of one test is judged under the model + it started with, so a test carried on with prior_pairs keeps it. + type: string + default: logistic prior_pairs: description: The pair counts of the batches already played. Empty for the first. type: string @@ -188,6 +195,25 @@ jobs: exit 1 } + # Also before a runner builds anything. The summary refuses both of + # these as well, but only once the batch has been played. + - name: Check the model and its bounds + env: + SPRT_MODEL: ${{ inputs.sprt_model }} + ELO0: ${{ inputs.elo0 }} + ELO1: ${{ inputs.elo1 }} + run: | + case "$SPRT_MODEL" in + logistic|normalized) ;; + *) echo "::error::sprt_model is logistic or normalized, and was ${SPRT_MODEL}" + exit 1 ;; + esac + if [ "$SPRT_MODEL" = "normalized" ] \ + && ! awk -v a="$ELO0" -v b="$ELO1" 'BEGIN { exit !(a >= -100 && a <= 100 && b >= -100 && b <= 100) }'; then + echo "::error::normalized bounds are between -100 and 100, and were ${ELO0} and ${ELO1}" + exit 1 + fi + - uses: aywrite/mache/actions/resolve-ref@v0.6.0 id: candidate with: @@ -266,6 +292,7 @@ jobs: sprt: ${{ inputs.sprt }} elo0: ${{ inputs.elo0 }} elo1: ${{ inputs.elo1 }} + sprt_model: ${{ inputs.sprt_model }} cache_paths: ${{ inputs.cache_paths }} cache_key: ${{ inputs.cache_key }} manifest_extra: ${{ inputs.manifest_extra }} @@ -307,6 +334,7 @@ jobs: sprt: ${{ inputs.sprt }} elo0: ${{ inputs.elo0 }} elo1: ${{ inputs.elo1 }} + sprt_model: ${{ inputs.sprt_model }} cache_paths: ${{ inputs.cache_paths }} cache_key: ${{ inputs.cache_key }} manifest_extra: ${{ inputs.manifest_extra }} @@ -348,6 +376,7 @@ jobs: sprt: ${{ inputs.sprt }} elo0: ${{ inputs.elo0 }} elo1: ${{ inputs.elo1 }} + sprt_model: ${{ inputs.sprt_model }} cache_paths: ${{ inputs.cache_paths }} cache_key: ${{ inputs.cache_key }} manifest_extra: ${{ inputs.manifest_extra }} @@ -389,6 +418,7 @@ jobs: sprt: ${{ inputs.sprt }} elo0: ${{ inputs.elo0 }} elo1: ${{ inputs.elo1 }} + sprt_model: ${{ inputs.sprt_model }} cache_paths: ${{ inputs.cache_paths }} cache_key: ${{ inputs.cache_key }} manifest_extra: ${{ inputs.manifest_extra }} diff --git a/README.md b/README.md index 7e0c53c..9726a9e 100644 --- a/README.md +++ b/README.md @@ -130,10 +130,10 @@ which three pairs in five score one point, shifted to each true difference: Those are averages, and the spread around them is wide: about one test in ten ran to nearly twice the average or longer. At 1,000 pairs a run, the averages come to about four to seven runs for [0, 10] and about thirteen to twenty-six -for [0, 5]. The normalized model is what -`match-estimate --model normalized` and the `sprt_model` input of -`actions/summarise-match` use. `strength.yml` takes its bounds in logistic elo, -so its default of [0, 10] is not the first row above, and how long a logistic +for [0, 5]. The normalized model is what `match-estimate --model normalized` +uses, and what `strength.yml` and `actions/summarise-match` use when given +`sprt_model: normalized`. Both default to logistic elo, so `strength.yml`'s +default bounds of [0, 10] are not the first row above, and how long a logistic test runs depends on the draw rate as well as the bounds. How long a batch takes depends on the time control, the engines and the @@ -485,7 +485,7 @@ carries the model in its `sprt` object. Every batch of one test is judged under the model it started with. The pair counts carried between batches are the same under either, so nothing stops a caller changing it part way, but the error rates only hold for a test that did -not. `actions/summarise-match` takes it as `sprt_model`. +not. `strength.yml` and `actions/summarise-match` take it as `sprt_model`. The ratio under the normalized model fits, for each hypothesis, the distribution over the five pair scores that is likeliest to have produced the