diff --git a/docs/superpowers/plans/2026-11-09-friction-mini-experiment.md b/docs/superpowers/plans/2026-11-09-friction-mini-experiment.md new file mode 100644 index 00000000..b08257be --- /dev/null +++ b/docs/superpowers/plans/2026-11-09-friction-mini-experiment.md @@ -0,0 +1,56 @@ +# Friction (RTCFR+-style) on sampled mini-drawmaha: a mini experiment + +**Goal.** Does a friction term toward a moving reference make the *current* strategy of +sampled tabular MCCFR converge on mini-drawmaha, and which inner rule carries it best: +CFR+ (W = 10), mini-batch CFR+ (W = 100), or LCFR (W = 10)? + +**Done when.** Nine lockstep runs (3 rules × μ ∈ {0, 0.1, 1}) reach 1M hands per seat +in parallel, each graded at 50k / 100k / 250k / 500k / 1M hands on its current strategy +and its linear average, and the results sit in one JSON per run plus a curves figure. +μ = 0 is each rule's no-friction control. + +## The friction (Meng et al. 2023, arXiv 2308.11256, Eq. 7–8, adapted to sampling) + +RTCFR+ adds μ·(x̂ − r̂) to each player's sequence-form loss: x̂(I,a) is the player's own +reach of sequence (I,a), r̂ the reference's. Its counterfactual value at (I,a) is the +payoff value minus μ·(x̂(I,a) − r̂(I,a)), plus the children's (already transformed) values. + +Under external sampling the payoff part of a banked regret is, in expectation, weighted +by the opponent-and-chance reach of I. We apply the friction undivided on each visit +(choice B), so its expected weight carries the same reach: friction and payoff keep the +same ratio at every infoset. The paper's unweighted form would need a 1/reach factor and +explode at rare rows. γ (the paper's 1e-10 mix toward uniform) is numerically inert at +this scale and is left out. Every R hands per seat the reference is reset to the current +strategy; regret totals are kept (the paper: resetting them stops convergence). + +## Pieces + +1. `solver/friction.py`: `Friction(mu, table, reference)`; `Friction.new(table, mu)` + (reference = current strategy, uniform at the start); `of(key)`; `reset()`. + Vectorised current strategy over a packed table. +2. `mccfr.traverse`: optional `friction`, carrying the traverser's own reach and the + reference's reach down its own actions; utilities at its spots minus μ(x − r). + `friction=None` is today's walk, bit for bit. +3. `lockstep`: `LockstepSolve.friction` handed to every walk; `save_lockstep` and + `ParallelLockstep` refuse a run with friction (not checkpointed, not parallelised). +4. `scripts/minidrawmaha/friction_experiment.py`: one run; resets, grades, JSON lines. +5. Tests (`tests/minidrawmaha/test_friction.py`), written first: + - friction with μ = 0 banks today's regrets bit for bit (Leduc, lockstep W = 4); + - a reference equal to the current strategy along with equal reaches cancels: at t = 1 + (all σ uniform, reference uniform) any μ banks today's regrets bit for bit; + - the hand-computed adjustment at one spot; + - the vectorised reset equals `RegretMatcher.strategy` row by row (packed slice); + - save and parallel refuse friction by name; + - Leduc gate (slow): friction + CFR+ drives the current strategy's exploitability + below no-friction CFR+'s current strategy. + +## As run (9 Nov 2026) + +The machine could not hold nine 1M-hand runs at once (12 concurrent runs land on the +efficiency cores at about 35 hands per seat per second), so the grid ran in waves: +μ ∈ {0, 10} for all three rules to 1M; μ = 0.1 stopped at 50k (indistinguishable +from μ = 0); μ ∈ {1, 100} for CFR+ and LCFR at W = 10 to 500k. The record is +`figures/rung3/friction_mini.jsonl`. Outcome: μ = 10 ends 2–4% below its control at +1M in all three pairs and is 5–11% worse at 50k; μ = 100 is 20–56% worse; the +current strategy beats the linear average at every mark of every arm; LCFR beats +CFR+ by 30%; W = 100 tracks W = 10 within 1.6%. diff --git a/figures/AGENTS.md b/figures/AGENTS.md index dbfa6680..79944fa6 100644 --- a/figures/AGENTS.md +++ b/figures/AGENTS.md @@ -99,3 +99,11 @@ by `run_deep_cfr.py --rule dcfr` and `--rule cfr+`, its marks graded by `grades.json`. `rung4/rule_sign_flips.json` is the measurement of why they lost, written by `scripts/deepcfr/rule_sign_flips.py` from the same runs' advantage memories. + +`rung3/friction_mini.jsonl` is the friction mini experiment's graded record: +every JSON line `scripts/minidrawmaha/friction_experiment.py` appended, one per +grading mark per arm (rule, hands per update W, friction strength μ, current +and linear-average exploitability), concatenated from the arms' output files. +Arms: CFR+ W = 10, CFR+ W = 100 and LCFR W = 10 at μ = 0 and 10 to 1M hands +per seat; at μ = 0.1 to 50k only; CFR+ and LCFR W = 10 at μ = 1 and 100 to +500k. One seed (0) per arm. diff --git a/figures/rung3/friction_mini.jsonl b/figures/rung3/friction_mini.jsonl new file mode 100644 index 00000000..ad7a06f0 --- /dev/null +++ b/figures/rung3/friction_mini.jsonl @@ -0,0 +1,49 @@ +{"rule": "cfr+", "workers": 10, "mu": 0.1, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 3.5562435935672143, "average": 3.731157099798681, "seconds": 594.4} +{"rule": "cfr+", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 3.5517268959295873, "average": 3.7356591304923175, "seconds": 576.0} +{"rule": "cfr+", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 100000, "current": 3.2509838120613503, "average": 3.417605395721135, "seconds": 1130.8} +{"rule": "cfr+", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 25000, "hands_per_seat": 250000, "current": 2.5997235419862417, "average": 2.9387484921647653, "seconds": 2253.6} +{"rule": "cfr+", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 50000, "hands_per_seat": 500000, "current": 1.8008132370632826, "average": 2.333622478692331, "seconds": 3948.0} +{"rule": "cfr+", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 100000, "hands_per_seat": 1000000, "current": 1.0736841309465013, "average": 1.5643887958807077, "seconds": 6909.9} +{"rule": "cfr+", "workers": 100, "mu": 0.1, "reset_hands": 10000, "seed": 0, "iteration": 500, "hands_per_seat": 50000, "current": 3.547235342566566, "average": 3.7194481621097455, "seconds": 618.8} +{"rule": "cfr+", "workers": 100, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 500, "hands_per_seat": 50000, "current": 3.531311741835638, "average": 3.6944582700377397, "seconds": 605.8} +{"rule": "cfr+", "workers": 100, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 1000, "hands_per_seat": 100000, "current": 3.2620827552899994, "average": 3.4310462865215943, "seconds": 1224.0} +{"rule": "cfr+", "workers": 100, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 2500, "hands_per_seat": 250000, "current": 2.61170158222152, "average": 2.9414771241850843, "seconds": 2487.5} +{"rule": "cfr+", "workers": 100, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 500000, "current": 1.777823102998135, "average": 2.3347398527773153, "seconds": 4421.1} +{"rule": "cfr+", "workers": 100, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 1000000, "current": 1.0911306210839766, "average": 1.5658667023559403, "seconds": 7729.8} +{"rule": "lcfr", "workers": 10, "mu": 0.1, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 3.1816373922487835, "average": 3.374124363882292, "seconds": 605.3} +{"rule": "lcfr", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 3.1867334340455207, "average": 3.4041814167616087, "seconds": 555.0} +{"rule": "lcfr", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 100000, "current": 2.894129293353006, "average": 3.048347266505258, "seconds": 1098.3} +{"rule": "lcfr", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 25000, "hands_per_seat": 250000, "current": 2.22591541247425, "average": 2.5380436308105256, "seconds": 2189.5} +{"rule": "lcfr", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 50000, "hands_per_seat": 500000, "current": 1.386015778439032, "average": 1.8762884428553426, "seconds": 3803.6} +{"rule": "lcfr", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 100000, "hands_per_seat": 1000000, "current": 0.767706163376485, "average": 1.1307747045097751, "seconds": 6629.3} +{"rule": "cfr+", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 3.7207998389882166, "average": 3.9205683871690127, "seconds": 591.9} +{"rule": "cfr+", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 100000, "current": 3.3316192002878555, "average": 3.5577799505608265, "seconds": 1126.9} +{"rule": "cfr+", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 25000, "hands_per_seat": 250000, "current": 2.6303120109575158, "average": 3.002654376742573, "seconds": 2309.0} +{"rule": "cfr+", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 50000, "hands_per_seat": 500000, "current": 1.7795442998298157, "average": 2.348595071065766, "seconds": 4086.8} +{"rule": "cfr+", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 100000, "hands_per_seat": 1000000, "current": 1.043179960227394, "average": 1.5518714019797173, "seconds": 7044.5} +{"rule": "cfr+", "workers": 100, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 500, "hands_per_seat": 50000, "current": 3.705894622009329, "average": 3.9170308452958436, "seconds": 610.3} +{"rule": "cfr+", "workers": 100, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 1000, "hands_per_seat": 100000, "current": 3.3195559232993808, "average": 3.5614458260723794, "seconds": 1173.3} +{"rule": "cfr+", "workers": 100, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 2500, "hands_per_seat": 250000, "current": 2.6109102211910113, "average": 2.994395712916459, "seconds": 2471.8} +{"rule": "cfr+", "workers": 100, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 500000, "current": 1.7855622367234023, "average": 2.350620118255279, "seconds": 4466.7} +{"rule": "cfr+", "workers": 100, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 1000000, "current": 1.0507719008332268, "average": 1.5545607674948498, "seconds": 7895.4} +{"rule": "lcfr", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 3.552082275982865, "average": 3.7637730030516785, "seconds": 591.3} +{"rule": "lcfr", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 100000, "current": 3.1253773110250855, "average": 3.3739519637026376, "seconds": 1132.6} +{"rule": "lcfr", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 25000, "hands_per_seat": 250000, "current": 2.361427008742529, "average": 2.7362553915378487, "seconds": 2298.9} +{"rule": "lcfr", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 50000, "hands_per_seat": 500000, "current": 1.5121865175932423, "average": 2.0468756266790447, "seconds": 4053.2} +{"rule": "lcfr", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 100000, "hands_per_seat": 1000000, "current": 0.7527588977522754, "average": 1.2125338427647108, "seconds": 6965.3} +{"rule": "cfr+", "workers": 10, "mu": 1.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 3.5231938498109763, "average": 3.708012881728954, "seconds": 603.9} +{"rule": "cfr+", "workers": 10, "mu": 1.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 100000, "current": 3.264553079307725, "average": 3.4141293108046815, "seconds": 1090.4} +{"rule": "cfr+", "workers": 10, "mu": 1.0, "reset_hands": 10000, "seed": 0, "iteration": 25000, "hands_per_seat": 250000, "current": 2.6076737447371947, "average": 2.9508023108123815, "seconds": 2101.5} +{"rule": "cfr+", "workers": 10, "mu": 1.0, "reset_hands": 10000, "seed": 0, "iteration": 50000, "hands_per_seat": 500000, "current": 1.7892240350990598, "average": 2.328847033417266, "seconds": 3620.6} +{"rule": "cfr+", "workers": 10, "mu": 100.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 4.256367092517985, "average": 4.347168468874717, "seconds": 590.1} +{"rule": "cfr+", "workers": 10, "mu": 100.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 100000, "current": 3.9722703282887686, "average": 4.135957583218573, "seconds": 1057.9} +{"rule": "cfr+", "workers": 10, "mu": 100.0, "reset_hands": 10000, "seed": 0, "iteration": 25000, "hands_per_seat": 250000, "current": 3.1379024016058743, "average": 3.5781596647132923, "seconds": 2072.6} +{"rule": "cfr+", "workers": 10, "mu": 100.0, "reset_hands": 10000, "seed": 0, "iteration": 50000, "hands_per_seat": 500000, "current": 2.2050062895255187, "average": 2.842050354505566, "seconds": 3594.5} +{"rule": "lcfr", "workers": 10, "mu": 1.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 3.2001286739426402, "average": 3.417872681156649, "seconds": 608.2} +{"rule": "lcfr", "workers": 10, "mu": 1.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 100000, "current": 2.8856417925374713, "average": 3.0570814178629706, "seconds": 1079.7} +{"rule": "lcfr", "workers": 10, "mu": 1.0, "reset_hands": 10000, "seed": 0, "iteration": 25000, "hands_per_seat": 250000, "current": 2.2387650009599414, "average": 2.5537640224399833, "seconds": 2065.7} +{"rule": "lcfr", "workers": 10, "mu": 1.0, "reset_hands": 10000, "seed": 0, "iteration": 50000, "hands_per_seat": 500000, "current": 1.389492853859736, "average": 1.8915239480846644, "seconds": 3523.9} +{"rule": "lcfr", "workers": 10, "mu": 100.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 4.191457154890394, "average": 4.261162260314845, "seconds": 594.5} +{"rule": "lcfr", "workers": 10, "mu": 100.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 100000, "current": 3.9202451147143664, "average": 4.055580788894654, "seconds": 1076.7} +{"rule": "lcfr", "workers": 10, "mu": 100.0, "reset_hands": 10000, "seed": 0, "iteration": 25000, "hands_per_seat": 250000, "current": 3.1501192015758406, "average": 3.552543871858124, "seconds": 2082.3} +{"rule": "lcfr", "workers": 10, "mu": 100.0, "reset_hands": 10000, "seed": 0, "iteration": 50000, "hands_per_seat": 500000, "current": 2.157896653195825, "average": 2.803019908303629, "seconds": 3609.6} diff --git a/scripts/minidrawmaha/friction_experiment.py b/scripts/minidrawmaha/friction_experiment.py new file mode 100644 index 00000000..36e71474 --- /dev/null +++ b/scripts/minidrawmaha/friction_experiment.py @@ -0,0 +1,130 @@ +"""One arm of the friction mini experiment: lockstep MCCFR, with or without RTCFR+-style friction. + + uv run python scripts/minidrawmaha/friction_experiment.py --rule cfr+ --workers 10 --mu 0.1 \ + --out runs/friction-mini/cfrplus-w10-mu0.1.jsonl + +Trains one run in this process (`train_lockstep`; friction is not parallelised) up +to the last of `--marks` hands per seat, where it stops. Every `--reset-hands` hands per seat it moves the friction's +reference to the current strategy (RTCFR+'s r ← x̂; see `solver/friction.py`). At each +mark it grades BOTH the current strategy (what the friction is meant to make converge) +and the linear average (what plain CFR ships) with the exact meter, and appends one +JSON line: hands per seat, iteration, the two exploitabilities in chips per hand, and +the seconds so far. `--mu 0` trains plain MCCFR, the arm's no-friction control. + +Costs: about 0.3 GB for the table and as much again for the reference, plus the +meter's 1.5 GB; about 200 hands per seat per second at W = 10 on one core, and two +grades of about two minutes each per mark. No checkpoint is written: a friction +run's reference is not saved, so an interrupted run is rerun. +""" + +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + +import numpy as np + +from drawmaha_solver.minidrawmaha.exploitability import exploitability, minidrawmaha_game +from drawmaha_solver.minidrawmaha.solver.friction import Friction +from drawmaha_solver.minidrawmaha.solver.infoset_table import ( + average_strategy, + current_strategy, +) +from drawmaha_solver.minidrawmaha.solver.lockstep import new_lockstep, train_lockstep + +DEFAULT_MARKS = (50_000, 100_000, 250_000, 500_000, 1_000_000) + +def main() -> None: + """Train one arm to its last mark, resetting the reference and grading on the way.""" + args = _parse() + reset_every = _iterations(args.reset_hands, args.workers, "--reset-hands") + marks = sorted({_iterations(m, args.workers, "--marks") for m in args.marks}) + solve = new_lockstep(args.seed, workers=args.workers, rule=args.rule) + if args.mu > 0: + solve.friction = Friction.new(solve.table, args.mu) + game = minidrawmaha_game() + args.out.parent.mkdir(parents=True, exist_ok=True) + started = time.perf_counter() + + for t in _events(reset_every, marks): + train_lockstep(solve, t - solve.iteration) + if solve.friction is not None and t % reset_every == 0: + solve.friction.reset() + if t in marks: + line = { + "rule": args.rule, + "workers": args.workers, + "mu": args.mu, + "reset_hands": args.reset_hands, + "seed": args.seed, + "iteration": t, + "hands_per_seat": t * args.workers, + "current": exploitability(current_strategy(solve.table), game=game), + "average": exploitability(average_strategy(solve.table), game=game), + "seconds": round(time.perf_counter() - started, 1), + } + with args.out.open("a") as file: + file.write(json.dumps(line) + "\n") + print(json.dumps(line), flush=True) + if args.save_table is not None: + _save_table(solve, args.save_table) + +def _save_table(solve, path: Path) -> None: + """The finished run's regrets, strategy sums and reference, for the row-level analysis. + + In: the trained run and a `.npz` path. Out: nothing; writes the packed table's + `cumulative_regret` and `strategy_sum` (one entry per column), its row widths + and, under friction, the last reference. Purpose: which rows were never + sampled, which sit uniform, and how far play sits from the reference are read + from these arrays after the run; this is not a resumable checkpoint. + """ + arrays = { + "cumulative_regret": solve.table.cumulative_regret, + "strategy_sum": solve.table.strategy_sum, + "widths": solve.table.widths(), + } + if solve.friction is not None: + arrays["reference"] = solve.friction.reference + path.parent.mkdir(parents=True, exist_ok=True) + np.savez(path, **arrays) + +def _events(reset_every: int, marks: list[int]) -> list[int]: + """Every iteration at which training must pause: each reset and each mark, in order. + + In: the reset cadence and the mark iterations. Out: the sorted union up to the + last mark. Purpose: `main` trains from one pause to the next, so a reset lands + exactly on its iteration and a grade sees the table as of its mark. + """ + last = marks[-1] + return sorted(set(range(reset_every, last + 1, reset_every)) | set(marks)) + +def _iterations(hands: int, workers: int, flag: str) -> int: + """Hands per seat as lockstep iterations (W hands per seat each); refused unless exact. + + Purpose: arms at different W are compared at equal hands, so a mark or a + reset that does not land on a whole iteration would compare unequal amounts. + """ + if hands <= 0 or hands % workers: + raise SystemExit(f"{flag}: {hands:,} hands is not a positive multiple of W = {workers}") + return hands // workers + +def _parse() -> argparse.Namespace: + """The command line: rule, W, μ, reset cadence, budget, seed and output file.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--rule", choices=["lcfr", "cfr+"], required=True) + parser.add_argument("--workers", type=int, default=10, help="W: hands per seat per iteration") + parser.add_argument("--mu", type=float, default=0.0, help="friction strength in chips; 0 = none") + parser.add_argument("--reset-hands", type=int, default=10_000, + help="hands per seat between reference resets") + parser.add_argument("--marks", type=int, nargs="+", default=list(DEFAULT_MARKS), + help="hands per seat at which to grade") + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--save-table", type=Path, default=None, + help="write the final table's arrays here (.npz) for row-level analysis") + return parser.parse_args() + +if __name__ == "__main__": + main() diff --git a/src/drawmaha_solver/minidrawmaha/solver/friction.py b/src/drawmaha_solver/minidrawmaha/solver/friction.py new file mode 100644 index 00000000..224670a6 --- /dev/null +++ b/src/drawmaha_solver/minidrawmaha/solver/friction.py @@ -0,0 +1,119 @@ +"""Friction toward a reference strategy: RTCFR+'s reward transformation, under sampling. + +Why it exists. A CFR run's CURRENT strategy need not converge (in the worst case +it circles the equilibrium); only its average is guaranteed to. RTCFR+ (Meng et al. 2023, arXiv 2308.11256, Eq. 7–8) changes +the game the regrets are learning so that the current strategy itself settles: each +player's loss gains μ·(x̂ − r̂), where x̂(I, a) is the player's own reach of the +sequence "arrive at I, then play a" and r̂(I, a) is the same for a REFERENCE +strategy. Playing a sequence more than the reference does is taxed; playing it less +earns a bonus. That pull makes the taxed game's equilibrium unique and the current +strategy spiral into it. The equilibrium is biased toward the reference, so every +so often the reference is reset to the current strategy (`Friction.reset`), and the +biased equilibria walk toward the real one. Regret totals are kept across a reset, +as in the paper's algorithm. + +What the walk does with it. At each of the traverser's spots `mccfr.traverse` +subtracts `friction_adjustment(...)` from the action values before the regret is +formed, and passes each child its own reach σ(a)·x and the reference's +reference(a)·r. Values returned up the tree already carry the deeper spots' +friction, as the paper's counterfactual-value recursion does. + +One departure from the paper, forced by sampling. In the paper the friction enters +every infoset's value with full weight, every iteration. External sampling banks a +row only when it is visited, and the payoff part of what it banks is weighted, in +expectation, by the chance-and-opponent reach π₋ᵢ(I) of the row; in the paper the +payoff is weighted the same way but the friction is not. The friction here is added +undivided on each visit, so in expectation it is scaled by π₋ᵢ(I) too: relative to +the paper the tax at row I is down-weighted by π₋ᵢ(I), i.e. μ acts as an effective +μ·π₋ᵢ(I), and the tax-to-payoff ratio is the same at every row instead of growing +as 1/π₋ᵢ at rare rows. Dividing by the visit probability would recover the paper's +weighting and hand rare rows enormous, noisy shoves. The paper's γ (a 1e-10 mix toward uniform) +is numerically inert at this scale and is left out. +""" + +from __future__ import annotations + +from collections.abc import Hashable +from dataclasses import dataclass, field + +import numpy as np + +from drawmaha_solver.minidrawmaha.solver.packed_table import PackedTable + +def friction_adjustment( + sigma: np.ndarray, + reference: np.ndarray, + *, + own_reach: float, + reference_reach: float, + mu: float, +) -> np.ndarray: + """μ·(x̂ − r̂) at one spot: what to subtract from each action's value there. + + In: `sigma` and `reference`, the spot's current and reference strategies (one + probability per legal action); `own_reach` and `reference_reach`, the chance + that the traverser's own play (resp. the reference's) gets to this spot, its + opponent and the deck not counted. Out: one tax per action, in chips. + Positive where play out-reaches the reference, negative where it lags. + """ + return mu * (own_reach * sigma - reference_reach * reference) + +@dataclass(slots=True) +class Friction: + """The tax's strength μ (chips) and the reference strategy it pulls toward. + + `table` is the run's table; `reference` holds one probability per column, a + flat array aligned with a packed table's `cumulative_regret`, or a dict of + arrays for a dict table (Leduc, tests). Built by `new`, never by hand. + """ + + mu: float + table: object + reference: np.ndarray | dict[Hashable, np.ndarray] = field(repr=False) + + @classmethod + def new(cls, table, mu: float) -> Friction: + """Friction of strength `mu` whose reference is the table's current play. + + On a fresh table that is uniform everywhere, which is RTCFR+'s starting + reference. A negative or non-finite μ is refused by name. + """ + if not np.isfinite(mu) or mu < 0: + raise ValueError(f"friction mu must be a finite number ≥ 0, got {mu!r}") + friction = cls(mu=float(mu), table=table, reference={}) + friction.reset() + return friction + + def of(self, key: Hashable) -> np.ndarray: + """The reference strategy at `key`, one probability per legal action.""" + if isinstance(self.table, PackedTable): + _, start, width = self.table.slot_of(key) + return self.reference[start : start + width] + return self.reference[key] + + def reset(self) -> None: + """Move the reference to what the table plays now (RTCFR+'s r ← x̂). + + Regrets are left alone. A packed table is copied in one vectorised pass; + a dict table ledger by ledger. + """ + if isinstance(self.table, PackedTable): + self.reference = packed_current_strategy(self.table) + else: + self.reference = {key: ledger.strategy() for key, ledger in self.table.items()} + +def packed_current_strategy(table: PackedTable) -> np.ndarray: + """Every row's current strategy at once, flat and column-aligned with the table. + + In: a packed table. Out: an array the length of `cumulative_regret` holding, + per row, the positive regrets normalised, or uniform where none is positive — + the same numbers `RegretMatcher.strategy` gives row by row, without building + 6.2 million ledgers. + """ + widths = table.widths() + starts = np.concatenate(([0], np.cumsum(widths)[:-1])) + positive = np.maximum(table.cumulative_regret, 0.0) + totals = np.repeat(np.add.reduceat(positive, starts), widths) + sizes = np.repeat(widths, widths).astype(float) + with np.errstate(invalid="ignore", divide="ignore"): + return np.where(totals > 0.0, positive / totals, 1.0 / sizes) diff --git a/src/drawmaha_solver/minidrawmaha/solver/lockstep.py b/src/drawmaha_solver/minidrawmaha/solver/lockstep.py index fce3ef33..220c1131 100644 --- a/src/drawmaha_solver/minidrawmaha/solver/lockstep.py +++ b/src/drawmaha_solver/minidrawmaha/solver/lockstep.py @@ -49,6 +49,7 @@ import numpy as np from drawmaha_solver.minidrawmaha.rules.game import random_deal +from drawmaha_solver.minidrawmaha.solver.friction import Friction from drawmaha_solver.minidrawmaha.solver.mccfr import ( RootSampler, Table, @@ -99,6 +100,9 @@ class LockstepSolve: iteration: int = 0 rule: RegretRule = RegretRule.VANILLA averages: tuple[Average, ...] = () + # RTCFR+-style friction toward a reference (`friction.py`); None is plain MCCFR. + # Set by the caller after `new_lockstep`; never checkpointed or parallelised. + friction: Friction | None = None def new_lockstep( seed: int, @@ -138,7 +142,16 @@ def train_lockstep(solve: LockstepSolve, iterations: int) -> LockstepSolve: t = solve.iteration + 1 for seat in SEATS: recorded = [ - walk(solve.table, solve.deal, solve.seed, t, seat, worker, solve.averages) + walk( + solve.table, + solve.deal, + solve.seed, + t, + seat, + worker, + solve.averages, + friction=solve.friction, + ) for worker in range(solve.workers) ] apply(solve.table, recorded, solve.rule, t) @@ -169,8 +182,14 @@ def walk( seat: int, worker: int, averages: tuple[Average, ...], + *, + friction: Friction | None = None, ) -> Recorded: - """Worker `worker`'s walk for `seat` at iteration t, reading σₜ and writing nothing.""" + """Worker `worker`'s walk for `seat` at iteration t, reading σₜ and writing nothing. + + `friction` taxes the traverser's values toward its reference (`friction.py`); + it reads the reference only, so the walk still writes nothing. + """ rng = stream(seed, t, seat, worker) recorder = _Recorder(table, len(averages)) traverse( @@ -181,6 +200,7 @@ def walk( t, bank=recorder.bank, column_weights=extra_weights(averages, t), + friction=friction, ) return recorder.recorded() @@ -371,6 +391,10 @@ def save_lockstep(solve: LockstepSolve, path: Path) -> None: """ if not solve.table: raise ValueError("the table is empty; there is nothing to checkpoint") + if solve.friction is not None: + # The reference and μ are not in the file, so a resume would silently + # train a different game. + raise ValueError("a run with friction is not checkpointed: its reference is not saved") path = Path(path) regret, strategy_sum, extra_sums, stamps = _flat_slots(solve.table) temporary = path.with_name(path.name + ".tmp") diff --git a/src/drawmaha_solver/minidrawmaha/solver/mccfr.py b/src/drawmaha_solver/minidrawmaha/solver/mccfr.py index 08875af9..1f3b7a11 100644 --- a/src/drawmaha_solver/minidrawmaha/solver/mccfr.py +++ b/src/drawmaha_solver/minidrawmaha/solver/mccfr.py @@ -98,6 +98,7 @@ import numpy as np from drawmaha_solver.minidrawmaha.rules.game import random_deal +from drawmaha_solver.minidrawmaha.solver.friction import Friction, friction_adjustment from drawmaha_solver.minidrawmaha.solver.infoset_table import new_infoset_table from drawmaha_solver.minidrawmaha.solver.packed_table import PackedTable from drawmaha_solver.minidrawmaha.solver.regret_rules import ( @@ -281,6 +282,9 @@ def traverse( *, bank: BankRegret = bank_vanilla, column_weights: np.ndarray | None = None, + friction: Friction | None = None, + own_reach: float = 1.0, + reference_reach: float = 1.0, ) -> float: """Chips to `traverser` from `state` along one sampled path, banking on the way up. @@ -297,6 +301,11 @@ def traverse( than through `RegretMatcher.update`, which would re-derive σ and re-check its inputs at every visit; under vanilla it does the same arithmetic, and `test_mccfr.py` holds the two to identical bytes. + + `friction` (see `friction.py`) taxes the traverser's action values by how far + its own play out-reaches a reference; `own_reach` and `reference_reach` are + the chances that the traverser's own play, and the reference's, lead here. + None is today's walk, bit for bit, and the reaches are then never read. """ if state.is_terminal(): return state.returns()[traverser] @@ -315,6 +324,9 @@ def traverse( t, bank=bank, column_weights=column_weights, + friction=friction, + own_reach=own_reach, + reference_reach=reference_reach, ) ledger = table[state.infoset()] @@ -326,20 +338,46 @@ def traverse( # was read before the subtree ran and is still this ledger's strategy: # perfect recall keeps the subtree from reaching this spot again. if state.current_player == traverser: - utilities = np.array( - [ - traverse( - state.apply(action), - table, - traverser, - rng, - t, - bank=bank, - column_weights=column_weights, - ) - for action in actions - ] - ) + if friction is None: + utilities = np.array( + [ + traverse( + state.apply(action), + table, + traverser, + rng, + t, + bank=bank, + column_weights=column_weights, + ) + for action in actions + ] + ) + else: + reference = friction.of(state.infoset()) + utilities = np.array( + [ + traverse( + state.apply(action), + table, + traverser, + rng, + t, + bank=bank, + column_weights=column_weights, + friction=friction, + own_reach=own_reach * sigma[k], + reference_reach=reference_reach * reference[k], + ) + for k, action in enumerate(actions) + ] + ) - friction_adjustment( + sigma, + reference, + own_reach=own_reach, + reference_reach=reference_reach, + mu=friction.mu, + ) value = sigma @ utilities bank(ledger, utilities - value, t) return float(value) @@ -359,6 +397,9 @@ def traverse( t, bank=bank, column_weights=column_weights, + friction=friction, + own_reach=own_reach, + reference_reach=reference_reach, ) def _pick(rng: np.random.Generator, probabilities: Sequence[float]) -> int: diff --git a/src/drawmaha_solver/minidrawmaha/solver/parallel.py b/src/drawmaha_solver/minidrawmaha/solver/parallel.py index 338807f1..c7fa63a0 100644 --- a/src/drawmaha_solver/minidrawmaha/solver/parallel.py +++ b/src/drawmaha_solver/minidrawmaha/solver/parallel.py @@ -218,6 +218,10 @@ class ParallelLockstep: def __init__(self, solve: LockstepSolve, *, keys: Sequence[InfoSet] | None = None): if not isinstance(solve.table, PackedTable): raise TypeError("a parallel run needs a PackedTable") + if solve.friction is not None: + # The workers would need the reference in shared memory, and a reset + # between phases; the friction experiment runs in one process instead. + raise ValueError("a run with friction is not parallelised; train it with train_lockstep") self.solve = solve self._keys = None if keys is None else tuple(keys) # The caller's table, set aside while the shared one stands in for it. diff --git a/tests/minidrawmaha/test_friction.py b/tests/minidrawmaha/test_friction.py new file mode 100644 index 00000000..0f011dc3 --- /dev/null +++ b/tests/minidrawmaha/test_friction.py @@ -0,0 +1,152 @@ +"""Friction toward a reference (RTCFR+-style) in the sampled walk. + +* **Off is today.** `friction=None` and friction at μ = 0 bank exactly today's regrets. +* **A reference equal to play cancels.** At t = 1 every σ is uniform; a uniform + reference has the same reach, so friction at any μ changes nothing. +* **One spot, by hand.** The adjustment is μ·(own reach·σ − reference reach·reference). +* **The reset** copies every row's current strategy, the same numbers + `RegretMatcher.strategy` gives. +* **Refusals.** A friction run is neither checkpointed nor parallelised. +* **Leduc gate (slow).** Friction + CFR+ makes the current strategy converge where + plain CFR+'s current strategy does not. +""" + +from __future__ import annotations + +import os + +import numpy as np +import pytest + +from drawmaha_solver.leduc.exploitability import exploitability +from drawmaha_solver.leduc.game import DEALS, LeducState +from drawmaha_solver.leduc.infoset_table import new_infoset_table as new_leduc_table +from drawmaha_solver.minidrawmaha.rules.enumeration import all_infosets +from drawmaha_solver.minidrawmaha.solver.friction import Friction, friction_adjustment +from drawmaha_solver.minidrawmaha.solver.lockstep import ( + new_lockstep, + save_lockstep, + train_lockstep, +) +from drawmaha_solver.minidrawmaha.solver.packed_table import PackedTable +from drawmaha_solver.minidrawmaha.solver.parallel import ParallelLockstep +from drawmaha_solver.regret_matching import RegretMatcher + +def leduc_deal(rng: np.random.Generator) -> LeducState: + return LeducState(cards=DEALS[rng.integers(len(DEALS))]) + +def fresh_leduc() -> dict: + return {key: RegretMatcher(ledger.n_actions) for key, ledger in new_leduc_table().items()} + +def regrets(table: dict) -> bytes: + return b"".join(table[key].cumulative_regret.tobytes() for key in table) + +def leduc_run(rule: str, workers: int, *, mu: float | None, iterations: int) -> dict: + table = fresh_leduc() + solve = new_lockstep(7, workers=workers, table=table, deal=leduc_deal, rule=rule) + if mu is not None: + solve.friction = Friction.new(table, mu) + train_lockstep(solve, iterations) + return table + +@pytest.mark.parametrize("rule", ["lcfr", "cfr+"]) +def test_friction_at_mu_zero_banks_todays_regrets(rule): + plain = leduc_run(rule, 4, mu=None, iterations=30) + zero = leduc_run(rule, 4, mu=0.0, iterations=30) + assert regrets(plain) == regrets(zero) + +def test_a_uniform_reference_cancels_at_the_first_iteration(): + plain = leduc_run("lcfr", 4, mu=None, iterations=1) + pulled = leduc_run("lcfr", 4, mu=5.0, iterations=1) + assert regrets(plain) == regrets(pulled) + +def test_friction_moves_regrets_once_play_leaves_the_reference(): + plain = leduc_run("lcfr", 4, mu=None, iterations=20) + pulled = leduc_run("lcfr", 4, mu=5.0, iterations=20) + assert regrets(plain) != regrets(pulled) + +def test_strong_friction_toward_a_fixed_uniform_reference_keeps_play_nearer_uniform(): + # Pins the sign and the reach plumbing end to end: with the reference held at + # uniform (no resets) a large μ must leave the current strategy closer to + # uniform than plain play does. A flipped sign or a dropped reach would push + # play away from the reference, or not at all. + def distance_to_uniform(table: dict) -> float: + return sum(np.abs(ledger.strategy() - 1.0 / ledger.n_actions).sum() for ledger in table.values()) + + plain = leduc_run("lcfr", 8, mu=None, iterations=300) + pulled = leduc_run("lcfr", 8, mu=50.0, iterations=300) + assert distance_to_uniform(pulled) < 0.8 * distance_to_uniform(plain) + +@pytest.mark.parametrize("mu", [-1.0, float("nan"), float("inf")]) +def test_a_negative_or_non_finite_mu_is_refused(mu): + with pytest.raises(ValueError, match="mu"): + Friction.new(fresh_leduc(), mu) + +def test_the_adjustment_at_one_spot_by_hand(): + sigma = np.array([0.5, 0.25, 0.25]) + reference = np.array([0.2, 0.2, 0.6]) + # own reach 0.4, reference reach 0.5, μ = 2: + # x = 0.4σ = (0.2, 0.1, 0.1); r = 0.5·ref = (0.1, 0.1, 0.3); μ(x − r) = (0.2, 0, −0.4) + got = friction_adjustment(sigma, reference, own_reach=0.4, reference_reach=0.5, mu=2.0) + np.testing.assert_allclose(got, [0.2, 0.0, -0.4], rtol=0, atol=1e-15) + +def test_the_reset_copies_every_rows_current_strategy(): + keys = list(all_infosets())[:5000] + table = PackedTable.listed(keys) + rng = np.random.default_rng(3) + table.cumulative_regret[:] = rng.normal(size=table.cumulative_regret.size) + # some rows all non-positive, so the uniform fallback is exercised + table.cumulative_regret[: 40] = -1.0 + friction = Friction.new(table, 1.0) + # Equal to the last bit on most rows; `np.add.reduceat` sums a 3- or 4-wide row + # in a different order than `ndarray.sum`, which moves ~2% of rows by one ulp. + # A reference only sets where the pull points, so one ulp is immaterial. + # every 37th row, plus every row touching the all-negative columns (the uniform branch) + for key in sorted(set(keys[::37]) | set(keys[:20]), key=keys.index): + np.testing.assert_allclose(friction.of(key), table[key].strategy(), rtol=0, atol=1e-15) + +def test_a_dict_reset_tracks_play(): + table = fresh_leduc() + friction = Friction.new(table, 1.0) + key = next(iter(table)) + table[key].cumulative_regret[:] = np.arange(table[key].n_actions, dtype=float) + friction.reset() + assert friction.of(key).tobytes() == table[key].strategy().tobytes() + +def test_a_friction_run_is_not_checkpointed(tmp_path): + table = fresh_leduc() + solve = new_lockstep(1, workers=2, table=table, deal=leduc_deal, rule="lcfr") + solve.friction = Friction.new(table, 1.0) + with pytest.raises(ValueError, match="friction"): + save_lockstep(solve, tmp_path / "solve.npz") + +def test_a_friction_run_is_not_parallelised(): + table = PackedTable.listed(list(all_infosets())[:100]) + solve = new_lockstep(1, workers=2, table=table, rule="lcfr") + solve.friction = Friction.new(table, 1.0) + with pytest.raises(ValueError, match="friction"): + ParallelLockstep(solve) + +@pytest.mark.skipif( + not os.environ.get("MINIDRAWMAHA_FULL_CALIBRATION"), + reason="Leduc friction gate: ~minutes; set MINIDRAWMAHA_FULL_CALIBRATION=1", +) +def test_leduc_friction_makes_cfr_plus_current_strategy_converge(): + def current_exploitability(table: dict) -> float: + return exploitability({key: ledger.strategy() for key, ledger in table.items()}) + + plain = fresh_leduc() + solve = new_lockstep(0, workers=32, table=plain, deal=leduc_deal, rule="cfr+") + train_lockstep(solve, 3000) + + # μ = 0.05 with resets every 100 iterations: Leduc payoffs are a few chips, so + # this tax is small against them, so the bias toward each reference stays small + # while the pull still damps CFR+'s overshoot. Seed 0; + # the strict < is the claim itself: friction's current strategy beats plain's. + pulled = fresh_leduc() + solve = new_lockstep(0, workers=32, table=pulled, deal=leduc_deal, rule="cfr+") + solve.friction = Friction.new(pulled, 0.05) + for _ in range(30): + train_lockstep(solve, 100) + solve.friction.reset() + assert current_exploitability(pulled) < current_exploitability(plain)