From 168bad50bf3642115c469eb87cc9b44c5b14c377 Mon Sep 17 00:00:00 2001 From: Alec Ondo Date: Mon, 9 Nov 2026 03:44:14 -0500 Subject: [PATCH 1/3] =?UTF-8?q?feat(rung3):=20RTCFR+-style=20friction=20to?= =?UTF-8?q?ward=20a=20moving=20reference=20in=20the=20sampled=20walk=20?= =?UTF-8?q?=E2=80=94=20=CE=BC=20=3D=200=20is=20today's=20walk=20bit=20for?= =?UTF-8?q?=20bit,=20and=20on=20Leduc=20it=20makes=20CFR+'s=20current=20st?= =?UTF-8?q?rategy=20converge?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Claude-Session: https://claude.ai/code/session_014D1ZwBFFxJAkajfcQPe6bx --- .../2026-11-09-friction-mini-experiment.md | 45 ++++++ scripts/minidrawmaha/friction_experiment.py | 105 ++++++++++++++ .../minidrawmaha/solver/friction.py | 117 ++++++++++++++++ .../minidrawmaha/solver/lockstep.py | 28 +++- .../minidrawmaha/solver/mccfr.py | 69 ++++++++-- .../minidrawmaha/solver/parallel.py | 4 + tests/minidrawmaha/test_friction.py | 130 ++++++++++++++++++ 7 files changed, 482 insertions(+), 16 deletions(-) create mode 100644 docs/superpowers/plans/2026-11-09-friction-mini-experiment.md create mode 100644 scripts/minidrawmaha/friction_experiment.py create mode 100644 src/drawmaha_solver/minidrawmaha/solver/friction.py create mode 100644 tests/minidrawmaha/test_friction.py diff --git a/docs/superpowers/plans/2026-11-09-friction-mini-experiment.md b/docs/superpowers/plans/2026-11-09-friction-mini-experiment.md new file mode 100644 index 00000000..167962f6 --- /dev/null +++ b/docs/superpowers/plans/2026-11-09-friction-mini-experiment.md @@ -0,0 +1,45 @@ +# Friction (RTCFR+-style) on sampled mini-drawmaha: a mini experiment + +**Goal.** Does a friction term toward a moving reference make the *current* strategy of +sampled tabular MCCFR converge on mini-drawmaha, and which inner rule carries it best: +CFR+ (W = 10), mini-batch CFR+ (W = 100), or LCFR (W = 10)? + +**Done when.** Nine lockstep runs (3 rules × μ ∈ {0, 0.1, 1}) reach 1M hands per seat +in parallel, each graded at 50k / 100k / 250k / 500k / 1M hands on its current strategy +and its linear average, and the results sit in one JSON per run plus a curves figure. +μ = 0 is each rule's no-friction control. + +## The friction (Meng et al. 2023, arXiv 2308.11256, Eq. 7–8, adapted to sampling) + +RTCFR+ adds μ·(x̂ − r̂) to each player's sequence-form loss: x̂(I,a) is the player's own +reach of sequence (I,a), r̂ the reference's. Its counterfactual value at (I,a) is the +payoff value minus μ·(x̂(I,a) − r̂(I,a)), plus the children's (already transformed) values. + +Under external sampling the payoff part of a banked regret is, in expectation, weighted +by the opponent-and-chance reach of I. We apply the friction undivided on each visit +(choice B), so its expected weight carries the same reach: friction and payoff keep the +same ratio at every infoset. The paper's unweighted form would need a 1/reach factor and +explode at rare rows. γ (the paper's 1e-10 mix toward uniform) is numerically inert at +this scale and is left out. Every R hands per seat the reference is reset to the current +strategy; regret totals are kept (the paper: resetting them stops convergence). + +## Pieces + +1. `solver/friction.py`: `Friction(mu, table, reference)`; `Friction.new(table, mu)` + (reference = current strategy, uniform at the start); `of(key)`; `reset()`. + Vectorised current strategy over a packed table. +2. `mccfr.traverse`: optional `friction`, carrying the traverser's own reach and the + reference's reach down its own actions; utilities at its spots minus μ(x − r). + `friction=None` is today's walk, bit for bit. +3. `lockstep`: `LockstepSolve.friction` handed to every walk; `save_lockstep` and + `ParallelLockstep` refuse a run with friction (not checkpointed, not parallelised). +4. `scripts/minidrawmaha/friction_experiment.py`: one run; resets, grades, JSON lines. +5. Tests (`tests/minidrawmaha/test_friction.py`), written first: + - friction with μ = 0 banks today's regrets bit for bit (Leduc, lockstep W = 4); + - a reference equal to the current strategy along with equal reaches cancels: at t = 1 + (all σ uniform, reference uniform) any μ banks today's regrets bit for bit; + - the hand-computed adjustment at one spot; + - the vectorised reset equals `RegretMatcher.strategy` row by row (packed slice); + - save and parallel refuse friction by name; + - Leduc gate (slow): friction + CFR+ drives the current strategy's exploitability + below no-friction CFR+'s current strategy. diff --git a/scripts/minidrawmaha/friction_experiment.py b/scripts/minidrawmaha/friction_experiment.py new file mode 100644 index 00000000..01ddecd2 --- /dev/null +++ b/scripts/minidrawmaha/friction_experiment.py @@ -0,0 +1,105 @@ +"""One arm of the friction mini experiment: lockstep MCCFR, with or without RTCFR+-style friction. + + uv run python scripts/minidrawmaha/friction_experiment.py --rule cfr+ --workers 10 --mu 0.1 \ + --out runs/friction-mini/cfrplus-w10-mu0.1.jsonl + +Trains one run in this process (`train_lockstep`; friction is not parallelised) to +`--hands` hands per seat. Every `--reset-hands` hands per seat it moves the friction's +reference to the current strategy (RTCFR+'s r ← x̂; see `solver/friction.py`). At each +mark it grades BOTH the current strategy (what the friction is meant to make converge) +and the linear average (what plain CFR ships) with the exact meter, and appends one +JSON line: hands per seat, iteration, the two exploitabilities in chips per hand, and +the seconds so far. `--mu 0` trains plain MCCFR, the arm's no-friction control. + +Costs: about 0.3 GB for the table and as much again for the reference, plus the +meter's 1.5 GB; about 200 hands per seat per second at W = 10 on one core, and two +grades of about two minutes each per mark. No checkpoint is written: a friction +run's reference is not saved, so an interrupted run is rerun. +""" + +from __future__ import annotations + +import argparse +import json +import time +from pathlib import Path + +from drawmaha_solver.minidrawmaha.exploitability import exploitability, minidrawmaha_game +from drawmaha_solver.minidrawmaha.solver.friction import Friction +from drawmaha_solver.minidrawmaha.solver.infoset_table import ( + average_strategy, + current_strategy, +) +from drawmaha_solver.minidrawmaha.solver.lockstep import new_lockstep, train_lockstep + +DEFAULT_MARKS = (50_000, 100_000, 250_000, 500_000, 1_000_000) + +def main() -> None: + """Train one arm to its last mark, resetting the reference and grading on the way.""" + args = _parse() + reset_every = _iterations(args.reset_hands, args.workers, "--reset-hands") + marks = sorted({_iterations(m, args.workers, "--marks") for m in args.marks}) + solve = new_lockstep(args.seed, workers=args.workers, rule=args.rule) + if args.mu > 0: + solve.friction = Friction.new(solve.table, args.mu) + game = minidrawmaha_game() + args.out.parent.mkdir(parents=True, exist_ok=True) + started = time.perf_counter() + + for t in _events(reset_every, marks): + train_lockstep(solve, t - solve.iteration) + if solve.friction is not None and t % reset_every == 0: + solve.friction.reset() + if t in marks: + line = { + "rule": args.rule, + "workers": args.workers, + "mu": args.mu, + "reset_hands": args.reset_hands, + "seed": args.seed, + "iteration": t, + "hands_per_seat": t * args.workers, + "current": exploitability(current_strategy(solve.table), game=game), + "average": exploitability(average_strategy(solve.table), game=game), + "seconds": round(time.perf_counter() - started, 1), + } + with args.out.open("a") as file: + file.write(json.dumps(line) + "\n") + print(json.dumps(line), flush=True) + +def _events(reset_every: int, marks: list[int]) -> list[int]: + """Every iteration at which training must pause: each reset and each mark, in order. + + In: the reset cadence and the mark iterations. Out: the sorted union up to the + last mark. Purpose: `main` trains from one pause to the next, so a reset lands + exactly on its iteration and a grade sees the table as of its mark. + """ + last = marks[-1] + return sorted(set(range(reset_every, last + 1, reset_every)) | set(marks)) + +def _iterations(hands: int, workers: int, flag: str) -> int: + """Hands per seat as lockstep iterations (W hands per seat each); refused unless exact. + + Purpose: arms at different W are compared at equal hands, so a mark or a + reset that does not land on a whole iteration would compare unequal amounts. + """ + if hands <= 0 or hands % workers: + raise SystemExit(f"{flag}: {hands:,} hands is not a positive multiple of W = {workers}") + return hands // workers + +def _parse() -> argparse.Namespace: + """The command line: rule, W, μ, reset cadence, budget, seed and output file.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--rule", choices=["lcfr", "cfr+"], required=True) + parser.add_argument("--workers", type=int, default=10, help="W: hands per seat per iteration") + parser.add_argument("--mu", type=float, default=0.0, help="friction strength in chips; 0 = none") + parser.add_argument("--reset-hands", type=int, default=10_000, + help="hands per seat between reference resets") + parser.add_argument("--marks", type=int, nargs="+", default=list(DEFAULT_MARKS), + help="hands per seat at which to grade") + parser.add_argument("--seed", type=int, default=0) + parser.add_argument("--out", type=Path, required=True) + return parser.parse_args() + +if __name__ == "__main__": + main() diff --git a/src/drawmaha_solver/minidrawmaha/solver/friction.py b/src/drawmaha_solver/minidrawmaha/solver/friction.py new file mode 100644 index 00000000..fd252c74 --- /dev/null +++ b/src/drawmaha_solver/minidrawmaha/solver/friction.py @@ -0,0 +1,117 @@ +"""Friction toward a reference strategy: RTCFR+'s reward transformation, under sampling. + +Why it exists. A CFR run's CURRENT strategy circles the equilibrium forever; only +its average converges. RTCFR+ (Meng et al. 2023, arXiv 2308.11256, Eq. 7–8) changes +the game the regrets are learning so that the current strategy itself settles: each +player's loss gains μ·(x̂ − r̂), where x̂(I, a) is the player's own reach of the +sequence "arrive at I, then play a" and r̂(I, a) is the same for a REFERENCE +strategy. Playing a sequence more than the reference does is taxed; playing it less +earns a bonus. That pull makes the taxed game's equilibrium unique and the current +strategy spiral into it. The equilibrium is biased toward the reference, so every +so often the reference is reset to the current strategy (`Friction.reset`), and the +biased equilibria walk toward the real one. Regret totals are kept across a reset; +the paper reports that zeroing them stops convergence. + +What the walk does with it. At each of the traverser's spots `mccfr.traverse` +subtracts `friction_adjustment(...)` from the action values before the regret is +formed, and passes each child its own reach σ(a)·x and the reference's +reference(a)·r. Values returned up the tree already carry the deeper spots' +friction, as the paper's counterfactual-value recursion does. + +One departure from the paper, forced by sampling. In the paper the friction enters +every infoset's value with full weight, every iteration. External sampling banks a +row only when it is visited, and the payoff part of what it banks is weighted, in +expectation, by the chance-and-opponent reach of the row. The friction here is added +undivided on each visit, so it carries the same reach weight: at every row the tax +and the payoff keep the ratio they have in the paper, but the tax is reach-weighted. +Dividing by the visit probability instead would recover the paper's weighting and +hand rare rows enormous, noisy shoves. The paper's γ (a 1e-10 mix toward uniform) +is numerically inert at this scale and is left out. +""" + +from __future__ import annotations + +from collections.abc import Hashable +from dataclasses import dataclass, field + +import numpy as np + +from drawmaha_solver.minidrawmaha.solver.packed_table import PackedTable + +def friction_adjustment( + sigma: np.ndarray, + reference: np.ndarray, + *, + own_reach: float, + reference_reach: float, + mu: float, +) -> np.ndarray: + """μ·(x̂ − r̂) at one spot: what to subtract from each action's value there. + + In: `sigma` and `reference`, the spot's current and reference strategies (one + probability per legal action); `own_reach` and `reference_reach`, the chance + that the traverser's own play (resp. the reference's) gets to this spot, its + opponent and the deck not counted. Out: one tax per action, in chips. + Positive where play out-reaches the reference, negative where it lags. + """ + return mu * (own_reach * sigma - reference_reach * reference) + +@dataclass(slots=True) +class Friction: + """The tax's strength μ (chips) and the reference strategy it pulls toward. + + `table` is the run's table; `reference` holds one probability per column, a + flat array aligned with a packed table's `cumulative_regret`, or a dict of + arrays for a dict table (Leduc, tests). Built by `new`, never by hand. + """ + + mu: float + table: object + reference: np.ndarray | dict[Hashable, np.ndarray] = field(repr=False) + + @classmethod + def new(cls, table, mu: float) -> Friction: + """Friction of strength `mu` whose reference is the table's current play. + + On a fresh table that is uniform everywhere, which is RTCFR+'s starting + reference. A negative or non-finite μ is refused by name. + """ + if not np.isfinite(mu) or mu < 0: + raise ValueError(f"friction mu must be a finite number ≥ 0, got {mu!r}") + friction = cls(mu=float(mu), table=table, reference={}) + friction.reset() + return friction + + def of(self, key: Hashable) -> np.ndarray: + """The reference strategy at `key`, one probability per legal action.""" + if isinstance(self.table, PackedTable): + _, start, width = self.table.slot_of(key) + return self.reference[start : start + width] + return self.reference[key] + + def reset(self) -> None: + """Move the reference to what the table plays now (RTCFR+'s r ← x̂). + + Regrets are left alone. A packed table is copied in one vectorised pass; + a dict table ledger by ledger. + """ + if isinstance(self.table, PackedTable): + self.reference = packed_current_strategy(self.table) + else: + self.reference = {key: ledger.strategy() for key, ledger in self.table.items()} + +def packed_current_strategy(table: PackedTable) -> np.ndarray: + """Every row's current strategy at once, flat and column-aligned with the table. + + In: a packed table. Out: an array the length of `cumulative_regret` holding, + per row, the positive regrets normalised, or uniform where none is positive — + the same numbers `RegretMatcher.strategy` gives row by row, without building + 6.2 million ledgers. + """ + widths = table.widths() + starts = np.concatenate(([0], np.cumsum(widths)[:-1])) + positive = np.maximum(table.cumulative_regret, 0.0) + totals = np.repeat(np.add.reduceat(positive, starts), widths) + sizes = np.repeat(widths, widths).astype(float) + with np.errstate(invalid="ignore", divide="ignore"): + return np.where(totals > 0.0, positive / totals, 1.0 / sizes) diff --git a/src/drawmaha_solver/minidrawmaha/solver/lockstep.py b/src/drawmaha_solver/minidrawmaha/solver/lockstep.py index fce3ef33..220c1131 100644 --- a/src/drawmaha_solver/minidrawmaha/solver/lockstep.py +++ b/src/drawmaha_solver/minidrawmaha/solver/lockstep.py @@ -49,6 +49,7 @@ import numpy as np from drawmaha_solver.minidrawmaha.rules.game import random_deal +from drawmaha_solver.minidrawmaha.solver.friction import Friction from drawmaha_solver.minidrawmaha.solver.mccfr import ( RootSampler, Table, @@ -99,6 +100,9 @@ class LockstepSolve: iteration: int = 0 rule: RegretRule = RegretRule.VANILLA averages: tuple[Average, ...] = () + # RTCFR+-style friction toward a reference (`friction.py`); None is plain MCCFR. + # Set by the caller after `new_lockstep`; never checkpointed or parallelised. + friction: Friction | None = None def new_lockstep( seed: int, @@ -138,7 +142,16 @@ def train_lockstep(solve: LockstepSolve, iterations: int) -> LockstepSolve: t = solve.iteration + 1 for seat in SEATS: recorded = [ - walk(solve.table, solve.deal, solve.seed, t, seat, worker, solve.averages) + walk( + solve.table, + solve.deal, + solve.seed, + t, + seat, + worker, + solve.averages, + friction=solve.friction, + ) for worker in range(solve.workers) ] apply(solve.table, recorded, solve.rule, t) @@ -169,8 +182,14 @@ def walk( seat: int, worker: int, averages: tuple[Average, ...], + *, + friction: Friction | None = None, ) -> Recorded: - """Worker `worker`'s walk for `seat` at iteration t, reading σₜ and writing nothing.""" + """Worker `worker`'s walk for `seat` at iteration t, reading σₜ and writing nothing. + + `friction` taxes the traverser's values toward its reference (`friction.py`); + it reads the reference only, so the walk still writes nothing. + """ rng = stream(seed, t, seat, worker) recorder = _Recorder(table, len(averages)) traverse( @@ -181,6 +200,7 @@ def walk( t, bank=recorder.bank, column_weights=extra_weights(averages, t), + friction=friction, ) return recorder.recorded() @@ -371,6 +391,10 @@ def save_lockstep(solve: LockstepSolve, path: Path) -> None: """ if not solve.table: raise ValueError("the table is empty; there is nothing to checkpoint") + if solve.friction is not None: + # The reference and μ are not in the file, so a resume would silently + # train a different game. + raise ValueError("a run with friction is not checkpointed: its reference is not saved") path = Path(path) regret, strategy_sum, extra_sums, stamps = _flat_slots(solve.table) temporary = path.with_name(path.name + ".tmp") diff --git a/src/drawmaha_solver/minidrawmaha/solver/mccfr.py b/src/drawmaha_solver/minidrawmaha/solver/mccfr.py index 08875af9..1f3b7a11 100644 --- a/src/drawmaha_solver/minidrawmaha/solver/mccfr.py +++ b/src/drawmaha_solver/minidrawmaha/solver/mccfr.py @@ -98,6 +98,7 @@ import numpy as np from drawmaha_solver.minidrawmaha.rules.game import random_deal +from drawmaha_solver.minidrawmaha.solver.friction import Friction, friction_adjustment from drawmaha_solver.minidrawmaha.solver.infoset_table import new_infoset_table from drawmaha_solver.minidrawmaha.solver.packed_table import PackedTable from drawmaha_solver.minidrawmaha.solver.regret_rules import ( @@ -281,6 +282,9 @@ def traverse( *, bank: BankRegret = bank_vanilla, column_weights: np.ndarray | None = None, + friction: Friction | None = None, + own_reach: float = 1.0, + reference_reach: float = 1.0, ) -> float: """Chips to `traverser` from `state` along one sampled path, banking on the way up. @@ -297,6 +301,11 @@ def traverse( than through `RegretMatcher.update`, which would re-derive σ and re-check its inputs at every visit; under vanilla it does the same arithmetic, and `test_mccfr.py` holds the two to identical bytes. + + `friction` (see `friction.py`) taxes the traverser's action values by how far + its own play out-reaches a reference; `own_reach` and `reference_reach` are + the chances that the traverser's own play, and the reference's, lead here. + None is today's walk, bit for bit, and the reaches are then never read. """ if state.is_terminal(): return state.returns()[traverser] @@ -315,6 +324,9 @@ def traverse( t, bank=bank, column_weights=column_weights, + friction=friction, + own_reach=own_reach, + reference_reach=reference_reach, ) ledger = table[state.infoset()] @@ -326,20 +338,46 @@ def traverse( # was read before the subtree ran and is still this ledger's strategy: # perfect recall keeps the subtree from reaching this spot again. if state.current_player == traverser: - utilities = np.array( - [ - traverse( - state.apply(action), - table, - traverser, - rng, - t, - bank=bank, - column_weights=column_weights, - ) - for action in actions - ] - ) + if friction is None: + utilities = np.array( + [ + traverse( + state.apply(action), + table, + traverser, + rng, + t, + bank=bank, + column_weights=column_weights, + ) + for action in actions + ] + ) + else: + reference = friction.of(state.infoset()) + utilities = np.array( + [ + traverse( + state.apply(action), + table, + traverser, + rng, + t, + bank=bank, + column_weights=column_weights, + friction=friction, + own_reach=own_reach * sigma[k], + reference_reach=reference_reach * reference[k], + ) + for k, action in enumerate(actions) + ] + ) - friction_adjustment( + sigma, + reference, + own_reach=own_reach, + reference_reach=reference_reach, + mu=friction.mu, + ) value = sigma @ utilities bank(ledger, utilities - value, t) return float(value) @@ -359,6 +397,9 @@ def traverse( t, bank=bank, column_weights=column_weights, + friction=friction, + own_reach=own_reach, + reference_reach=reference_reach, ) def _pick(rng: np.random.Generator, probabilities: Sequence[float]) -> int: diff --git a/src/drawmaha_solver/minidrawmaha/solver/parallel.py b/src/drawmaha_solver/minidrawmaha/solver/parallel.py index 338807f1..c7fa63a0 100644 --- a/src/drawmaha_solver/minidrawmaha/solver/parallel.py +++ b/src/drawmaha_solver/minidrawmaha/solver/parallel.py @@ -218,6 +218,10 @@ class ParallelLockstep: def __init__(self, solve: LockstepSolve, *, keys: Sequence[InfoSet] | None = None): if not isinstance(solve.table, PackedTable): raise TypeError("a parallel run needs a PackedTable") + if solve.friction is not None: + # The workers would need the reference in shared memory, and a reset + # between phases; the friction experiment runs in one process instead. + raise ValueError("a run with friction is not parallelised; train it with train_lockstep") self.solve = solve self._keys = None if keys is None else tuple(keys) # The caller's table, set aside while the shared one stands in for it. diff --git a/tests/minidrawmaha/test_friction.py b/tests/minidrawmaha/test_friction.py new file mode 100644 index 00000000..cfbbf5d0 --- /dev/null +++ b/tests/minidrawmaha/test_friction.py @@ -0,0 +1,130 @@ +"""Friction toward a reference (RTCFR+-style) in the sampled walk. + +* **Off is today.** `friction=None` and friction at μ = 0 bank exactly today's regrets. +* **A reference equal to play cancels.** At t = 1 every σ is uniform; a uniform + reference has the same reach, so friction at any μ changes nothing. +* **One spot, by hand.** The adjustment is μ·(own reach·σ − reference reach·reference). +* **The reset** copies every row's current strategy, the same numbers + `RegretMatcher.strategy` gives. +* **Refusals.** A friction run is neither checkpointed nor parallelised. +* **Leduc gate (slow).** Friction + CFR+ makes the current strategy converge where + plain CFR+'s current strategy does not. +""" + +from __future__ import annotations + +import os + +import numpy as np +import pytest + +from drawmaha_solver.leduc.exploitability import exploitability +from drawmaha_solver.leduc.game import DEALS, LeducState +from drawmaha_solver.leduc.infoset_table import new_infoset_table as new_leduc_table +from drawmaha_solver.minidrawmaha.rules.enumeration import all_infosets +from drawmaha_solver.minidrawmaha.solver.friction import Friction, friction_adjustment +from drawmaha_solver.minidrawmaha.solver.lockstep import ( + new_lockstep, + save_lockstep, + train_lockstep, +) +from drawmaha_solver.minidrawmaha.solver.packed_table import PackedTable +from drawmaha_solver.minidrawmaha.solver.parallel import ParallelLockstep +from drawmaha_solver.regret_matching import RegretMatcher + +def leduc_deal(rng: np.random.Generator) -> LeducState: + return LeducState(cards=DEALS[rng.integers(len(DEALS))]) + +def fresh_leduc() -> dict: + return {key: RegretMatcher(ledger.n_actions) for key, ledger in new_leduc_table().items()} + +def regrets(table: dict) -> bytes: + return b"".join(table[key].cumulative_regret.tobytes() for key in table) + +def leduc_run(rule: str, workers: int, *, mu: float | None, iterations: int) -> dict: + table = fresh_leduc() + solve = new_lockstep(7, workers=workers, table=table, deal=leduc_deal, rule=rule) + if mu is not None: + solve.friction = Friction.new(table, mu) + train_lockstep(solve, iterations) + return table + +@pytest.mark.parametrize("rule", ["lcfr", "cfr+"]) +def test_friction_at_mu_zero_banks_todays_regrets(rule): + plain = leduc_run(rule, 4, mu=None, iterations=30) + zero = leduc_run(rule, 4, mu=0.0, iterations=30) + assert regrets(plain) == regrets(zero) + +def test_a_uniform_reference_cancels_at_the_first_iteration(): + plain = leduc_run("lcfr", 4, mu=None, iterations=1) + pulled = leduc_run("lcfr", 4, mu=5.0, iterations=1) + assert regrets(plain) == regrets(pulled) + +def test_friction_moves_regrets_once_play_leaves_the_reference(): + plain = leduc_run("lcfr", 4, mu=None, iterations=20) + pulled = leduc_run("lcfr", 4, mu=5.0, iterations=20) + assert regrets(plain) != regrets(pulled) + +def test_the_adjustment_at_one_spot_by_hand(): + sigma = np.array([0.5, 0.25, 0.25]) + reference = np.array([0.2, 0.2, 0.6]) + # own reach 0.4, reference reach 0.5, μ = 2: + # x = 0.4σ = (0.2, 0.1, 0.1); r = 0.5·ref = (0.1, 0.1, 0.3); μ(x − r) = (0.2, 0, −0.4) + got = friction_adjustment(sigma, reference, own_reach=0.4, reference_reach=0.5, mu=2.0) + np.testing.assert_allclose(got, [0.2, 0.0, -0.4], rtol=0, atol=1e-15) + +def test_the_reset_copies_every_rows_current_strategy(): + keys = list(all_infosets())[:5000] + table = PackedTable.listed(keys) + rng = np.random.default_rng(3) + table.cumulative_regret[:] = rng.normal(size=table.cumulative_regret.size) + # some rows all non-positive, so the uniform fallback is exercised + table.cumulative_regret[: 40] = -1.0 + friction = Friction.new(table, 1.0) + # Equal to the last bit on most rows; `np.add.reduceat` sums a 3- or 4-wide row + # in a different order than `ndarray.sum`, which moves ~2% of rows by one ulp. + # A reference only sets where the pull points, so one ulp is immaterial. + for key in keys[::37]: + np.testing.assert_allclose(friction.of(key), table[key].strategy(), rtol=0, atol=1e-15) + +def test_a_dict_reset_tracks_play(): + table = fresh_leduc() + friction = Friction.new(table, 1.0) + key = next(iter(table)) + table[key].cumulative_regret[:] = np.arange(table[key].n_actions, dtype=float) + friction.reset() + assert friction.of(key).tobytes() == table[key].strategy().tobytes() + +def test_a_friction_run_is_not_checkpointed(tmp_path): + table = fresh_leduc() + solve = new_lockstep(1, workers=2, table=table, deal=leduc_deal, rule="lcfr") + solve.friction = Friction.new(table, 1.0) + with pytest.raises(ValueError, match="friction"): + save_lockstep(solve, tmp_path / "solve.npz") + +def test_a_friction_run_is_not_parallelised(): + table = PackedTable.listed(list(all_infosets())[:100]) + solve = new_lockstep(1, workers=2, table=table, rule="lcfr") + solve.friction = Friction.new(table, 1.0) + with pytest.raises(ValueError, match="friction"): + ParallelLockstep(solve) + +@pytest.mark.skipif( + not os.environ.get("MINIDRAWMAHA_FULL_CALIBRATION"), + reason="Leduc friction gate: ~minutes; set MINIDRAWMAHA_FULL_CALIBRATION=1", +) +def test_leduc_friction_makes_cfr_plus_current_strategy_converge(): + def current_exploitability(table: dict) -> float: + return exploitability({key: ledger.strategy() for key, ledger in table.items()}) + + plain = fresh_leduc() + solve = new_lockstep(0, workers=32, table=plain, deal=leduc_deal, rule="cfr+") + train_lockstep(solve, 3000) + + pulled = fresh_leduc() + solve = new_lockstep(0, workers=32, table=pulled, deal=leduc_deal, rule="cfr+") + solve.friction = Friction.new(pulled, 0.05) + for _ in range(30): + train_lockstep(solve, 100) + solve.friction.reset() + assert current_exploitability(pulled) < current_exploitability(plain) From 931ca0a2da54230e2a94a643f39eb6035e6571b0 Mon Sep 17 00:00:00 2001 From: Alec Ondo Date: Mon, 9 Nov 2026 08:01:11 -0500 Subject: [PATCH 2/3] =?UTF-8?q?feat(rung3):=20friction=20at=20=CE=BC=20=3D?= =?UTF-8?q?=2010=20ends=202=E2=80=934%=20below=20its=20control=20at=201M?= =?UTF-8?q?=20hands=20and=20costs=205=E2=80=9311%=20before=20that=20?= =?UTF-8?q?=E2=80=94=20the=20current=20strategy=20already=20beats=20the=20?= =?UTF-8?q?average=20at=20every=20mark=20without=20it?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The friction experiment script gains --save-table (the finished table's regrets, strategy sums, widths and reference, for row-level analysis), and the graded record of every arm lands in figures/rung3/friction_mini.jsonl: CFR+ W = 10, CFR+ W = 100 and LCFR W = 10 at μ ∈ {0, 10} to 1M hands per seat, μ = 0.1 to 50k, μ ∈ {1, 100} for CFR+ and LCFR W = 10 to 500k. One seed per arm. Claude-Session: https://claude.ai/code/session_014D1ZwBFFxJAkajfcQPe6bx --- .../2026-11-09-friction-mini-experiment.md | 11 +++++ figures/AGENTS.md | 8 +++ figures/rung3/friction_mini.jsonl | 49 +++++++++++++++++++ scripts/minidrawmaha/friction_experiment.py | 25 ++++++++++ 4 files changed, 93 insertions(+) create mode 100644 figures/rung3/friction_mini.jsonl diff --git a/docs/superpowers/plans/2026-11-09-friction-mini-experiment.md b/docs/superpowers/plans/2026-11-09-friction-mini-experiment.md index 167962f6..b08257be 100644 --- a/docs/superpowers/plans/2026-11-09-friction-mini-experiment.md +++ b/docs/superpowers/plans/2026-11-09-friction-mini-experiment.md @@ -43,3 +43,14 @@ strategy; regret totals are kept (the paper: resetting them stops convergence). - save and parallel refuse friction by name; - Leduc gate (slow): friction + CFR+ drives the current strategy's exploitability below no-friction CFR+'s current strategy. + +## As run (9 Nov 2026) + +The machine could not hold nine 1M-hand runs at once (12 concurrent runs land on the +efficiency cores at about 35 hands per seat per second), so the grid ran in waves: +μ ∈ {0, 10} for all three rules to 1M; μ = 0.1 stopped at 50k (indistinguishable +from μ = 0); μ ∈ {1, 100} for CFR+ and LCFR at W = 10 to 500k. The record is +`figures/rung3/friction_mini.jsonl`. Outcome: μ = 10 ends 2–4% below its control at +1M in all three pairs and is 5–11% worse at 50k; μ = 100 is 20–56% worse; the +current strategy beats the linear average at every mark of every arm; LCFR beats +CFR+ by 30%; W = 100 tracks W = 10 within 1.6%. diff --git a/figures/AGENTS.md b/figures/AGENTS.md index dbfa6680..79944fa6 100644 --- a/figures/AGENTS.md +++ b/figures/AGENTS.md @@ -99,3 +99,11 @@ by `run_deep_cfr.py --rule dcfr` and `--rule cfr+`, its marks graded by `grades.json`. `rung4/rule_sign_flips.json` is the measurement of why they lost, written by `scripts/deepcfr/rule_sign_flips.py` from the same runs' advantage memories. + +`rung3/friction_mini.jsonl` is the friction mini experiment's graded record: +every JSON line `scripts/minidrawmaha/friction_experiment.py` appended, one per +grading mark per arm (rule, hands per update W, friction strength μ, current +and linear-average exploitability), concatenated from the arms' output files. +Arms: CFR+ W = 10, CFR+ W = 100 and LCFR W = 10 at μ = 0 and 10 to 1M hands +per seat; at μ = 0.1 to 50k only; CFR+ and LCFR W = 10 at μ = 1 and 100 to +500k. One seed (0) per arm. diff --git a/figures/rung3/friction_mini.jsonl b/figures/rung3/friction_mini.jsonl new file mode 100644 index 00000000..ad7a06f0 --- /dev/null +++ b/figures/rung3/friction_mini.jsonl @@ -0,0 +1,49 @@ +{"rule": "cfr+", "workers": 10, "mu": 0.1, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 3.5562435935672143, "average": 3.731157099798681, "seconds": 594.4} +{"rule": "cfr+", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 3.5517268959295873, "average": 3.7356591304923175, "seconds": 576.0} +{"rule": "cfr+", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 100000, "current": 3.2509838120613503, "average": 3.417605395721135, "seconds": 1130.8} +{"rule": "cfr+", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 25000, "hands_per_seat": 250000, "current": 2.5997235419862417, "average": 2.9387484921647653, "seconds": 2253.6} +{"rule": "cfr+", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 50000, "hands_per_seat": 500000, "current": 1.8008132370632826, "average": 2.333622478692331, "seconds": 3948.0} +{"rule": "cfr+", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 100000, "hands_per_seat": 1000000, "current": 1.0736841309465013, "average": 1.5643887958807077, "seconds": 6909.9} +{"rule": "cfr+", "workers": 100, "mu": 0.1, "reset_hands": 10000, "seed": 0, "iteration": 500, "hands_per_seat": 50000, "current": 3.547235342566566, "average": 3.7194481621097455, "seconds": 618.8} +{"rule": "cfr+", "workers": 100, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 500, "hands_per_seat": 50000, "current": 3.531311741835638, "average": 3.6944582700377397, "seconds": 605.8} +{"rule": "cfr+", "workers": 100, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 1000, "hands_per_seat": 100000, "current": 3.2620827552899994, "average": 3.4310462865215943, "seconds": 1224.0} +{"rule": "cfr+", "workers": 100, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 2500, "hands_per_seat": 250000, "current": 2.61170158222152, "average": 2.9414771241850843, "seconds": 2487.5} +{"rule": "cfr+", "workers": 100, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 500000, "current": 1.777823102998135, "average": 2.3347398527773153, "seconds": 4421.1} +{"rule": "cfr+", "workers": 100, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 1000000, "current": 1.0911306210839766, "average": 1.5658667023559403, "seconds": 7729.8} +{"rule": "lcfr", "workers": 10, "mu": 0.1, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 3.1816373922487835, "average": 3.374124363882292, "seconds": 605.3} +{"rule": "lcfr", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 3.1867334340455207, "average": 3.4041814167616087, "seconds": 555.0} +{"rule": "lcfr", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 100000, "current": 2.894129293353006, "average": 3.048347266505258, "seconds": 1098.3} +{"rule": "lcfr", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 25000, "hands_per_seat": 250000, "current": 2.22591541247425, "average": 2.5380436308105256, "seconds": 2189.5} +{"rule": "lcfr", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 50000, "hands_per_seat": 500000, "current": 1.386015778439032, "average": 1.8762884428553426, "seconds": 3803.6} +{"rule": "lcfr", "workers": 10, "mu": 0.0, "reset_hands": 10000, "seed": 0, "iteration": 100000, "hands_per_seat": 1000000, "current": 0.767706163376485, "average": 1.1307747045097751, "seconds": 6629.3} +{"rule": "cfr+", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 3.7207998389882166, "average": 3.9205683871690127, "seconds": 591.9} +{"rule": "cfr+", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 100000, "current": 3.3316192002878555, "average": 3.5577799505608265, "seconds": 1126.9} +{"rule": "cfr+", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 25000, "hands_per_seat": 250000, "current": 2.6303120109575158, "average": 3.002654376742573, "seconds": 2309.0} +{"rule": "cfr+", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 50000, "hands_per_seat": 500000, "current": 1.7795442998298157, "average": 2.348595071065766, "seconds": 4086.8} +{"rule": "cfr+", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 100000, "hands_per_seat": 1000000, "current": 1.043179960227394, "average": 1.5518714019797173, "seconds": 7044.5} +{"rule": "cfr+", "workers": 100, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 500, "hands_per_seat": 50000, "current": 3.705894622009329, "average": 3.9170308452958436, "seconds": 610.3} +{"rule": "cfr+", "workers": 100, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 1000, "hands_per_seat": 100000, "current": 3.3195559232993808, "average": 3.5614458260723794, "seconds": 1173.3} +{"rule": "cfr+", "workers": 100, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 2500, "hands_per_seat": 250000, "current": 2.6109102211910113, "average": 2.994395712916459, "seconds": 2471.8} +{"rule": "cfr+", "workers": 100, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 500000, "current": 1.7855622367234023, "average": 2.350620118255279, "seconds": 4466.7} +{"rule": "cfr+", "workers": 100, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 1000000, "current": 1.0507719008332268, "average": 1.5545607674948498, "seconds": 7895.4} +{"rule": "lcfr", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 3.552082275982865, "average": 3.7637730030516785, "seconds": 591.3} +{"rule": "lcfr", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 100000, "current": 3.1253773110250855, "average": 3.3739519637026376, "seconds": 1132.6} +{"rule": "lcfr", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 25000, "hands_per_seat": 250000, "current": 2.361427008742529, "average": 2.7362553915378487, "seconds": 2298.9} +{"rule": "lcfr", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 50000, "hands_per_seat": 500000, "current": 1.5121865175932423, "average": 2.0468756266790447, "seconds": 4053.2} +{"rule": "lcfr", "workers": 10, "mu": 10.0, "reset_hands": 10000, "seed": 0, "iteration": 100000, "hands_per_seat": 1000000, "current": 0.7527588977522754, "average": 1.2125338427647108, "seconds": 6965.3} +{"rule": "cfr+", "workers": 10, "mu": 1.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 3.5231938498109763, "average": 3.708012881728954, "seconds": 603.9} +{"rule": "cfr+", "workers": 10, "mu": 1.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 100000, "current": 3.264553079307725, "average": 3.4141293108046815, "seconds": 1090.4} +{"rule": "cfr+", "workers": 10, "mu": 1.0, "reset_hands": 10000, "seed": 0, "iteration": 25000, "hands_per_seat": 250000, "current": 2.6076737447371947, "average": 2.9508023108123815, "seconds": 2101.5} +{"rule": "cfr+", "workers": 10, "mu": 1.0, "reset_hands": 10000, "seed": 0, "iteration": 50000, "hands_per_seat": 500000, "current": 1.7892240350990598, "average": 2.328847033417266, "seconds": 3620.6} +{"rule": "cfr+", "workers": 10, "mu": 100.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 4.256367092517985, "average": 4.347168468874717, "seconds": 590.1} +{"rule": "cfr+", "workers": 10, "mu": 100.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 100000, "current": 3.9722703282887686, "average": 4.135957583218573, "seconds": 1057.9} +{"rule": "cfr+", "workers": 10, "mu": 100.0, "reset_hands": 10000, "seed": 0, "iteration": 25000, "hands_per_seat": 250000, "current": 3.1379024016058743, "average": 3.5781596647132923, "seconds": 2072.6} +{"rule": "cfr+", "workers": 10, "mu": 100.0, "reset_hands": 10000, "seed": 0, "iteration": 50000, "hands_per_seat": 500000, "current": 2.2050062895255187, "average": 2.842050354505566, "seconds": 3594.5} +{"rule": "lcfr", "workers": 10, "mu": 1.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 3.2001286739426402, "average": 3.417872681156649, "seconds": 608.2} +{"rule": "lcfr", "workers": 10, "mu": 1.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 100000, "current": 2.8856417925374713, "average": 3.0570814178629706, "seconds": 1079.7} +{"rule": "lcfr", "workers": 10, "mu": 1.0, "reset_hands": 10000, "seed": 0, "iteration": 25000, "hands_per_seat": 250000, "current": 2.2387650009599414, "average": 2.5537640224399833, "seconds": 2065.7} +{"rule": "lcfr", "workers": 10, "mu": 1.0, "reset_hands": 10000, "seed": 0, "iteration": 50000, "hands_per_seat": 500000, "current": 1.389492853859736, "average": 1.8915239480846644, "seconds": 3523.9} +{"rule": "lcfr", "workers": 10, "mu": 100.0, "reset_hands": 10000, "seed": 0, "iteration": 5000, "hands_per_seat": 50000, "current": 4.191457154890394, "average": 4.261162260314845, "seconds": 594.5} +{"rule": "lcfr", "workers": 10, "mu": 100.0, "reset_hands": 10000, "seed": 0, "iteration": 10000, "hands_per_seat": 100000, "current": 3.9202451147143664, "average": 4.055580788894654, "seconds": 1076.7} +{"rule": "lcfr", "workers": 10, "mu": 100.0, "reset_hands": 10000, "seed": 0, "iteration": 25000, "hands_per_seat": 250000, "current": 3.1501192015758406, "average": 3.552543871858124, "seconds": 2082.3} +{"rule": "lcfr", "workers": 10, "mu": 100.0, "reset_hands": 10000, "seed": 0, "iteration": 50000, "hands_per_seat": 500000, "current": 2.157896653195825, "average": 2.803019908303629, "seconds": 3609.6} diff --git a/scripts/minidrawmaha/friction_experiment.py b/scripts/minidrawmaha/friction_experiment.py index 01ddecd2..be05fb85 100644 --- a/scripts/minidrawmaha/friction_experiment.py +++ b/scripts/minidrawmaha/friction_experiment.py @@ -24,6 +24,8 @@ import time from pathlib import Path +import numpy as np + from drawmaha_solver.minidrawmaha.exploitability import exploitability, minidrawmaha_game from drawmaha_solver.minidrawmaha.solver.friction import Friction from drawmaha_solver.minidrawmaha.solver.infoset_table import ( @@ -66,6 +68,27 @@ def main() -> None: with args.out.open("a") as file: file.write(json.dumps(line) + "\n") print(json.dumps(line), flush=True) + if args.save_table is not None: + _save_table(solve, args.save_table) + +def _save_table(solve, path: Path) -> None: + """The finished run's regrets, strategy sums and reference, for the row-level analysis. + + In: the trained run and a `.npz` path. Out: nothing; writes the packed table's + `cumulative_regret` and `strategy_sum` (one entry per column), its row widths + and, under friction, the last reference. Purpose: which rows were never + sampled, which sit uniform, and how far play sits from the reference are read + from these arrays after the run; this is not a resumable checkpoint. + """ + arrays = { + "cumulative_regret": solve.table.cumulative_regret, + "strategy_sum": solve.table.strategy_sum, + "widths": solve.table.widths(), + } + if solve.friction is not None: + arrays["reference"] = solve.friction.reference + path.parent.mkdir(parents=True, exist_ok=True) + np.savez(path, **arrays) def _events(reset_every: int, marks: list[int]) -> list[int]: """Every iteration at which training must pause: each reset and each mark, in order. @@ -99,6 +122,8 @@ def _parse() -> argparse.Namespace: help="hands per seat at which to grade") parser.add_argument("--seed", type=int, default=0) parser.add_argument("--out", type=Path, required=True) + parser.add_argument("--save-table", type=Path, default=None, + help="write the final table's arrays here (.npz) for row-level analysis") return parser.parse_args() if __name__ == "__main__": From cb0f4aa04b58cad62abe7e0bb33ce99c9eb29786 Mon Sep 17 00:00:00 2001 From: Alec Ondo Date: Mon, 9 Nov 2026 08:13:56 -0500 Subject: [PATCH 3/3] =?UTF-8?q?feat(rung3):=20friction's=20docstring=20sta?= =?UTF-8?q?tes=20the=20tax=20is=20down-weighted=20by=20=CF=80=E2=82=8B?= =?UTF-8?q?=E1=B5=A2=20against=20the=20paper,=20and=20tests=20pin=20its=20?= =?UTF-8?q?sign,=20=CE=BC=20refusals=20and=20the=20uniform=20reset=20branc?= =?UTF-8?q?h?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review follow-up on #95: the module docstring said the sampled tax keeps the paper's tax-to-payoff ratio; it keeps a constant ratio, i.e. μ acts as μ·π₋ᵢ(I). New tests: strong friction toward a fixed uniform reference leaves Leduc play nearer uniform than plain play; negative or non-finite μ is refused; the reset check covers the all-negative (uniform) rows. The script docstring drops a nonexistent --hands flag. Claude-Session: https://claude.ai/code/session_014D1ZwBFFxJAkajfcQPe6bx --- scripts/minidrawmaha/friction_experiment.py | 4 ++-- .../minidrawmaha/solver/friction.py | 20 +++++++++------- tests/minidrawmaha/test_friction.py | 24 ++++++++++++++++++- 3 files changed, 36 insertions(+), 12 deletions(-) diff --git a/scripts/minidrawmaha/friction_experiment.py b/scripts/minidrawmaha/friction_experiment.py index be05fb85..36e71474 100644 --- a/scripts/minidrawmaha/friction_experiment.py +++ b/scripts/minidrawmaha/friction_experiment.py @@ -3,8 +3,8 @@ uv run python scripts/minidrawmaha/friction_experiment.py --rule cfr+ --workers 10 --mu 0.1 \ --out runs/friction-mini/cfrplus-w10-mu0.1.jsonl -Trains one run in this process (`train_lockstep`; friction is not parallelised) to -`--hands` hands per seat. Every `--reset-hands` hands per seat it moves the friction's +Trains one run in this process (`train_lockstep`; friction is not parallelised) up +to the last of `--marks` hands per seat, where it stops. Every `--reset-hands` hands per seat it moves the friction's reference to the current strategy (RTCFR+'s r ← x̂; see `solver/friction.py`). At each mark it grades BOTH the current strategy (what the friction is meant to make converge) and the linear average (what plain CFR ships) with the exact meter, and appends one diff --git a/src/drawmaha_solver/minidrawmaha/solver/friction.py b/src/drawmaha_solver/minidrawmaha/solver/friction.py index fd252c74..224670a6 100644 --- a/src/drawmaha_solver/minidrawmaha/solver/friction.py +++ b/src/drawmaha_solver/minidrawmaha/solver/friction.py @@ -1,7 +1,7 @@ """Friction toward a reference strategy: RTCFR+'s reward transformation, under sampling. -Why it exists. A CFR run's CURRENT strategy circles the equilibrium forever; only -its average converges. RTCFR+ (Meng et al. 2023, arXiv 2308.11256, Eq. 7–8) changes +Why it exists. A CFR run's CURRENT strategy need not converge (in the worst case +it circles the equilibrium); only its average is guaranteed to. RTCFR+ (Meng et al. 2023, arXiv 2308.11256, Eq. 7–8) changes the game the regrets are learning so that the current strategy itself settles: each player's loss gains μ·(x̂ − r̂), where x̂(I, a) is the player's own reach of the sequence "arrive at I, then play a" and r̂(I, a) is the same for a REFERENCE @@ -9,8 +9,8 @@ earns a bonus. That pull makes the taxed game's equilibrium unique and the current strategy spiral into it. The equilibrium is biased toward the reference, so every so often the reference is reset to the current strategy (`Friction.reset`), and the -biased equilibria walk toward the real one. Regret totals are kept across a reset; -the paper reports that zeroing them stops convergence. +biased equilibria walk toward the real one. Regret totals are kept across a reset, +as in the paper's algorithm. What the walk does with it. At each of the traverser's spots `mccfr.traverse` subtracts `friction_adjustment(...)` from the action values before the regret is @@ -21,11 +21,13 @@ One departure from the paper, forced by sampling. In the paper the friction enters every infoset's value with full weight, every iteration. External sampling banks a row only when it is visited, and the payoff part of what it banks is weighted, in -expectation, by the chance-and-opponent reach of the row. The friction here is added -undivided on each visit, so it carries the same reach weight: at every row the tax -and the payoff keep the ratio they have in the paper, but the tax is reach-weighted. -Dividing by the visit probability instead would recover the paper's weighting and -hand rare rows enormous, noisy shoves. The paper's γ (a 1e-10 mix toward uniform) +expectation, by the chance-and-opponent reach π₋ᵢ(I) of the row; in the paper the +payoff is weighted the same way but the friction is not. The friction here is added +undivided on each visit, so in expectation it is scaled by π₋ᵢ(I) too: relative to +the paper the tax at row I is down-weighted by π₋ᵢ(I), i.e. μ acts as an effective +μ·π₋ᵢ(I), and the tax-to-payoff ratio is the same at every row instead of growing +as 1/π₋ᵢ at rare rows. Dividing by the visit probability would recover the paper's +weighting and hand rare rows enormous, noisy shoves. The paper's γ (a 1e-10 mix toward uniform) is numerically inert at this scale and is left out. """ diff --git a/tests/minidrawmaha/test_friction.py b/tests/minidrawmaha/test_friction.py index cfbbf5d0..0f011dc3 100644 --- a/tests/minidrawmaha/test_friction.py +++ b/tests/minidrawmaha/test_friction.py @@ -65,6 +65,23 @@ def test_friction_moves_regrets_once_play_leaves_the_reference(): pulled = leduc_run("lcfr", 4, mu=5.0, iterations=20) assert regrets(plain) != regrets(pulled) +def test_strong_friction_toward_a_fixed_uniform_reference_keeps_play_nearer_uniform(): + # Pins the sign and the reach plumbing end to end: with the reference held at + # uniform (no resets) a large μ must leave the current strategy closer to + # uniform than plain play does. A flipped sign or a dropped reach would push + # play away from the reference, or not at all. + def distance_to_uniform(table: dict) -> float: + return sum(np.abs(ledger.strategy() - 1.0 / ledger.n_actions).sum() for ledger in table.values()) + + plain = leduc_run("lcfr", 8, mu=None, iterations=300) + pulled = leduc_run("lcfr", 8, mu=50.0, iterations=300) + assert distance_to_uniform(pulled) < 0.8 * distance_to_uniform(plain) + +@pytest.mark.parametrize("mu", [-1.0, float("nan"), float("inf")]) +def test_a_negative_or_non_finite_mu_is_refused(mu): + with pytest.raises(ValueError, match="mu"): + Friction.new(fresh_leduc(), mu) + def test_the_adjustment_at_one_spot_by_hand(): sigma = np.array([0.5, 0.25, 0.25]) reference = np.array([0.2, 0.2, 0.6]) @@ -84,7 +101,8 @@ def test_the_reset_copies_every_rows_current_strategy(): # Equal to the last bit on most rows; `np.add.reduceat` sums a 3- or 4-wide row # in a different order than `ndarray.sum`, which moves ~2% of rows by one ulp. # A reference only sets where the pull points, so one ulp is immaterial. - for key in keys[::37]: + # every 37th row, plus every row touching the all-negative columns (the uniform branch) + for key in sorted(set(keys[::37]) | set(keys[:20]), key=keys.index): np.testing.assert_allclose(friction.of(key), table[key].strategy(), rtol=0, atol=1e-15) def test_a_dict_reset_tracks_play(): @@ -121,6 +139,10 @@ def current_exploitability(table: dict) -> float: solve = new_lockstep(0, workers=32, table=plain, deal=leduc_deal, rule="cfr+") train_lockstep(solve, 3000) + # μ = 0.05 with resets every 100 iterations: Leduc payoffs are a few chips, so + # this tax is small against them, so the bias toward each reference stays small + # while the pull still damps CFR+'s overshoot. Seed 0; + # the strict < is the claim itself: friction's current strategy beats plain's. pulled = fresh_leduc() solve = new_lockstep(0, workers=32, table=pulled, deal=leduc_deal, rule="cfr+") solve.friction = Friction.new(pulled, 0.05)