From 5f3f504fb0f8542afe8004d46268c3a5ab74f887 Mon Sep 17 00:00:00 2001 From: Yakiv Kovalskyi Date: Sun, 12 Jul 2026 18:25:00 +0200 Subject: [PATCH] =?UTF-8?q?Engine:=20add=20measurement=20harness=20?= =?UTF-8?q?=E2=80=94=20fixed-node=20bench=20+=20self-play=20match=20runner?= =?UTF-8?q?=20(closes=20#158)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit There was no way to measure engine strength/speed changes — no bench, perft, self-play, or SPRT anywhere — so tuning was unmeasurable. This adds an internal, non-shipping `EngineLab` package target (not a library product, so it is never linked into the iOS app) plus an `engine-lab` executable. Both only *call* `NegamaxEngine.search`, so the engine's determinism guarantee is untouched. - bench: searches a fixed 20-position suite (openings, tactical middlegames, endgames) under one reproducible limit and prints total nodes, nodes/sec, and a signature — a checksum over each position's best move, score, depth, and node count. Doubles as a determinism regression guard (pinned snapshot in tests) and a speed proxy. - match: engine-vs-engine self-play at fixed nodes/depth (never wall-clock), each opening played twice with colors swapped, reporting W/D/L, score %, and Elo(A - B) with a 95% error margin. Reproducibility: every observable is a pure-integer function of the fixed limits and start position — verified byte-identical across two runs and across debug/release builds. EngineLabTests (run by `swift test`, the ChessKit tests lane) guards the bench signature (stable across runs + pinned snapshot), the Elo/error-margin math, and self-play mechanics: identical engines net exactly even (color-swap fairness), a deeper config outscores a shallower one (the Elo pipeline points the right way), games are reproducible, and every game terminates. Test limits are kept small so the debug suite stays fast. Co-Authored-By: Claude Opus 4.8 --- ChessKit/Package.swift | 12 ++ ChessKit/Sources/EngineLab/Bench.swift | 137 ++++++++++++ ChessKit/Sources/EngineLab/BenchSuite.swift | 55 +++++ ChessKit/Sources/EngineLab/CLI.swift | 197 ++++++++++++++++++ ChessKit/Sources/EngineLab/Elo.swift | 45 ++++ ChessKit/Sources/EngineLab/README.md | 83 ++++++++ ChessKit/Sources/EngineLab/SelfPlay.swift | 191 +++++++++++++++++ ChessKit/Sources/engine-lab/main.swift | 6 + .../Tests/EngineLabTests/BenchTests.swift | 71 +++++++ ChessKit/Tests/EngineLabTests/EloTests.swift | 48 +++++ .../Tests/EngineLabTests/SelfPlayTests.swift | 58 ++++++ 11 files changed, 903 insertions(+) create mode 100644 ChessKit/Sources/EngineLab/Bench.swift create mode 100644 ChessKit/Sources/EngineLab/BenchSuite.swift create mode 100644 ChessKit/Sources/EngineLab/CLI.swift create mode 100644 ChessKit/Sources/EngineLab/Elo.swift create mode 100644 ChessKit/Sources/EngineLab/README.md create mode 100644 ChessKit/Sources/EngineLab/SelfPlay.swift create mode 100644 ChessKit/Sources/engine-lab/main.swift create mode 100644 ChessKit/Tests/EngineLabTests/BenchTests.swift create mode 100644 ChessKit/Tests/EngineLabTests/EloTests.swift create mode 100644 ChessKit/Tests/EngineLabTests/SelfPlayTests.swift diff --git a/ChessKit/Package.swift b/ChessKit/Package.swift index 26d78fa..ac732f4 100644 --- a/ChessKit/Package.swift +++ b/ChessKit/Package.swift @@ -13,8 +13,20 @@ let package = Package( .target(name: "ChessKit"), .target(name: "ChessProtocol", dependencies: ["ChessKit"]), .target(name: "ChessOnline"), + // Engine measurement harness (bench + self-play). Deliberately NOT a + // library `product`, so it is never linked into the iOS app; it only + // *calls* the engine, preserving its determinism guarantee. Covered by + // `swift test` via EngineLabTests, and driven ad hoc by the `engine-lab` + // executable. + .target( + name: "EngineLab", + dependencies: ["ChessProtocol", "ChessKit"], + exclude: ["README.md"] + ), + .executableTarget(name: "engine-lab", dependencies: ["EngineLab"]), .testTarget(name: "ChessKitTests", dependencies: ["ChessKit"]), .testTarget(name: "ChessProtocolTests", dependencies: ["ChessProtocol", "ChessKit"]), .testTarget(name: "ChessOnlineTests", dependencies: ["ChessOnline"]), + .testTarget(name: "EngineLabTests", dependencies: ["EngineLab", "ChessProtocol", "ChessKit"]), ] ) diff --git a/ChessKit/Sources/EngineLab/Bench.swift b/ChessKit/Sources/EngineLab/Bench.swift new file mode 100644 index 0000000..4495079 --- /dev/null +++ b/ChessKit/Sources/EngineLab/Bench.swift @@ -0,0 +1,137 @@ +import ChessKit +import ChessProtocol +import Foundation + +/// A 64-bit rolling checksum. Deterministic and order-sensitive, so it detects +/// any change in the per-position results (nodes, depth, score, best move) even +/// when — as under a hard node cap — the total node count is pinned to the +/// budget and would not move on its own. +struct Signature { + private(set) var value: UInt64 = 0xCBF2_9CE4_8422_2325 // FNV offset basis + + mutating func fold(_ v: UInt64) { + value ^= v + value = value &* 0x9E37_79B9_7F4A_7C15 // fibonacci hashing multiplier + value ^= value >> 29 + } + + mutating func fold(_ v: Int) { + fold(UInt64(bitPattern: Int64(v))) + } +} + +/// Per-position bench measurement. +public struct BenchPositionResult: Sendable { + public let name: String + public let nodes: Int + public let depth: Int + public let scoreCentipawns: Int + public let bestMove: String +} + +/// The result of a whole bench run. +public struct BenchResult: Sendable { + public let limit: SearchLimit + public let perPosition: [BenchPositionResult] + public let totalNodes: Int + public let elapsedSeconds: Double + /// Behavioral fingerprint of the run — stable across runs, moves only when + /// search behavior changes. The determinism regression guard. + public let signature: UInt64 + + public var nodesPerSecond: Double { + elapsedSeconds > 0 ? Double(totalNodes) / elapsedSeconds : 0 + } +} + +/// Fixed-workload bench: search each suite position under one reproducible +/// limit and summarize. Reproducibility comes from fixed node/depth limits (no +/// wall-clock) and no opening book, so `signature` and `totalNodes` are +/// identical on every machine and every run. +public enum Bench { + /// A fixed per-position node budget for the executable's default run. + /// Sized so the whole suite finishes in ~20s in a release build while still + /// reaching non-trivial depths — the engine is deliberately simple and only + /// searches a few hundred thousand nodes/sec. + public static let defaultNodeBudget = 200_000 + + /// A limit that caps each search at `nodes` visited (with a high depth + /// ceiling so the node budget, not depth, is the binding constraint). + public static func nodeLimit(_ nodes: Int) -> SearchLimit { + SearchLimit(depth: 64, maxNodes: nodes) + } + + public static func run( + limit: SearchLimit = Bench.nodeLimit(defaultNodeBudget), + positions: [BenchPosition] = BenchSuite.positions + ) -> BenchResult { + // A fresh, book-less engine: the deterministic configuration. + let engine = NegamaxEngine() + var perPosition: [BenchPositionResult] = [] + perPosition.reserveCapacity(positions.count) + var totalNodes = 0 + var signature = Signature() + + let clock = ContinuousClock() + let start = clock.now + for (index, position) in positions.enumerated() { + let board = parseFEN(position.fen) + let result = engine.search(board, limit: limit) + totalNodes += result.nodes + + // Fold every observable of the search into the signature, keyed by + // position index so a reordering is also detected. + signature.fold(index) + signature.fold(result.nodes) + signature.fold(result.depth) + signature.fold(result.scoreCentipawns) + signature.fold(moveCode(result.bestMove)) + + perPosition.append(BenchPositionResult( + name: position.name, + nodes: result.nodes, + depth: result.depth, + scoreCentipawns: result.scoreCentipawns, + bestMove: result.bestMove?.uci ?? "(none)" + )) + } + let elapsed = clock.now - start + + return BenchResult( + limit: limit, + perPosition: perPosition, + totalNodes: totalNodes, + elapsedSeconds: elapsed.seconds, + signature: signature.value + ) + } + + /// Encodes a move as a small integer for the signature (0 = no move). + private static func moveCode(_ move: Move?) -> Int { + guard let move else { return 0 } + let promo = move.promotion.map { $0.rawValueForSignature } ?? 0 + return 1 + move.from * 64 * 8 + move.to * 8 + promo + } +} + +private extension PieceKind { + /// A stable small index for folding a promotion choice into the signature. + var rawValueForSignature: Int { + switch self { + case .pawn: return 1 + case .knight: return 2 + case .bishop: return 3 + case .rook: return 4 + case .queen: return 5 + case .king: return 6 + } + } +} + +extension Duration { + /// Whole-plus-fractional seconds as a `Double`. + var seconds: Double { + let (secs, atto) = components + return Double(secs) + Double(atto) / 1e18 + } +} diff --git a/ChessKit/Sources/EngineLab/BenchSuite.swift b/ChessKit/Sources/EngineLab/BenchSuite.swift new file mode 100644 index 0000000..ff05f0c --- /dev/null +++ b/ChessKit/Sources/EngineLab/BenchSuite.swift @@ -0,0 +1,55 @@ +import ChessKit + +/// A named position in the bench suite. +public struct BenchPosition: Sendable { + public let name: String + public let fen: String + + public init(name: String, fen: String) { + self.name = name + self.fen = fen + } +} + +/// Parses a FEN known to be valid (the built-in suites are hardcoded). Traps on +/// a malformed constant rather than force-unwrapping, which keeps the production +/// target free of `!` and turns a typo into a clear message. +func parseFEN(_ fen: String) -> Board { + guard let board = Board(fen: fen) else { + preconditionFailure("EngineLab: invalid FEN in built-in suite: \(fen)") + } + return board +} + +/// The fixed bench suite: 20 positions spanning openings, tactical middlegames, +/// and endgames, chosen for a mix of branching factors so the total node count +/// is a broad, stable fingerprint of search behavior. Order is fixed — the +/// bench signature folds in each position's index, so reordering changes it. +public enum BenchSuite { + public static let positions: [BenchPosition] = [ + // Openings / early middlegame + .init(name: "startpos", fen: "rnbqkbnr/pppppppp/8/8/8/8/PPPPPPPP/RNBQKBNR w KQkq - 0 1"), + .init(name: "italian", fen: "r1bqk1nr/pppp1ppp/2n5/2b1p3/2B1P3/5N2/PPPP1PPP/RNBQK2R w KQkq - 4 4"), + .init(name: "ruy-lopez", fen: "r1bqkbnr/1ppp1ppp/p1n5/1B2p3/4P3/5N2/PPPP1PPP/RNBQK2R w KQkq - 0 4"), + .init(name: "najdorf", fen: "rnbqkb1r/1p2pppp/p2p1n2/8/3NP3/2N5/PPP2PPP/R1BQKB1R w KQkq - 0 6"), + .init(name: "qgd", fen: "rnbqkb1r/ppp1pppp/5n2/3p4/2PP4/8/PP2PPPP/RNBQKBNR w KQkq - 0 3"), + .init(name: "french", fen: "rnbqkb1r/ppp2ppp/4pn2/3p4/2PP4/2N5/PP2PPPP/R1BQKBNR w KQkq - 0 4"), + // Rich middlegames (high branching factor) + .init(name: "kid-middlegame", fen: "r1bq1rk1/pp2bppp/2n1pn2/3p4/2PP4/2N1PN2/PP2BPPP/R1BQ1RK1 w - - 0 8"), + .init(name: "kiwipete", fen: "r3k2r/p1ppqpb1/bn2pnp1/3PN3/1p2P3/2N2Q1p/PPPBBPPP/R3K2R w KQkq - 0 1"), + .init(name: "perft-pos4", fen: "r3k2r/Pppp1ppp/1b3nbN/nP6/BBP1P3/q4N2/Pp1P2PP/R2Q1RK1 w kq - 0 1"), + .init(name: "perft-pos5", fen: "rnbq1k1r/pp1Pbppp/2p5/8/2B5/8/PPP1NnPP/RNBQK2R w KQ - 1 8"), + .init(name: "perft-pos6", fen: "r4rk1/1pp1qppp/p1np1n2/2b1p1B1/2B1P1b1/P1NP1N2/1PP1QPPP/R4RK1 w - - 0 10"), + // Tactical (Win At Chess) test positions + .init(name: "wac-001", fen: "2rr3k/pp3pp1/1nnqbN1p/3pN3/2pP4/2P3Q1/PPB4P/R4RK1 w - - 0 1"), + .init(name: "wac-002", fen: "8/7p/5k2/5p2/p1p2P2/Pr1pPK2/1P1R3P/8 b - - 0 1"), + .init(name: "wac-010", fen: "3r1r1k/1p3pp1/p2q1n1p/8/2p1B3/P1P3Q1/1P3PPP/2K1R2R w - - 0 1"), + // Endgames (low branching, deep tactics) + .init(name: "rook-mate", fen: "6k1/5ppp/8/8/8/8/8/R3K3 w - - 0 1"), + .init(name: "kq-vs-k", fen: "8/8/8/4k3/8/8/8/3QK3 w - - 0 1"), + .init(name: "kr-vs-k", fen: "8/8/8/4k3/8/8/8/R3K3 w - - 0 1"), + .init(name: "pawn-endgame", fen: "8/5p2/5k2/8/5K2/8/5P2/8 w - - 0 1"), + .init(name: "rook-endgame", fen: "8/8/5k2/8/8/2K5/4R3/6r1 w - - 0 1"), + .init(name: "opposite-bishops", fen: "8/2k5/3b4/8/8/4B3/2K5/8 w - - 0 1"), + ] +} diff --git a/ChessKit/Sources/EngineLab/CLI.swift b/ChessKit/Sources/EngineLab/CLI.swift new file mode 100644 index 0000000..2879eb1 --- /dev/null +++ b/ChessKit/Sources/EngineLab/CLI.swift @@ -0,0 +1,197 @@ +import ChessProtocol +import Foundation + +/// Command-line front end for the measurement harness. All parsing, dispatch, +/// and formatting live here (not in the executable's `main`) so they are +/// exercised by `swift test`; the executable is a one-line shim. +public enum CLI { + public static let usage = """ + engine-lab — ChessKit-Negamax measurement harness + + USAGE: + engine-lab bench [--nodes N | --depth D] + engine-lab match [--depth-a D] [--depth-b D] [--nodes-a N] [--nodes-b N] + [--games G] [--max-plies P] + + bench Search the fixed 20-position suite under one reproducible limit and + print total nodes, nodes/sec, and a behavioral signature. Default + limit: a \(Bench.defaultNodeBudget)-node budget per position. + + match Play a self-play match (each opening twice, colors swapped) and + report W/D/L, score %, and Elo(A - B) with a 95% margin. Configs A + and B default to depth 4. All limits are fixed nodes/depth, so runs + are reproducible. + """ + + /// Entry point. `arguments` is the full `CommandLine.arguments` (with the + /// program name at index 0). Returns a process exit code. + public static func run(_ arguments: [String]) -> Int32 { + var args = Array(arguments.dropFirst()) + guard let subcommand = args.first else { + print(usage) + return 2 + } + args.removeFirst() + + switch subcommand { + case "bench": + return runBench(args) + case "match": + return runMatch(args) + case "-h", "--help", "help": + print(usage) + return 0 + default: + print("engine-lab: unknown subcommand '\(subcommand)'\n") + print(usage) + return 2 + } + } + + // MARK: - bench + + static func runBench(_ args: [String]) -> Int32 { + let options = parseOptions(args) + let depth = options["depth"].flatMap(Int.init) + let nodes = options["nodes"].flatMap(Int.init) + + let limit: SearchLimit + switch (depth, nodes) { + case (let d?, let n?): + limit = SearchLimit(depth: d, maxNodes: n) + case (let d?, nil): + limit = SearchLimit(depth: d) + case (nil, let n?): + limit = Bench.nodeLimit(n) + case (nil, nil): + limit = Bench.nodeLimit(Bench.defaultNodeBudget) + } + + let result = Bench.run(limit: limit) + print(format(result)) + return 0 + } + + static func format(_ result: BenchResult) -> String { + var lines: [String] = [] + lines.append("ChessKit-Negamax bench") + lines.append("limit: \(describe(result.limit))") + lines.append("") + for row in result.perPosition { + let name = row.name.padding(toLength: 18, withPad: " ", startingAt: 0) + let score = signedString(row.scoreCentipawns) + lines.append( + " \(name) depth \(pad(row.depth, 3)) " + + "score \(pad(score, 8)) nodes \(pad(row.nodes, 10)) \(row.bestMove)" + ) + } + lines.append("") + lines.append( + "positions: \(result.perPosition.count) " + + "total nodes: \(result.totalNodes) " + + "time: \(seconds(result.elapsedSeconds)) " + + "speed: \(mnps(result.nodesPerSecond))" + ) + lines.append("signature: \(hex(result.signature))") + return lines.joined(separator: "\n") + } + + // MARK: - match + + static func runMatch(_ args: [String]) -> Int32 { + let options = parseOptions(args) + let a = config(depthKey: "depth-a", nodesKey: "nodes-a", defaultDepth: 4, options: options) + let b = config(depthKey: "depth-b", nodesKey: "nodes-b", defaultDepth: 4, options: options) + let maxPlies = options["max-plies"].flatMap(Int.init) ?? SelfPlay.defaultMaxPlies + + // `--games G` caps how many openings are used (each played twice). + var openings = Openings.standard + if let games = options["games"].flatMap(Int.init), games > 0 { + let pairs = max(1, (games + 1) / 2) + openings = Array(openings.prefix(pairs)) + } + + let result = SelfPlay.playMatch(a: a, b: b, openings: openings, maxPlies: maxPlies) + print(format(result, openings: openings.count)) + return 0 + } + + static func config( + depthKey: String, nodesKey: String, defaultDepth: Int, options: [String: String] + ) -> EngineConfig { + if let nodes = options[nodesKey].flatMap(Int.init) { + return EngineConfig(label: "nodes-\(nodes)", limit: Bench.nodeLimit(nodes)) + } + let depth = options[depthKey].flatMap(Int.init) ?? defaultDepth + return EngineConfig(label: "depth-\(depth)", limit: SearchLimit(depth: depth)) + } + + static func format(_ result: MatchResult, openings: Int) -> String { + var lines: [String] = [] + lines.append("Self-play match") + lines.append("A: \(result.aLabel) B: \(result.bLabel)") + lines.append("games: \(result.games) (\(openings) openings × 2 colors)") + lines.append( + "A results: +\(result.wins) =\(result.draws) -\(result.losses) " + + "score: \(percent(result.scoreA))" + ) + lines.append( + "Elo(A - B): \(signedString(Int(result.eloDelta.rounded()))) " + + "± \(Int(result.eloMargin.rounded())) (95%)" + ) + return lines.joined(separator: "\n") + } + + // MARK: - Option parsing & formatting helpers + + /// Reads `--key value` pairs into a dictionary (keys without the dashes). + static func parseOptions(_ args: [String]) -> [String: String] { + var options: [String: String] = [:] + var index = 0 + while index < args.count { + let token = args[index] + if token.hasPrefix("--"), index + 1 < args.count { + options[String(token.dropFirst(2))] = args[index + 1] + index += 2 + } else { + index += 1 + } + } + return options + } + + static func describe(_ limit: SearchLimit) -> String { + if let nodes = limit.maxNodes { + return "nodes<=\(nodes) (depth ceiling \(limit.depth))" + } + return "depth \(limit.depth)" + } + + static func signedString(_ value: Int) -> String { + value >= 0 ? "+\(value)" : "\(value)" + } + + static func pad(_ value: Int, _ width: Int) -> String { + pad("\(value)", width) + } + + static func pad(_ text: String, _ width: Int) -> String { + text.count >= width ? text : String(repeating: " ", count: width - text.count) + text + } + + static func hex(_ value: UInt64) -> String { + "0x" + String(value, radix: 16) + } + + static func seconds(_ value: Double) -> String { + String(format: "%.2fs", value) + } + + static func mnps(_ nodesPerSecond: Double) -> String { + String(format: "%.2f Mnps", nodesPerSecond / 1_000_000) + } + + static func percent(_ fraction: Double) -> String { + String(format: "%.1f%%", fraction * 100) + } +} diff --git a/ChessKit/Sources/EngineLab/Elo.swift b/ChessKit/Sources/EngineLab/Elo.swift new file mode 100644 index 0000000..768334c --- /dev/null +++ b/ChessKit/Sources/EngineLab/Elo.swift @@ -0,0 +1,45 @@ +import Foundation + +/// Elo arithmetic for match results. +/// +/// The logistic (Elo) model maps a rating difference to an expected score: +/// `E = 1 / (1 + 10^(-Δ/400))`. Inverting it turns an observed score fraction +/// back into the rating difference it implies. The error margin comes from the +/// binomial/trinomial spread of the actual W/D/L sample, so it is exact for the +/// (fixed, reproducible) games that were played. +public enum Elo { + /// 95% two-sided normal quantile. + static let z95 = 1.959963984540054 + + /// Score fractions are clamped this far from 0 and 1 before inversion so a + /// clean sweep reports a large-but-finite rating gap instead of infinity. + static let scoreEpsilon = 0.5 / 1000.0 + + /// The rating difference implied by an expected score in (0, 1). Positive + /// when `score > 0.5`. + public static func difference(forScore score: Double) -> Double { + let s = min(1 - scoreEpsilon, max(scoreEpsilon, score)) + return -400 * log10(1 / s - 1) + } + + /// 95% confidence half-width (in Elo) around the point estimate, derived + /// from the standard error of the observed score. + public static func errorMargin95(wins: Int, draws: Int, losses: Int) -> Double { + let n = wins + draws + losses + guard n > 0 else { return 0 } + let total = Double(n) + let score = (Double(wins) + 0.5 * Double(draws)) / total + + // Sample variance of a single game's points (1 / 0.5 / 0), then the + // standard error of the mean over n games. + let variance = + (Double(wins) * pow(1 - score, 2) + + Double(draws) * pow(0.5 - score, 2) + + Double(losses) * pow(0 - score, 2)) / total + let standardError = (variance / total).squareRoot() + + let low = difference(forScore: score - z95 * standardError) + let high = difference(forScore: score + z95 * standardError) + return (high - low) / 2 + } +} diff --git a/ChessKit/Sources/EngineLab/README.md b/ChessKit/Sources/EngineLab/README.md new file mode 100644 index 0000000..a3696a9 --- /dev/null +++ b/ChessKit/Sources/EngineLab/README.md @@ -0,0 +1,83 @@ +# EngineLab — engine measurement harness + +Tooling to measure strength and speed changes in `ChessKit-Negamax` +(`ChessProtocol/NegamaxEngine.swift`). It is the prerequisite for any engine +tuning: without it, an "improvement" can't be told from a regression. + +EngineLab is an **internal, non-shipping** package target — it is not exported +as a library product, so it is never linked into the iOS app. It only *calls* +the engine's `search`, so the engine's determinism guarantee is untouched. + +## Tools + +Everything runs through the `engine-lab` executable (all logic lives in the +`EngineLab` target and is unit-tested; the executable is a one-line shim): + +``` +swift run -c release engine-lab bench [--nodes N | --depth D] +swift run -c release engine-lab match [--depth-a D] [--depth-b D] + [--nodes-a N] [--nodes-b N] + [--games G] [--max-plies P] +``` + +Use `-c release` — the search is only a few hundred thousand nodes/sec, so a +debug build is ~40× slower. + +### `bench` + +Searches a fixed 20-position suite (openings, tactical middlegames, endgames) +under one reproducible limit and prints total nodes, nodes/sec (a speed proxy), +and a **signature** — a checksum over every position's best move, score, depth, +and node count. + +``` +$ swift run -c release engine-lab bench --nodes 2000 +ChessKit-Negamax bench +limit: nodes<=2000 (depth ceiling 64) + + startpos depth 4 score +0 nodes 2000 b1c3 + ... +positions: 20 total nodes: 42798 time: 0.33s speed: 0.13 Mnps +signature: 0xcd7fa918c21eafc2 +``` + +The signature is a **determinism regression guard**: it is byte-identical on +every machine and every run (engine evaluation is pure integer), and moves only +when search behavior changes. It is pinned in `EngineLabTests`; a behavior +change trips CI, and the fix is to update the pinned value in the same PR — that +diff is the review signal. With a fixed `--depth`, the *total node count* is +itself the fingerprint. + +### `match` + +Plays an engine-vs-engine self-play match: every opening in a small balanced set +is played twice, colors swapped, so color bias cancels. Reports W/D/L, score %, +and Elo(A − B) with a 95% error margin. + +``` +$ swift run -c release engine-lab match --depth-a 3 --depth-b 1 +Self-play match +A: depth-3 B: depth-1 +games: 24 (12 openings × 2 colors) +A results: +15 =9 -0 score: 81.2% +Elo(A - B): +255 ± 120 (95%) +``` + +Limits are fixed **nodes or depth** — never wall-clock — so a match is fully +reproducible. Two configs that differ only in depth (or node budget) measure how +much that extra search is worth in Elo. To compare two *versions* of the engine, +build the tool on each commit and run the same match; the fixed limits make the +numbers comparable. + +Leave books off (the default) to keep games deterministic — a book's random move +choice would make runs non-reproducible. + +## What CI guards + +`swift test` (the `ChessKit tests` lane) runs `EngineLabTests`: + +- the bench signature is stable across two runs and matches its pinned snapshot; +- Elo conversion and error-margin math; +- self-play mechanics: identical engines net exactly even (color-swap fairness), + a deeper config outscores a shallower one (the Elo pipeline points the right + way), games are reproducible, and every game terminates. diff --git a/ChessKit/Sources/EngineLab/SelfPlay.swift b/ChessKit/Sources/EngineLab/SelfPlay.swift new file mode 100644 index 0000000..a67adcf --- /dev/null +++ b/ChessKit/Sources/EngineLab/SelfPlay.swift @@ -0,0 +1,191 @@ +import ChessKit +import ChessProtocol + +/// A named opening position for self-play. Games start here rather than from +/// the initial position so a deterministic engine produces a spread of games +/// instead of replaying one line. +public struct Opening: Sendable { + public let name: String + public let fen: String + + public init(name: String, fen: String) { + self.name = name + self.fen = fen + } +} + +/// A small, balanced opening set. Each opening is played twice in a match (once +/// with each engine as White), so a match is `2 * openings.count` games. +public enum Openings { + public static let standard: [Opening] = [ + .init(name: "open-game", fen: "rnbqkbnr/pppp1ppp/8/4p3/4P3/8/PPPP1PPP/RNBQKBNR w KQkq - 0 2"), + .init(name: "sicilian", fen: "rnbqkbnr/pp1ppppp/8/2p5/4P3/8/PPPP1PPP/RNBQKBNR w KQkq - 0 2"), + .init(name: "french", fen: "rnbqkbnr/pppp1ppp/4p3/8/4P3/8/PPPP1PPP/RNBQKBNR w KQkq - 0 2"), + .init(name: "caro-kann", fen: "rnbqkbnr/pp1ppppp/2p5/8/4P3/8/PPPP1PPP/RNBQKBNR w KQkq - 0 2"), + .init(name: "scandinavian", fen: "rnbqkbnr/ppp1pppp/8/3p4/4P3/8/PPPP1PPP/RNBQKBNR w KQkq - 0 2"), + .init(name: "closed-d4", fen: "rnbqkbnr/ppp1pppp/8/3p4/3P4/8/PPP1PPPP/RNBQKBNR w KQkq - 0 2"), + .init(name: "indian", fen: "rnbqkb1r/pppppppp/5n2/8/3P4/8/PPP1PPPP/RNBQKBNR w KQkq - 1 2"), + .init(name: "qgd", fen: "rnbqkb1r/pppp1ppp/4pn2/8/2PP4/8/PP2PPPP/RNBQKBNR w KQkq - 0 3"), + .init(name: "kings-indian", fen: "rnbqkb1r/pppppp1p/5np1/8/2PP4/8/PP2PPPP/RNBQKBNR w KQkq - 0 3"), + .init(name: "english", fen: "rnbqkbnr/pppppppp/8/8/2P5/8/PP1PPPPP/RNBQKBNR b KQkq - 0 1"), + .init(name: "reti", fen: "rnbqkbnr/ppp1pppp/8/3p4/8/5N2/PPPPPPPP/RNBQKB1R w KQkq - 0 2"), + .init(name: "pirc", fen: "rnbqkb1r/ppp1pppp/3p1n2/8/3PP3/8/PPP2PPP/RNBQKBNR w KQkq - 0 3"), + ] +} + +/// One engine participant in a match: a label plus the search limit (and +/// optional book) it plays under. Two configs differing only in `limit` (depth +/// or node budget) let a match measure how much a given amount of extra search +/// is worth in Elo. Leave `book` nil to keep games reproducible. +public struct EngineConfig: Sendable { + public let label: String + public let limit: SearchLimit + public let book: OpeningBook? + + public init(label: String, limit: SearchLimit, book: OpeningBook? = nil) { + self.label = label + self.limit = limit + self.book = book + } + + func makeEngine() -> NegamaxEngine { + NegamaxEngine(book: book) + } +} + +public enum GameResult: Sendable { + case whiteWin + case blackWin + case draw +} + +public enum GameEndReason: String, Sendable { + case checkmate + case stalemate + case fiftyMove + case insufficientMaterial + case threefold + case plyCap +} + +public struct GameOutcome: Sendable { + public let result: GameResult + public let reason: GameEndReason + public let plies: Int +} + +/// Match result from engine A's perspective (`wins`/`losses` are A's). +public struct MatchResult: Sendable { + public let aLabel: String + public let bLabel: String + public let wins: Int + public let draws: Int + public let losses: Int + + public var games: Int { wins + draws + losses } + + /// A's score fraction: (wins + ½·draws) / games. + public var scoreA: Double { + guard games > 0 else { return 0 } + return (Double(wins) + 0.5 * Double(draws)) / Double(games) + } + + /// Elo of A relative to B (positive ⇒ A is stronger over these games). + public var eloDelta: Double { Elo.difference(forScore: scoreA) } + + /// 95% confidence half-width around `eloDelta`. + public var eloMargin: Double { Elo.errorMargin95(wins: wins, draws: draws, losses: losses) } +} + +/// Deterministic engine-vs-engine play. Every game is fully determined by the +/// two configs, the start position, and the ply cap — no clocks, no randomness +/// (as long as neither config carries a book), so a match is reproducible. +public enum SelfPlay { + /// Absolute cap on game length; a game hitting it is scored as a draw. Well + /// above the 50-move rule, so it only fires on genuine non-progress the + /// draw rules somehow miss. + public static let defaultMaxPlies = 400 + + /// Plays a single game. `white`/`black` are the configs by color. + public static func playGame( + white: EngineConfig, + black: EngineConfig, + from start: Board, + maxPlies: Int = defaultMaxPlies + ) -> GameOutcome { + let whiteEngine = white.makeEngine() + let blackEngine = black.makeEngine() + var board = start + + // Track positions for threefold repetition. The invariant across the + // loop is that the current board's key has already been counted. + var seen: [String: Int] = [:] + seen[board.repetitionKey, default: 0] += 1 + + for ply in 0..= 3 { + return GameOutcome(result: .draw, reason: .threefold, plies: ply) + } + + let config = board.sideToMove == .white ? white : black + let engine = board.sideToMove == .white ? whiteEngine : blackEngine + let search = engine.search(board, limit: config.limit) + guard let move = search.bestMove, let next = board.making(move) else { + // `status` was `.ongoing`, so a legal move exists; reaching here + // would mean the engine returned nothing. Adjudicate a draw + // rather than trap — the harness must always finish a game. + return GameOutcome(result: .draw, reason: .stalemate, plies: ply) + } + board = next + seen[board.repetitionKey, default: 0] += 1 + } + return GameOutcome(result: .draw, reason: .plyCap, plies: maxPlies) + } + + /// Plays a full match: every opening twice, colors swapped, so color bias + /// cancels. Returns results from A's perspective. + public static func playMatch( + a: EngineConfig, + b: EngineConfig, + openings: [Opening] = Openings.standard, + maxPlies: Int = defaultMaxPlies + ) -> MatchResult { + var wins = 0 + var draws = 0 + var losses = 0 + + func tally(_ outcome: GameOutcome, aIsWhite: Bool) { + switch outcome.result { + case .draw: + draws += 1 + case .whiteWin: + if aIsWhite { wins += 1 } else { losses += 1 } + case .blackWin: + if aIsWhite { losses += 1 } else { wins += 1 } + } + } + + for opening in openings { + let start = parseFEN(opening.fen) + tally(playGame(white: a, black: b, from: start, maxPlies: maxPlies), aIsWhite: true) + tally(playGame(white: b, black: a, from: start, maxPlies: maxPlies), aIsWhite: false) + } + + return MatchResult(aLabel: a.label, bLabel: b.label, wins: wins, draws: draws, losses: losses) + } +} diff --git a/ChessKit/Sources/engine-lab/main.swift b/ChessKit/Sources/engine-lab/main.swift new file mode 100644 index 0000000..024c6ba --- /dev/null +++ b/ChessKit/Sources/engine-lab/main.swift @@ -0,0 +1,6 @@ +import EngineLab +import Foundation + +// Thin shim: every bit of real logic lives in `EngineLab.CLI` (and is unit +// tested there); this just wires arguments to it and forwards the exit code. +exit(CLI.run(CommandLine.arguments)) diff --git a/ChessKit/Tests/EngineLabTests/BenchTests.swift b/ChessKit/Tests/EngineLabTests/BenchTests.swift new file mode 100644 index 0000000..a558357 --- /dev/null +++ b/ChessKit/Tests/EngineLabTests/BenchTests.swift @@ -0,0 +1,71 @@ +import ChessKit +import ChessProtocol +import XCTest +@testable import EngineLab + +final class BenchTests: XCTestCase { + /// The CI node budget. Small so `swift test` stays fast (the engine only + /// searches a few thousand nodes/sec in a debug build), but large enough + /// that most positions reach several plies, exercising real search. + static let ciNodeBudget = 2000 + + /// The signature the suite produces at `ciNodeBudget`. Pinned so any change + /// in search behavior trips CI; update it in the same PR that changes the + /// engine — that diff is the review signal. Engine evaluation is pure + /// integer, so the value is byte-identical across machines and across + /// debug/release builds (verified). + static let signatureAtCIBudget: UInt64 = 0xcd7f_a918_c21e_afc2 + + /// Every suite FEN must parse and be a real, non-terminal position — a + /// typo'd or already-checkmated FEN would silently search nothing. + func testSuitePositionsAreValidAndPlayable() { + XCTAssertEqual(BenchSuite.positions.count, 20) + for position in BenchSuite.positions { + let board = Board(fen: position.fen) + XCTAssertNotNil(board, "unparseable FEN for \(position.name): \(position.fen)") + XCTAssertFalse( + board?.legalMoves().isEmpty ?? true, + "\(position.name) has no legal moves" + ) + } + } + + /// The determinism regression guard, in one shot: two independent runs must + /// agree with each other on every observable, and the signature must match + /// the pinned snapshot. If the two runs disagree, the engine stopped being + /// deterministic; if they agree but differ from the snapshot, search + /// behavior changed. + func testBenchIsDeterministicAndMatchesSnapshot() { + let limit = Bench.nodeLimit(Self.ciNodeBudget) + let first = Bench.run(limit: limit) + let second = Bench.run(limit: limit) + + XCTAssertEqual(first.totalNodes, second.totalNodes) + XCTAssertEqual(first.perPosition.count, second.perPosition.count) + for (a, b) in zip(first.perPosition, second.perPosition) { + XCTAssertEqual(a.nodes, b.nodes, "\(a.name) nodes") + XCTAssertEqual(a.depth, b.depth, "\(a.name) depth") + XCTAssertEqual(a.scoreCentipawns, b.scoreCentipawns, "\(a.name) score") + XCTAssertEqual(a.bestMove, b.bestMove, "\(a.name) bestmove") + } + + XCTAssertEqual(first.signature, second.signature) + XCTAssertEqual( + first.signature, Self.signatureAtCIBudget, + "bench signature changed — if this is an intentional engine change, update " + + "signatureAtCIBudget to 0x\(String(first.signature, radix: 16))" + ) + } + + /// Fixed-depth bench is deterministic too, and there the *total node count* + /// is itself the behavioral fingerprint (not pinned to a budget). Depth 2 + /// over a subset keeps this cheap. + func testFixedDepthBenchIsStable() { + let subset = Array(BenchSuite.positions.prefix(8)) + let a = Bench.run(limit: SearchLimit(depth: 2), positions: subset) + let b = Bench.run(limit: SearchLimit(depth: 2), positions: subset) + XCTAssertEqual(a.totalNodes, b.totalNodes) + XCTAssertEqual(a.signature, b.signature) + XCTAssertGreaterThan(a.totalNodes, 0) + } +} diff --git a/ChessKit/Tests/EngineLabTests/EloTests.swift b/ChessKit/Tests/EngineLabTests/EloTests.swift new file mode 100644 index 0000000..5ff0cca --- /dev/null +++ b/ChessKit/Tests/EngineLabTests/EloTests.swift @@ -0,0 +1,48 @@ +import XCTest +@testable import EngineLab + +final class EloTests: XCTestCase { + func testEvenScoreIsZeroElo() { + XCTAssertEqual(Elo.difference(forScore: 0.5), 0, accuracy: 1e-6) + } + + func testKnownScoreConversions() { + // -400·log10(1/0.75 − 1) ≈ 190.85; symmetric below 0.5. + XCTAssertEqual(Elo.difference(forScore: 0.75), 190.85, accuracy: 0.5) + XCTAssertEqual(Elo.difference(forScore: 0.25), -190.85, accuracy: 0.5) + } + + func testMonotonicInScore() { + XCTAssertLessThan(Elo.difference(forScore: 0.4), Elo.difference(forScore: 0.6)) + XCTAssertLessThan(Elo.difference(forScore: 0.6), Elo.difference(forScore: 0.9)) + } + + /// A clean sweep must not blow up to infinity — the score is clamped so the + /// gap is large but finite. + func testSweepsAreFinite() { + XCTAssertTrue(Elo.difference(forScore: 1.0).isFinite) + XCTAssertTrue(Elo.difference(forScore: 0.0).isFinite) + XCTAssertGreaterThan(Elo.difference(forScore: 1.0), 400) + XCTAssertLessThan(Elo.difference(forScore: 0.0), -400) + } + + func testAllDrawsHaveZeroMargin() { + // No variance in the outcomes ⇒ no uncertainty in the score. + XCTAssertEqual(Elo.errorMargin95(wins: 0, draws: 20, losses: 0), 0, accuracy: 1e-9) + } + + func testDecisiveResultsHavePositiveMargin() { + XCTAssertGreaterThan(Elo.errorMargin95(wins: 10, draws: 0, losses: 10), 0) + } + + /// More games at the same score ratio tighten the confidence interval. + func testMarginShrinksWithMoreGames() { + let few = Elo.errorMargin95(wins: 6, draws: 0, losses: 4) + let many = Elo.errorMargin95(wins: 60, draws: 0, losses: 40) + XCTAssertLessThan(many, few) + } + + func testEmptyMatchIsZero() { + XCTAssertEqual(Elo.errorMargin95(wins: 0, draws: 0, losses: 0), 0) + } +} diff --git a/ChessKit/Tests/EngineLabTests/SelfPlayTests.swift b/ChessKit/Tests/EngineLabTests/SelfPlayTests.swift new file mode 100644 index 0000000..e4528a5 --- /dev/null +++ b/ChessKit/Tests/EngineLabTests/SelfPlayTests.swift @@ -0,0 +1,58 @@ +import ChessKit +import ChessProtocol +import XCTest +@testable import EngineLab + +final class SelfPlayTests: XCTestCase { + /// Two identical engines, colors swapped per opening, must net to exactly + /// even: whatever happens with A as White happens identically with B as + /// White, so wins and losses cancel. Checks both the fairness of the + /// color-swap pairing and whole-game determinism. Depth 1 keeps it cheap. + func testNullMatchIsExactlyBalanced() { + let config = EngineConfig(label: "depth-1", limit: SearchLimit(depth: 1)) + let openings = Array(Openings.standard.prefix(2)) + let result = SelfPlay.playMatch(a: config, b: config, openings: openings, maxPlies: 80) + + XCTAssertEqual(result.games, openings.count * 2) + XCTAssertEqual(result.wins, result.losses, "identical engines must net even") + XCTAssertEqual(result.scoreA, 0.5, accuracy: 1e-9) + XCTAssertEqual(result.eloDelta, 0, accuracy: 1e-6) + } + + /// The Elo pipeline points the right way: a deeper-searching engine outscores + /// a shallower one, giving a positive Elo delta. A robust property (not a + /// pinned count), so ordinary engine tweaks don't make it noisy. + func testDeeperEngineOutscoresShallower() { + let deep = EngineConfig(label: "depth-2", limit: SearchLimit(depth: 2)) + let shallow = EngineConfig(label: "depth-1", limit: SearchLimit(depth: 1)) + let openings = Array(Openings.standard.prefix(2)) + + let result = SelfPlay.playMatch(a: deep, b: shallow, openings: openings, maxPlies: 80) + XCTAssertGreaterThan(result.scoreA, 0.5, "deeper search should score higher") + XCTAssertGreaterThan(result.eloDelta, 0) + XCTAssertGreaterThanOrEqual(result.eloMargin, 0) + } + + /// A game is reproducible: replaying it yields the same result, reason, and + /// length. Combined with the bench determinism guard, this establishes that + /// a whole match is reproducible. + func testGameIsReproducible() { + let config = EngineConfig(label: "depth-1", limit: SearchLimit(depth: 1)) + let first = SelfPlay.playGame(white: config, black: config, from: Board(), maxPlies: 60) + let second = SelfPlay.playGame(white: config, black: config, from: Board(), maxPlies: 60) + XCTAssertEqual(first.result, second.result) + XCTAssertEqual(first.reason, second.reason) + XCTAssertEqual(first.plies, second.plies) + } + + /// Every game terminates with a definite outcome; the ply cap guarantees it + /// even if the draw rules somehow don't fire first. + func testGamesAlwaysTerminate() { + let config = EngineConfig(label: "depth-2", limit: SearchLimit(depth: 2)) + let outcome = SelfPlay.playGame( + white: config, black: config, + from: Board(), maxPlies: 30 + ) + XCTAssertLessThanOrEqual(outcome.plies, 30) + } +}