From ff2c2ad7665265e35a3d6a5c98ca4113c2a7761c Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:00:34 +0200 Subject: [PATCH 01/23] bench: simfps takes a device set, a common start time and writes per-frame stamps; GlassBench -static and an input marker (not compiled yet: no Swift on the authoring host) --- src/bench/GlassBench/main.swift | 45 ++++++++++++++++++++++++++++++--- src/bench/build.sh | 4 +++ src/bench/simfps.swift | 40 +++++++++++++++++++++++++++-- 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/src/bench/GlassBench/main.swift b/src/bench/GlassBench/main.swift index 95612da..f8b1bf3 100644 --- a/src/bench/GlassBench/main.swift +++ b/src/bench/GlassBench/main.swift @@ -2,7 +2,13 @@ // A table of cards scrolls by itself at a fixed speed (time-based, so a dropped // frame skips content rather than slowing the scroll) under a glass navigation bar, // a glass tab bar and a floating glass toolbar with a glass button that keeps pulsing. -// Launch arguments: -noglass (no floating glass, opaque bars), -speed , -nostrip. +// Launch arguments: -noglass (no floating glass, opaque bars), -speed , -nostrip, +// -static (nothing animates: a fixed picture for pixel-exact checks and for a simulator that sits idle), +// -probe (an input marker, see below). +// +// The input marker (-probe): a 30 pt square at (21, 160) pt that flips between white and black on every +// tap and on every glassbench://toggle URL (`simctl openurl`). The bench times from the injection to the +// first surface whose marker flipped (bench/README.md, "input proxy"). // // The frame strip: every display-link tick draws a 16-bit counter as blocks at the top left, so a // frame can be told apart from the one before it all the way to a viewer (simfps --strip at the @@ -14,6 +20,34 @@ import UIKit let launchArgs = ProcessInfo.processInfo.arguments let noGlass = launchArgs.contains("-noglass") let showStrip = !launchArgs.contains("-nostrip") +let isStatic = launchArgs.contains("-static") +let showProbe = launchArgs.contains("-probe") + +/// The input marker: white until toggled, then black, then white again. It follows the system appearance +/// until the first toggle, so `simctl ui appearance` flips it too (the bench's fallback injection). +final class ProbeMarker: UIView { + private var dark = false + init() { + super.init(frame: CGRect(x: 21, y: 160, width: 30, height: 30)) + backgroundColor = .systemBackground + layer.borderWidth = 1; layer.borderColor = UIColor.gray.cgColor + isUserInteractionEnabled = false + } + required init?(coder: NSCoder) { fatalError() } + func toggle() { + dark.toggle() + overrideUserInterfaceStyle = dark ? .dark : .light + } +} +var probeMarker: ProbeMarker? + +/// Any touch on the window flips the marker (the bench's tap injection). +final class BenchWindow: UIWindow { + override func sendEvent(_ event: UIEvent) { + if event.type == .touches, event.allTouches?.contains(where: { $0.phase == .began }) == true { probeMarker?.toggle() } + super.sendEvent(event) + } +} final class FrameStrip: UIView { static let origin = CGPoint(x: 21, y: 120), block: CGFloat = 20 @@ -155,7 +189,7 @@ final class BenchViewController: UITableViewController { override func viewDidAppear(_ animated: Bool) { super.viewDidAppear(animated) if !noGlass, let host = navigationController?.view, pulse.superview == nil { addFloatingGlass(to: host) } - guard link == nil else { return } // viewDidAppear runs again on every return to the app + guard link == nil, !isStatic else { return } // viewDidAppear runs again on every return to the app link = CADisplayLink(target: self, selector: #selector(step)) link?.preferredFrameRateRange = CAFrameRateRange(minimum: 60, maximum: 60, preferred: 60) startTime = CACurrentMediaTime() @@ -243,8 +277,13 @@ final class SceneDelegate: UIResponder, UIWindowSceneDelegate { other.tabBarItem = UITabBarItem(title: "Other", image: UIImage(systemName: "square.grid.2x2"), tag: 1) let tabs = UITabBarController(); tabs.viewControllers = [nav, other] if noGlass { tabs.tabBar.isHidden = true } - let w = UIWindow(windowScene: ws); w.rootViewController = tabs; w.makeKeyAndVisible() + let w = BenchWindow(windowScene: ws); w.rootViewController = tabs; w.makeKeyAndVisible() if showStrip { let f = FrameStrip(); w.addSubview(f); frameStrip = f } + if showProbe { let m = ProbeMarker(); w.addSubview(m); probeMarker = m } window = w } + + func scene(_ scene: UIScene, openURLContexts contexts: Set) { + if contexts.contains(where: { $0.url.scheme == "glassbench" }) { probeMarker?.toggle() } + } } diff --git a/src/bench/build.sh b/src/bench/build.sh index b1279c3..25898a6 100755 --- a/src/bench/build.sh +++ b/src/bench/build.sh @@ -25,6 +25,10 @@ cat > "$APP/Info.plist" <<'PLIST' MinimumOSVersion26.0 UIDeviceFamily1 UILaunchScreen + CFBundleURLTypes + CFBundleURLNamedev.macbox.glassbench + CFBundleURLSchemesglassbench + UIApplicationSceneManifest UIApplicationSupportsMultipleScenes diff --git a/src/bench/simfps.swift b/src/bench/simfps.swift index 58b1f7c..dc50aa6 100644 --- a/src/bench/simfps.swift +++ b/src/bench/simfps.swift @@ -6,6 +6,13 @@ // // simfps [label] [--strip] prints one JSON line // +// Environment (all optional; used by `macbox bench`): +// SIMFPS_SET= a simulator device set of its own, instead of the default one +// SIMFPS_START_AT= attach first, then wait for this wall-clock second to start counting, so that +// several simfps (one per simulator) measure the same window +// SIMFPS_PROBE=, a point in points (402 pt wide screen): its luma is read from every new surface +// SIMFPS_STAMPS= one line per new surface: " " +// // --strip also reads GlassBench's frame strip (a 16-bit counter the app draws every tick, see // GlassBench/main.swift) from each new surface: how many app frames reached the screen, how many were // lost on the way (gaps in the counter) and how many surface frames showed no new app frame (repeats). @@ -39,7 +46,7 @@ for p in ["/Library/Developer/PrivateFrameworks/CoreSimulator.framework/CoreSimu guard let ctxClass = NSClassFromString("SimServiceContext") as? NSObject.Type, let ctx = ctxClass.perform(NSSelectorFromString("sharedServiceContextForDeveloperDir:error:"), with: dev, with: nil)? .takeUnretainedValue() as? NSObject, - let set = ctx.perform(NSSelectorFromString("defaultDeviceSetWithError:"), with: nil)?.takeUnretainedValue() as? NSObject, + let set = (ProcessInfo.processInfo.environment["SIMFPS_SET"].map { ctx.perform(NSSelectorFromString("deviceSetWithPath:error:"), with: $0, with: nil) } ?? ctx.perform(NSSelectorFromString("defaultDeviceSetWithError:"), with: nil))?.takeUnretainedValue() as? NSObject, let byUDID = set.value(forKey: "devicesByUDID") as? [NSUUID: NSObject], let device = byUDID[NSUUID(uuidString: udid)!] else { die("simulator \(udid) not found") } @@ -96,6 +103,24 @@ var tb = mach_timebase_info_data_t(); mach_timebase_info(&tb) func nowNs() -> UInt64 { mach_absolute_time() * UInt64(tb.numer) / UInt64(tb.denom) } var stamps: [UInt64] = [] var strip: [(UInt64, Int)] = [] // (time, counter or -1 when unreadable) +let env = ProcessInfo.processInfo.environment +let rawPath = env["SIMFPS_STAMPS"] +let probe: (Double, Double)? = env["SIMFPS_PROBE"].flatMap { s -> (Double, Double)? in + let p = s.split(separator: ",").compactMap { Double($0) } + return p.count == 2 ? (p[0], p[1]) : nil +} +var raw: [(UInt64, Int, Int)] = [] // (time, counter or -1, probe luma or -1) of every new surface +/// The luma (0-255) at a point given in points of a 402 pt wide screen, or -1. +func readLuma(_ s: IOSurface, _ pt: (Double, Double)) -> Int { + let w = IOSurfaceGetWidth(s), h = IOSurfaceGetHeight(s), bpr = IOSurfaceGetBytesPerRow(s) + let scale = Double(w) / 402.0 + guard IOSurfaceLock(s, .readOnly, nil) == kIOReturnSuccess else { return -1 } + defer { IOSurfaceUnlock(s, .readOnly, nil) } + let x = Int(pt.0 * scale), y = Int(pt.1 * scale) + guard x < w, y < h else { return -1 } + let p = IOSurfaceGetBaseAddress(s).assumingMemoryBound(to: UInt8.self) + y * bpr + x * 4 + return (Int(p[0]) + Int(p[1]) + Int(p[2])) / 3 +} /// GlassBench's counter from a BGRA surface: 18 blocks of 20 pt from (21, 120) pt, at the surface's scale. func readCounter(_ s: IOSurface) -> Int { let w = IOSurfaceGetWidth(s), h = IOSurfaceGetHeight(s), bpr = IOSurfaceGetBytesPerRow(s) @@ -118,6 +143,11 @@ func readCounter(_ s: IOSurface) -> Int { } stamps.reserveCapacity(Int(secs * 200)) var polls = 0 +// SIMFPS_START_AT: attached by now; count from that wall-clock second on. +if let at = env["SIMFPS_START_AT"].flatMap({ Double($0) }) { + while Date().timeIntervalSince1970 < at { usleep(1000) } +} +let wallStart = Date().timeIntervalSince1970 let start = nowNs(), end = start + UInt64(secs * 1e9) var finished = false // Poll on a thread of its own; the main thread keeps its run loop going for CoreSimulator's replies. @@ -131,7 +161,9 @@ let poller = Thread { let seed = IOSurfaceGetSeed(s) if s !== lastSurf || seed != lastSeed { stamps.append(t); lastSeed = seed; lastSurf = s - if readStrip { strip.append((t, readCounter(s))) } + let counter = readStrip ? readCounter(s) : -1 + if readStrip { strip.append((t, counter)) } + if rawPath != nil { raw.append((t, counter, probe.map { readLuma(s, $0) } ?? -1)) } } polls += 1 usleep(500) @@ -187,5 +219,9 @@ if readStrip { "gt20ms": uiv.filter { $0 > 20 }.count, "gt34ms": uiv.filter { $0 > 34 }.count, ] as [String: Any] } +if let path = rawPath { + let lines = raw.map { String(format: "%.6f %ld %ld", wallStart + Double($0.0 - start) / 1e9, $0.1, $0.2) }.joined(separator: "\n") + try? (lines + "\n").write(toFile: path, atomically: true, encoding: .utf8) +} let data = try! JSONSerialization.data(withJSONObject: result, options: [.sortedKeys]) print(String(data: data, encoding: .utf8)!) From 853f488e53a13e2d8d9dc2d75a1f07c9b32f1536 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:03:38 +0200 Subject: [PATCH 02/23] bench: simfps stamps start with the screen as it stands --- src/bench/simfps.swift | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bench/simfps.swift b/src/bench/simfps.swift index dc50aa6..c1d9ab2 100644 --- a/src/bench/simfps.swift +++ b/src/bench/simfps.swift @@ -154,6 +154,8 @@ var finished = false let poller = Thread { var lastSeed = IOSurfaceGetSeed(surf0) var lastSurf = surf0 + // The first line of SIMFPS_STAMPS is the screen as it stands at the start (a static screen makes no new surface). + if rawPath != nil { raw.append((start, readStrip ? readCounter(surf0) : -1, probe.map { readLuma(surf0, $0) } ?? -1)) } while true { let t = nowNs() if t >= end { break } From ac533f968cb8fbaf310dde308998e40a56a498d4 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:05:16 +0200 Subject: [PATCH 03/23] bench: the suite's pure parts (statistics, frame strip, PNG and tear detection, targets, A/B rule, result schema, compare, chart) with unit tests --- src/bench/macbench/__init__.py | 1 + src/bench/macbench/abrule.py | 60 ++++++++ src/bench/macbench/analysis.py | 128 +++++++++++++++++ src/bench/macbench/chart.py | 122 +++++++++++++++++ src/bench/macbench/compare.py | 104 ++++++++++++++ src/bench/macbench/png.py | 110 +++++++++++++++ src/bench/macbench/schema.py | 139 +++++++++++++++++++ src/bench/macbench/stats.py | 119 ++++++++++++++++ src/bench/macbench/strip.py | 88 ++++++++++++ src/bench/macbench/targets.py | 76 ++++++++++ src/bench/macbench/tears.py | 58 ++++++++ src/bench/tests/__init__.py | 0 src/bench/tests/helpers.py | 63 +++++++++ src/bench/tests/test_pure.py | 244 +++++++++++++++++++++++++++++++++ 14 files changed, 1312 insertions(+) create mode 100644 src/bench/macbench/__init__.py create mode 100644 src/bench/macbench/abrule.py create mode 100644 src/bench/macbench/analysis.py create mode 100644 src/bench/macbench/chart.py create mode 100644 src/bench/macbench/compare.py create mode 100644 src/bench/macbench/png.py create mode 100644 src/bench/macbench/schema.py create mode 100644 src/bench/macbench/stats.py create mode 100644 src/bench/macbench/strip.py create mode 100644 src/bench/macbench/targets.py create mode 100644 src/bench/macbench/tears.py create mode 100644 src/bench/tests/__init__.py create mode 100644 src/bench/tests/helpers.py create mode 100644 src/bench/tests/test_pure.py diff --git a/src/bench/macbench/__init__.py b/src/bench/macbench/__init__.py new file mode 100644 index 0000000..0488359 --- /dev/null +++ b/src/bench/macbench/__init__.py @@ -0,0 +1 @@ +"""macbox bench: the scaling, regression and A/B suite (see ../README.md). Python 3.8+, standard library only.""" diff --git a/src/bench/macbench/abrule.py b/src/bench/macbench/abrule.py new file mode 100644 index 0000000..2b216c8 --- /dev/null +++ b/src/bench/macbench/abrule.py @@ -0,0 +1,60 @@ +"""The A/B decision rule (docs: README "A/B discipline"). + +Arms run interleaved A B B A inside one window; only the difference inside that window counts. Two pairs: +pair 1 = the first A with the first B, pair 2 = the second B with the second A. The rule is fixed BEFORE the run +(threshold on the command line) and has no re-run: a result below the bar, or with pairs that disagree, CLOSES +the lever.""" +from . import stats + +DEFAULT_THRESHOLD = 0.04 # the run-to-run noise on this stack + + +def pair_effects(a1, b1, a2, b2, higher_better=True): + """Relative improvement of B over A in each pair (positive = B better). None where A is 0 or missing.""" + def eff(a, b): + d = stats.rel_diff(a, b) + return None if d is None else (d if higher_better else -d) + return eff(a1, b1), eff(a2, b2) + + +def decide(a1, b1, a2, b2, higher_better=True, threshold=DEFAULT_THRESHOLD, tails=None, tail_tol=DEFAULT_THRESHOLD): + """tails: {name: (a1, b1, a2, b2, higher_better)} metrics that must not get worse (p95, max, gaps...). + Returns {"decision": "ADOPT"|"CLOSE", "effect", "pairs", "why": [...], "tails": {...}}""" + e1, e2 = pair_effects(a1, b1, a2, b2, higher_better) + why = [] + if e1 is None or e2 is None: + return {"decision": "CLOSE", "effect": None, "pairs": [e1, e2], "why": ["a pair has no usable value"], "tails": {}} + effect = (e1 + e2) / 2 + agree = (e1 > 0 and e2 > 0) or (e1 < 0 and e2 < 0) + adopt = True + if not agree: + adopt = False + why.append("the pairs disagree (%+.1f%%, %+.1f%%)" % (e1 * 100, e2 * 100)) + if effect <= threshold: + adopt = False + why.append("effect %+.1f%% does not exceed %.1f%%" % (effect * 100, threshold * 100)) + tail_out = {} + for name, (ta1, tb1, ta2, tb2, hb) in (tails or {}).items(): + t1, t2 = pair_effects(ta1, tb1, ta2, tb2, hb) + te = None if t1 is None or t2 is None else (t1 + t2) / 2 + worse = te is not None and te < -tail_tol + tail_out[name] = {"effect": te, "worse": worse} + if worse: + adopt = False + why.append("tail %s worsens %+.1f%%" % (name, te * 100)) + return {"decision": "ADOPT" if adopt else "CLOSE", "effect": effect, "pairs": [e1, e2], "why": why, "tails": tail_out} + + +def abba_order(): + return ["A", "B", "B", "A"] + + +def decision_line(knob, metric, arms, res, threshold): + """One human line, the one stored in the result.""" + f = lambda x: "n/a" if x is None else "%.2f" % x + pairs = ", ".join("n/a" if p is None else "%+.1f%%" % (p * 100) for p in res["pairs"]) + eff = "n/a" if res["effect"] is None else "%+.1f%%" % (res["effect"] * 100) + why = ("; " + "; ".join(res["why"])) if res["why"] else "" + return "AB %s: %s | %s A %s/%s B %s/%s | effect %s (pairs %s) | bar %.1f%%%s" % ( + knob, res["decision"], metric, f(arms[0][0]), f(arms[0][1]), f(arms[1][0]), f(arms[1][1]), eff, pairs, + threshold * 100, why) diff --git a/src/bench/macbench/analysis.py b/src/bench/macbench/analysis.py new file mode 100644 index 0000000..97cdf5c --- /dev/null +++ b/src/bench/macbench/analysis.py @@ -0,0 +1,128 @@ +"""Pure analysis of one measurement window: raw per-frame lines and probes in, per-simulator metrics out.""" +from . import stats, strip, targets + +LUMA_MID = 127 + + +def parse_raw(text): + """simfps's SIMFPS_STAMPS file: ' ' per line -> [(t, counter, luma)].""" + out = [] + for line in text.splitlines(): + p = line.split() + if len(p) == 3: + try: + out.append((float(p[0]), int(p[1]), int(p[2]))) + except ValueError: + pass + return out + + +def probe_latencies(raw, injections, timeout_s=2.0): + """Input-latency PROXY for one simulator. + + raw: [(t, counter, luma)] ascending (the first line is the screen at the start); injections: [(t0, t_return)] + guest wall-clock seconds, t0 taken just before the injecting command was started. A probe is answered by the + first surface after t0 whose marker (luma above/below 127) differs from the last one before t0. No such + surface within timeout_s is a MISS; it counts at the timeout in the percentiles (never dropped). + What it includes: starting `simctl` (its own duration is `call_ms`), CoreSimulator, the app's handling and + one frame's composite. What it leaves out: the Device panel stream, the viewer, a real finger's HID path. + Returns {"n", "misses", "p50_ms", "p95_ms", "max_ms", "call_p50_ms", "latencies_ms": [...]}""" + lat, calls, misses = [], [], 0 + for t0, tret in injections: + before = None + for t, _, luma in raw: + if t >= t0: + break + if luma >= 0: + before = luma > LUMA_MID + hit = None + if before is not None: + for t, _, luma in raw: + if t < t0 or luma < 0: + continue + if t > t0 + timeout_s: + break + if (luma > LUMA_MID) != before: + hit = t - t0 + break + calls.append((tret - t0) * 1000.0) + if hit is None: + misses += 1 + lat.append(timeout_s * 1000.0) + else: + lat.append(hit * 1000.0) + return {"n": len(injections), "misses": misses, "p50_ms": stats.r1(stats.percentile(lat, 0.5)), + "p95_ms": stats.r1(stats.percentile(lat, 0.95)), "max_ms": stats.r1(max(lat)) if lat else None, + "call_p50_ms": stats.r1(stats.percentile(calls, 0.5)), "latencies_ms": [round(x, 1) for x in lat]} + + +def sim_metrics(idx, role, raw, injections, nominal_fps=60.0, timeout_s=2.0, app_ticks=None, window=None): + """One simulator's entry in a step. A busy simulator gets frame metrics from its strip counters (distinct + frames), every simulator with probes gets the input proxy. `window` = (t_start, t_end) clips the raw lines.""" + if window: + raw = [r for r in raw if window[0] <= r[0] <= window[1]] or raw[:1] + sim = {"idx": idx, "role": role} + if role == "busy": + a = strip.analyze_counters([(t, c) for t, c, _ in raw]) + fm = stats.frame_stats([t for t, _ in a["unique"]], nominal_fps) + sim.update(fm) + sim["strip"] = {k: a[k] for k in ("repeats", "lost", "backwards", "jumps", "unreadable", "span_ticks")} + ok, why = strip.strip_gate(a, app_ticks) + sim["strip_gate"] = {"ok": ok, "why": why} + sim["app_ticks"] = app_ticks + if injections: + pl = probe_latencies(raw, injections, timeout_s) + sim.update({"input_n": pl["n"], "input_misses": pl["misses"], "input_p50_ms": pl["p50_ms"], + "input_p95_ms": pl["p95_ms"], "input_max_ms": pl["max_ms"], "input_call_p50_ms": pl["call_p50_ms"]}) + return sim + + +def fairness(sims): + """Jain's index and min/max over the distinct fps of the simulators that were meant to animate.""" + f = [s["fps"] for s in sims if s.get("role") == "busy" and s.get("fps") is not None] + return {"jain": stats.r1(stats.jain(f), 3), "min_max": stats.r1(stats.min_max_ratio(f), 3), "n": len(f)} + + +STANDING = ("qemu_cpu_cores", "gpu_busy_pct", "guest_cpu_total_pct", "guest_mem_used_mb", "guest_compressed_mb", + "host_mem_used_mb", "gpu_ms_per_frame", "sims_cpu_pct", "sims_rss_mb") + + +def flat_system(step): + """The step's system numbers as one flat {name: value} (nested dicts joined with no prefix clash).""" + out = {} + s = step.get("system", {}) + for k, v in s.items(): + if isinstance(v, (int, float)): + out[k] = v + elif isinstance(v, dict): + for k2, v2 in v.items(): + if isinstance(v2, (int, float)): + out[k + "_" + k2 if not k2.startswith(k) else k2] = v2 + return out + + +def standing_cost(steps, names=None): + """Per-simulator standing cost: the least-squares slope of each system number over N (units per simulator), + its intercept (the cost with no simulator) and R^2. Steps from `idle` give the pure standing cost.""" + xs = [s["n"] for s in steps] + flats = [flat_system(s) for s in steps] + keys = names or sorted({k for f in flats for k in f}) + out = {} + for k in keys: + b, a, r2 = stats.slope(xs, [f.get(k) for f in flats]) + if b is not None: + out[k] = {"per_sim": round(b, 3), "at_zero": round(a, 3), "r2": round(r2, 3)} + return out + + +def summarize_scale(steps, target_specs, stopped=None): + tg = [targets.parse_target(t) for t in target_specs] + mx = {} + for t in tg: + mx[t["name"]] = targets.max_sustainable(steps, t) + for st in steps: + st["targets"] = {t["name"]: targets.evaluate(t, st["sims"])["pass"] for t in tg} + st["fairness"] = fairness(st["sims"]) + return {"max_n": mx, "stopped": stopped, "standing_cost": standing_cost(steps), + "fairness_min_jain": min([s["fairness"]["jain"] for s in steps if s["fairness"]["jain"] is not None] or [None]) + if any(s["fairness"]["jain"] is not None for s in steps) else None} diff --git a/src/bench/macbench/chart.py b/src/bench/macbench/chart.py new file mode 100644 index 0000000..55bac10 --- /dev/null +++ b/src/bench/macbench/chart.py @@ -0,0 +1,122 @@ +"""A self-contained HTML/SVG scaling chart from a scale result: no scripts, no fonts, no external files.""" +import html + +COLORS = ["#2a6fdb", "#d9534f", "#2e9e5b", "#e08a1e", "#8e5ac8", "#17a2b8", "#a0522d", "#6c757d"] +W, H, L, R, T, B = 460, 260, 52, 14, 28, 40 + + +def _nice(hi): + if hi <= 0: + return 1.0 + import math + e = 10 ** math.floor(math.log10(hi)) + for m in (1, 2, 2.5, 5, 10): + if hi <= m * e: + return m * e + return 10 * e + + +def panel(title, xs, series, ylabel, hlines=(), ymax=None): + """One SVG chart. series: [(name, [(x, y)], style)] style 'line' or 'dots'. hlines: [(y, label)].""" + pts = [p for _, ps, _ in series for p in ps if p[1] is not None] + hi = ymax or _nice(max([p[1] for p in pts] + [h[0] for h in hlines] + [0]) * 1.05 or 1) + x0, x1 = min(xs), max(xs) + span = (x1 - x0) or 1 + + def px(x): + return L + (x - x0) / span * (W - L - R) if x1 != x0 else (L + W - R) / 2 + + def py(y): + return H - B - (y / hi) * (H - T - B) + o = ['' % (W, H, W, H), + '' % (W, H), + '%s' % (L, html.escape(title))] + for i in range(5): + y = hi * i / 4 + o.append('' % (L, W - R, py(y), py(y))) + o.append('%g' % (L - 4, py(y) + 4, round(y, 2))) + for x in xs: + o.append('%g' % (px(x), H - B + 14, x)) + o.append('simulators' % ((L + W - R) // 2, H - 6)) + o.append('%s' % ((T + H - B) // 2 + 20, (T + H - B) // 2 + 20, html.escape(ylabel))) + for y, label in hlines: + o.append('' % (L, W - R, py(y), py(y))) + o.append('%s' % (W - R, py(y) - 3, html.escape(label))) + for i, (name, ps, style) in enumerate(series): + c = COLORS[i % len(COLORS)] + ps = [p for p in ps if p[1] is not None] + if style == "line" and len(ps) > 1: + o.append('' % (c, " ".join("%.1f,%.1f" % (px(x), py(y)) for x, y in ps))) + for x, y in ps: + o.append('%s N=%g: %g' % ( + px(x), py(y), 3 if style == "line" else 2.5, c, "1" if style == "line" else "0.55", html.escape(name), x, y)) + lx = L + 6 + for i, (name, ps, style) in enumerate(series): + o.append('%s' % (lx, T - 12 + 0, COLORS[i % len(COLORS)], lx + 12, T - 4, html.escape(name))) + lx += 12 + 6 * len(name) + 14 + o.append("") + return "\n".join(o) + + +def _per_sim(steps, key, role="busy"): + pts = [] + for st in steps: + for s in st["sims"]: + if s.get("role") == role and s.get(key) is not None: + pts.append((st["n"], s[key])) + return pts + + +def _agg(steps, fn, key, role="busy"): + out = [] + for st in steps: + v = [s[key] for s in st["sims"] if s.get("role") == role and s.get(key) is not None] + out.append((st["n"], fn(v) if v else None)) + return out + + +def _sys(steps, *path): + out = [] + for st in steps: + v = st["system"] + for p in path: + v = v.get(p) if isinstance(v, dict) else None + out.append((st["n"], v)) + return out + + +def scale_chart(doc): + d = doc["data"] + steps = sorted(d["steps"], key=lambda s: s["n"]) + xs = [s["n"] for s in steps] + nominal = d.get("nominal_fps", 60) + panels = [ + panel("distinct frames/s per simulator", xs, [("each sim", _per_sim(steps, "fps"), "dots"), + ("min", _agg(steps, min, "fps"), "line"), ("mean", _agg(steps, lambda v: sum(v) / len(v), "fps"), "line")], + "frames/s", hlines=[(nominal, "nominal %g" % nominal)]), + panel("frame interval p95 / p99 (ms)", xs, [("p95 each", _per_sim(steps, "p95_ms"), "dots"), + ("worst p95", _agg(steps, max, "p95_ms"), "line"), ("worst p99", _agg(steps, max, "p99_ms"), "line")], + "ms", hlines=[(25, "25 ms")]), + panel("input proxy p95 (ms)", xs, [("each sim", [(st["n"], s["input_p95_ms"]) for st in steps for s in st["sims"] if s.get("input_p95_ms") is not None], "dots"), + ("worst", [(st["n"], max([s["input_p95_ms"] for s in st["sims"] if s.get("input_p95_ms") is not None] or [None])) for st in steps], "line")], + "ms", hlines=[(150, "150 ms")]), + panel("QEMU CPU (cores) and GPU busy (%)", xs, [("QEMU cores", _sys(steps, "qemu_cpu_cores"), "line"), + ("GPU busy %", [(x, None if y is None else y / 10.0) for x, y in _sys(steps, "gpu_busy_pct")], "line")], + "cores | % / 10"), + panel("guest CPU of the simulators (%)", xs, [("simulators", _sys(steps, "guest", "sims_cpu_pct"), "line"), + ("everything else", _sys(steps, "guest", "other_cpu_pct"), "line")], "% of one core"), + panel("guest memory (MB)", xs, [("used", _sys(steps, "guest", "mem_used_mb"), "line"), + ("compressed", _sys(steps, "guest", "compressed_mb"), "line"), ("simulators RSS", _sys(steps, "guest", "sims_rss_mb"), "line")], "MB"), + panel("GPU ms per frame", xs, [("GPU ms/frame", _sys(steps, "gpu_ms_per_frame"), "line")], "ms"), + panel("fairness (Jain index x 100)", xs, [("Jain x100", [(st["n"], None if st.get("fairness", {}).get("jain") is None else st["fairness"]["jain"] * 100) for st in steps], "line"), + ("min/max x100", [(st["n"], None if st.get("fairness", {}).get("min_max") is None else st["fairness"]["min_max"] * 100) for st in steps], "line")], + "index x100", ymax=100), + ] + mx = d.get("summary", {}).get("max_n", {}) + rows = "".join("%s%s%s" % (html.escape(k), v.get("max_n"), v.get("first_fail")) for k, v in mx.items()) + return ('macbox bench %s' + '

macbox bench: %s, %sx, %s

' + '

run %s, build %s

' + '%s
targetmax sustainable Nfirst failing N
%s
' + % (html.escape(doc["id"]), html.escape(str(d.get("mix"))), d.get("scale_x"), html.escape(str(d.get("device", ""))), + html.escape(doc["id"]), html.escape(str(doc["env"].get("build_id"))), rows, "\n".join(panels))) diff --git a/src/bench/macbench/compare.py b/src/bench/macbench/compare.py new file mode 100644 index 0000000..f162cf0 --- /dev/null +++ b/src/bench/macbench/compare.py @@ -0,0 +1,104 @@ +"""compare two result files: what moved between a baseline and a candidate, beyond the noise.""" +from . import stats + +NOISE = 0.04 # run-to-run noise on this stack + +# (label, getter on a step, higher is better) +STEP_ROWS = [ + ("min distinct fps", lambda st: _min([s.get("fps") for s in st["sims"] if s.get("role") == "busy"]), True), + ("worst p95 ms", lambda st: _max([s.get("p95_ms") for s in st["sims"] if s.get("role") == "busy"]), False), + ("worst p99 ms", lambda st: _max([s.get("p99_ms") for s in st["sims"] if s.get("role") == "busy"]), False), + ("longest gap s", lambda st: _max([s.get("longest_gap_s") for s in st["sims"]]), False), + ("input p95 ms", lambda st: _max([s.get("input_p95_ms") for s in st["sims"]]), False), + ("Jain index", lambda st: (st.get("fairness") or {}).get("jain"), True), + ("QEMU cores", lambda st: st["system"].get("qemu_cpu_cores"), False), + ("GPU busy %", lambda st: st["system"].get("gpu_busy_pct"), False), + ("GPU ms/frame", lambda st: st["system"].get("gpu_ms_per_frame"), False), + ("guest mem used MB", lambda st: (st["system"].get("guest") or {}).get("mem_used_mb"), False), +] + + +def _vals(xs): + return [x for x in xs if x is not None] + + +def _min(xs): + v = _vals(xs) + return min(v) if v else None + + +def _max(xs): + v = _vals(xs) + return max(v) if v else None + + +def verdict(a, b, higher_better, noise=NOISE): + d = stats.rel_diff(a, b) + if d is None: + return "n/a", None + if abs(d) <= noise: + return "same", d + better = d > 0 if higher_better else d < 0 + return ("better" if better else "WORSE"), d + + +def compare_scale(a, b, noise=NOISE): + """Lines and a worse-count for two scale results. Rows are per N present in both.""" + lines, worse = [], 0 + da, db = a["data"], b["data"] + lines.append("A %s B %s mix %s/%s scale %sx/%sx noise bar %.0f%%" % ( + a["id"], b["id"], da.get("mix"), db.get("mix"), da.get("scale_x"), db.get("scale_x"), noise * 100)) + if (da.get("mix"), da.get("scale_x")) != (db.get("mix"), db.get("scale_x")): + lines.append(" warning: different mix or scale: the numbers are not comparable") + for name in sorted(set(da["summary"]["max_n"]) | set(db["summary"]["max_n"])): + ma = (da["summary"]["max_n"].get(name) or {}).get("max_n") + mb = (db["summary"]["max_n"].get(name) or {}).get("max_n") + tag = "same" if ma == mb else ("better" if (mb or 0) > (ma or 0) else "WORSE") + worse += tag == "WORSE" + lines.append(" max sustainable N %-12s A %s B %s %s" % (name, ma, mb, tag)) + sa = {s["n"]: s for s in da["steps"]} + sb = {s["n"]: s for s in db["steps"]} + for n in sorted(set(sa) & set(sb)): + lines.append(" N=%d" % n) + for label, get, hb in STEP_ROWS: + try: + x, y = get(sa[n]), get(sb[n]) + except (KeyError, TypeError): + continue + v, d = verdict(x, y, hb, noise) + if v == "n/a": + continue + worse += v == "WORSE" + lines.append(" %-18s %10s -> %-10s %+6.1f%% %s" % (label, _f(x), _f(y), d * 100, v)) + return lines, worse + + +def _f(x): + return "n/a" if x is None else ("%.2f" % x if abs(x) < 100 else "%.0f" % x) + + +def compare_sections(a, b): + sa = {s["name"]: s["status"] for s in a["sections"]} + out = [] + for s in b["sections"]: + was = sa.get(s["name"]) + mark = "" if was == s["status"] else " (was %s)" % was + out.append(" %s %s%s" % (s["status"], s["line"], mark)) + return out + + +def compare(a, b, noise=NOISE): + """(text, regressions): regressions counts rows that got worse beyond the noise bar and sections that + went from PASS to FAIL.""" + if a.get("kind") != b.get("kind"): + return "different kinds: %s vs %s" % (a.get("kind"), b.get("kind")), 1 + lines, worse = [], 0 + if a["kind"] == "scale": + l, worse = compare_scale(a, b, noise) + lines += l + lines.append("sections of B:") + lines += compare_sections(a, b) + sa = {s["name"]: s["status"] for s in a["sections"]} + worse += sum(1 for s in b["sections"] if sa.get(s["name"]) == "PASS" and s["status"] == "FAIL") + lines.append("%d regression(s) beyond the %.0f%% bar" % (worse, noise * 100)) + return "\n".join(lines), worse diff --git a/src/bench/macbench/png.py b/src/bench/macbench/png.py new file mode 100644 index 0000000..fd332c9 --- /dev/null +++ b/src/bench/macbench/png.py @@ -0,0 +1,110 @@ +"""A small PNG reader and writer (8-bit gray, gray+alpha, RGB, RGBA; not interlaced), standard library only. +simctl's screenshots are of that kind. No Pillow.""" +import hashlib +import struct +import zlib + + +class Image: + def __init__(self, width, height, channels, rows): + self.w, self.h, self.ch, self.rows = width, height, channels, rows # rows: list of bytes + + def px(self, x, y): + """(r, g, b) of a pixel.""" + r = self.rows[y] + i = x * self.ch + if self.ch >= 3: + return r[i], r[i + 1], r[i + 2] + return r[i], r[i], r[i] + + def luma(self, x, y): + r, g, b = self.px(x, y) + return (r + g + b) // 3 + + def pixel_hash(self): + """sha256 over the RGB values of every pixel (alpha ignored), with the size mixed in: two screenshots + have the same hash only when every pixel is the same.""" + h = hashlib.sha256() + h.update(struct.pack(">II", self.w, self.h)) + for y in range(self.h): + r = self.rows[y] + if self.ch == 3: + h.update(r) + elif self.ch == 4: + h.update(bytes(v for i, v in enumerate(r) if i % 4 != 3)) + elif self.ch == 2: + h.update(bytes(r[i] for i in range(0, len(r), 2))) + else: + h.update(r) + return h.hexdigest() + + +def _paeth(a, b, c): + p = a + b - c + pa, pb, pc = abs(p - a), abs(p - b), abs(p - c) + return a if pa <= pb and pa <= pc else (b if pb <= pc else c) + + +def read_png(data): + if data[:8] != b"\x89PNG\r\n\x1a\n": + raise ValueError("not a PNG") + pos, idat, w = 8, [], None + while pos + 8 <= len(data): + n, typ = struct.unpack(">I4s", data[pos:pos + 8]) + body = data[pos + 8:pos + 8 + n] + pos += 12 + n + if typ == b"IHDR": + w, h, depth, ctype, _, _, interlace = struct.unpack(">IIBBBBB", body) + if depth != 8 or interlace != 0 or ctype not in (0, 2, 4, 6): + raise ValueError("unsupported PNG (need 8-bit, not interlaced, gray/RGB/RGBA)") + ch = {0: 1, 2: 3, 4: 2, 6: 4}[ctype] + elif typ == b"IDAT": + idat.append(body) + elif typ == b"IEND": + break + if w is None: + raise ValueError("no IHDR") + raw = zlib.decompress(b"".join(idat)) + stride = w * ch + rows, prev = [], bytearray(stride) + p = 0 + for _ in range(h): + ft = raw[p] + line = bytearray(raw[p + 1:p + 1 + stride]) + p += 1 + stride + if ft == 1: + for i in range(ch, stride): + line[i] = (line[i] + line[i - ch]) & 255 + elif ft == 2: + for i in range(stride): + line[i] = (line[i] + prev[i]) & 255 + elif ft == 3: + for i in range(stride): + left = line[i - ch] if i >= ch else 0 + line[i] = (line[i] + ((left + prev[i]) >> 1)) & 255 + elif ft == 4: + for i in range(stride): + left = line[i - ch] if i >= ch else 0 + ul = prev[i - ch] if i >= ch else 0 + line[i] = (line[i] + _paeth(left, prev[i], ul)) & 255 + elif ft != 0: + raise ValueError("bad PNG filter %d" % ft) + rows.append(bytes(line)) + prev = line + return Image(w, h, ch, rows) + + +def write_png(img_or_rows, width=None, height=None, channels=3): + """PNG bytes from an Image, or from rows (a list of bytes, each width*channels long). For tests and fixtures.""" + if isinstance(img_or_rows, Image): + rows, width, height, channels = img_or_rows.rows, img_or_rows.w, img_or_rows.h, img_or_rows.ch + else: + rows = img_or_rows + ctype = {1: 0, 2: 4, 3: 2, 4: 6}[channels] + + def chunk(t, b): + c = struct.pack(">I", len(b)) + t + b + return c + struct.pack(">I", zlib.crc32(t + b) & 0xFFFFFFFF) + raw = b"".join(b"\x00" + bytes(r) for r in rows) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, ctype, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 6)) + chunk(b"IEND", b"")) diff --git a/src/bench/macbench/schema.py b/src/bench/macbench/schema.py new file mode 100644 index 0000000..cc3b6e0 --- /dev/null +++ b/src/bench/macbench/schema.py @@ -0,0 +1,139 @@ +"""The result file: one JSON document per run, schema versioned. Nothing in it names a host, an account or a path.""" +import datetime +import json +import os +import re +import socket + +SCHEMA = 1 +KINDS = ("scale", "ab", "idle", "lifecycle", "build", "soak", "panel", "gates") +SECTION_STATUS = ("PASS", "FAIL", "SKIP", "INFO") + +_PATH = re.compile(r"(?:^|[\s\"'=:(,])(/(?:srv|home|Users|var|tmp|private|etc|opt|mnt|root|usr|run|Volumes|proc|sys)/[^\s\"',)]*)") +_IPV4 = re.compile(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b") + + +def now_utc(): + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def stamp(): + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d-%H%M%S") + + +def scrubber(extra_names=()): + """A function that cleans a string: home paths -> ~, the host name, ssh alias and other names -> , + IPv4 addresses -> . Paths not under a home directory are left for validate() to refuse.""" + names = {n for n in [socket.gethostname(), socket.gethostname().split(".")[0], *extra_names] if n and len(n) > 2} + home = os.path.expanduser("~") + + def clean(s): + if home and home != "/": + s = s.replace(home, "~") + s = re.sub(r"/Users/[^/\s\"']+", "~", s) + for n in sorted(names, key=len, reverse=True): + s = re.sub(r"(?", s) + return _IPV4.sub("", s) + return clean + + +def scrub(obj, clean): + if isinstance(obj, str): + return clean(obj) + if isinstance(obj, list): + return [scrub(x, clean) for x in obj] + if isinstance(obj, dict): + return {k: scrub(v, clean) for k, v in obj.items()} + return obj + + +def _strings(obj): + if isinstance(obj, str): + yield obj + elif isinstance(obj, list): + for x in obj: + yield from _strings(x) + elif isinstance(obj, dict): + for k, v in obj.items(): + yield str(k) + yield from _strings(v) + + +def privacy_problems(doc, extra_names=()): + """Strings in a result that would not be safe to publish: absolute paths, IPv4 addresses, host names.""" + names = {n for n in [socket.gethostname(), socket.gethostname().split(".")[0], *extra_names] if n and len(n) > 2} + bad = [] + for s in _strings(doc): + if _PATH.search(s): + bad.append("path: " + s[:80]) + if _IPV4.search(s): + bad.append("ip: " + s[:80]) + for n in names: + if re.search(r"(? None.""" + if not values: + return None + s = sorted(values) + return s[min(len(s) - 1, int((len(s) - 1) * p))] + + +def mean(values): + return sum(values) / len(values) if values else None + + +def median(values): + return percentile(values, 0.5) + + +def r1(x, digits=1): + return None if x is None else round(x, digits) + + +def intervals_ms(times_s): + """Gaps between consecutive times (seconds, ascending) in milliseconds.""" + return [(b - a) * 1000.0 for a, b in zip(times_s, times_s[1:])] + + +def frame_stats(times_s, nominal_fps=60.0): + """Per-simulator frame metrics from the times (seconds) of its distinct frames. + + fps (n - 1) / (last - first) + p50..max of the intervals between consecutive distinct frames, ms + over2x intervals longer than 2 / nominal_fps + longest_gap_s the longest interval in seconds + """ + n = len(times_s) + out = {"frames": n, "fps": 0.0, "p50_ms": None, "p95_ms": None, "p99_ms": None, "max_ms": None, + "over2x": 0, "longest_gap_s": None} + if n < 2: + return out + iv = intervals_ms(times_s) + span = times_s[-1] - times_s[0] + out["fps"] = r1((n - 1) / span if span > 0 else 0.0) + out["p50_ms"] = r1(percentile(iv, 0.5)) + out["p95_ms"] = r1(percentile(iv, 0.95)) + out["p99_ms"] = r1(percentile(iv, 0.99)) + out["max_ms"] = r1(max(iv)) + thr = 2000.0 / nominal_fps + out["over2x"] = sum(1 for x in iv if x > thr) + out["longest_gap_s"] = r1(max(iv) / 1000.0, 3) + return out + + +def stretches(times_s, still_ms=250.0): + """Split frame times into stretches of motion: a gap above still_ms ends a stretch (the screen stood + still or the app froze that long). Returns [(first, last, count)] for each stretch of 2+ frames.""" + out, cur = [], [] + for t in times_s: + if cur and (t - cur[-1]) * 1000.0 > still_ms: + if len(cur) > 1: + out.append((cur[0], cur[-1], len(cur))) + cur = [] + cur.append(t) + if len(cur) > 1: + out.append((cur[0], cur[-1], len(cur))) + return out + + +def jain(values): + """Jain's fairness index (sum x)^2 / (n * sum x^2): 1.0 when every value is equal, 1/n when one + simulator gets everything. None for no values; 1.0 for all zeros.""" + if not values: + return None + s2 = sum(v * v for v in values) + if s2 == 0: + return 1.0 + return sum(values) ** 2 / (len(values) * s2) + + +def min_max_ratio(values): + """min / max (1.0 = equal). None for no values, 1.0 for all zeros.""" + if not values: + return None + hi = max(values) + return 1.0 if hi == 0 else min(values) / hi + + +def slope(xs, ys): + """Least-squares line through (x, y): (slope, intercept, r2). Needs 2+ distinct x, else (None, None, None). + The per-simulator standing cost of a resource is the slope over N.""" + pts = [(x, y) for x, y in zip(xs, ys) if x is not None and y is not None] + n = len(pts) + if n < 2: + return None, None, None + mx = sum(p[0] for p in pts) / n + my = sum(p[1] for p in pts) / n + sxx = sum((p[0] - mx) ** 2 for p in pts) + if sxx == 0: + return None, None, None + sxy = sum((p[0] - mx) * (p[1] - my) for p in pts) + b = sxy / sxx + a = my - b * mx + syy = sum((p[1] - my) ** 2 for p in pts) + r2 = 1.0 if syy == 0 else (sxy * sxy) / (sxx * syy) + return b, a, r2 + + +def rel_diff(a, b): + """(b - a) / a as a fraction; None when a is 0 or missing.""" + if a is None or b is None or a == 0: + return None + return (b - a) / a + + +def finite(x): + return x is not None and isinstance(x, (int, float)) and math.isfinite(x) diff --git a/src/bench/macbench/strip.py b/src/bench/macbench/strip.py new file mode 100644 index 0000000..299d2de --- /dev/null +++ b/src/bench/macbench/strip.py @@ -0,0 +1,88 @@ +"""GlassBench's frame strip: a 16-bit counter drawn on every display-link tick as 18 squares of 20 pt from +(21, 120) pt of a 402 pt wide screen: white and black references, then bits 0..15 (white = 1).""" +from . import stats + +WRAP = 65536 +MAX_FWD = 1000 # a forward step of up to this many ticks is a stall (frames lost); larger is a jump + + +def read_counter_luma(luma_at, width_px): + """The counter from a luma function luma_at(x_px, y_px). Same geometry as simfps's readCounter. + Returns the counter, or -1 when the references are unreadable (no strip, or it is covered).""" + scale = width_px / 402.0 + + def block(i): + return luma_at(int((21 + 20 * i + 10) * scale), int(130 * scale)) + white, black = block(0), block(1) + if white - black <= 100: + return -1 + mid = (white + black) // 2 + n = 0 + for b in range(16): + if block(b + 2) > mid: + n |= 1 << b + return n + + +def read_counter_image(img): + return read_counter_luma(img.luma, img.w) + + +def analyze_counters(samples): + """Distinct app frames from the strip samples [(t_seconds, counter or -1)] of one simulator. + + A sample whose counter equals the previous distinct one is a repeat (the compositor made another + surface; normal). A counter that moved forward by d: d - 1 ticks never reached the screen (lost). + A counter that moved BACKWARD by up to 1000 (an older frame shown after a newer one) or jumped + forward by more than 1000 is `backwards` / `jumps`: never normal, and the strip gate fails on them. + Returns {"unique": [(t, counter)], "repeats", "lost", "backwards", "jumps", "unreadable", "span_ticks"} + where span_ticks is the counter distance covered, i.e. ticks the app made between first and last frame.""" + unique, repeats, lost, back, jumps, unreadable, span = [], 0, 0, 0, 0, 0, 0 + for t, c in samples: + if c < 0: + unreadable += 1 + continue + if unique: + d = (c - unique[-1][1]) % WRAP + if d == 0: + repeats += 1 + continue + if d <= MAX_FWD: + lost += d - 1 + span += d + elif d >= WRAP - MAX_FWD: + back += 1 # went back: do not advance the reference, the next sample is judged against it + continue + else: + jumps += 1 + span += d + unique.append((t, c)) + return {"unique": unique, "repeats": repeats, "lost": lost, "backwards": back, "jumps": jumps, + "unreadable": unreadable, "span_ticks": span} + + +def strip_gate(analysis, app_ticks=None, max_unreadable_frac=0.01): + """(ok, reason) of the strip gate. + FAIL: a frame out of order (backwards), a jump, a counter that advanced more than the app says it ticked + (reading garbage), or more than 1% of samples unreadable. Lost ticks are NOT a failure: they are the fps + metric (a simulator that drops frames is slow, not wrong). Skips are judged against the app's own tick + count (GlassBench's pace.json) when given.""" + a = analysis + total = len(a["unique"]) + a["repeats"] + a["unreadable"] + a["backwards"] + if a["backwards"]: + return False, "%d frame(s) shown out of order" % a["backwards"] + if a["jumps"]: + return False, "%d counter jump(s) over %d ticks" % (a["jumps"], MAX_FWD) + if app_ticks is not None and a["span_ticks"] > app_ticks * 1.02 + 2: + return False, "counter advanced %d ticks but the app ticked %d times" % (a["span_ticks"], app_ticks) + if total and a["unreadable"] / total > max_unreadable_frac: + return False, "%d of %d samples unreadable" % (a["unreadable"], total) + return True, "ok" + + +def distinct_metrics(samples, nominal_fps=60.0): + """frame_stats of the distinct frames, plus the strip bookkeeping, for one simulator.""" + a = analyze_counters(samples) + m = stats.frame_stats([t for t, _ in a["unique"]], nominal_fps) + m.update({k: a[k] for k in ("repeats", "lost", "backwards", "jumps", "unreadable", "span_ticks")}) + return m diff --git a/src/bench/macbench/targets.py b/src/bench/macbench/targets.py new file mode 100644 index 0000000..f18c39e --- /dev/null +++ b/src/bench/macbench/targets.py @@ -0,0 +1,76 @@ +"""Quality targets ("every sim >= 55 distinct/s and p95 <= 25 ms") and the maximum sustainable N for each. + +A target is `name: cond, cond, ...`; a cond is ` ` with op in >= <= > <. +Metrics (per simulator): fps (distinct app frames/s), p50, p95, p99, max (ms between distinct frames), over2x, +stall (longest gap between distinct frames, s), input_p50, input_p95 (ms, proxy; misses count at the timeout), +cpu (guest CPU % of one core), rss (MB). A metric a simulator has no value for (fps of an idle simulator, +the input proxy where no probe ran) is not judged for that simulator; a target no simulator of an N has any +value for is `None` (unknown), never a pass.""" +import re + +METRICS = {"fps": "fps", "p50": "p50_ms", "p95": "p95_ms", "p99": "p99_ms", "max": "max_ms", "over2x": "over2x", + "stall": "longest_gap_s", "input_p50": "input_p50_ms", "input_p95": "input_p95_ms", + "cpu": "guest_cpu_pct", "rss": "guest_rss_mb"} +OPS = {">=": lambda a, b: a >= b, "<=": lambda a, b: a <= b, ">": lambda a, b: a > b, "<": lambda a, b: a < b} + +DEFAULT_TARGETS = [ + "smooth60: fps>=55, p95<=25", + "smooth30: fps>=30, p95<=50", + "responsive: input_p95<=150, stall<0.3", +] +_COND = re.compile(r"^\s*([a-z0-9_]+)\s*(>=|<=|>|<)\s*([0-9.]+)\s*$") + + +def parse_target(spec): + """'name: fps>=55, p95<=25' -> {"name", "conds": [(metric, op, value)]}""" + if ":" not in spec: + raise ValueError("target %r: expected name: cond, cond" % spec) + name, rest = spec.split(":", 1) + conds = [] + for c in rest.split(","): + m = _COND.match(c) + if not m or m.group(1) not in METRICS: + raise ValueError("target %r: bad condition %r (metrics: %s)" % (spec, c.strip(), ", ".join(METRICS))) + conds.append((m.group(1), m.group(2), float(m.group(3)))) + if not conds: + raise ValueError("target %r has no condition" % spec) + return {"name": name.strip(), "conds": conds} + + +def evaluate(target, sims): + """target against one N's per-simulator dicts: {"pass": True|False|None, "failing": [...], "judged": int}""" + failing, judged = [], 0 + for sim in sims: + for metric, op, val in target["conds"]: + v = sim.get(METRICS[metric]) + if v is None: + continue + judged += 1 + if not OPS[op](v, val): + failing.append({"sim": sim.get("idx"), "cond": "%s%s%g" % (metric, op, val), "value": v}) + if judged == 0: + return {"pass": None, "failing": [], "judged": 0} + return {"pass": not failing, "failing": failing, "judged": judged} + + +def max_sustainable(steps, target): + """steps: [{"n", "sims": [...], "gates_ok": bool, "stable": bool}] in any order. The largest N such that this + N and every smaller measured N passed the target with its gates ok and no instability. The first N that does + not pass ends it (a pass above a fail is noise or luck, not sustainability). + Returns {"max_n": int|None, "first_fail": int|None, "unknown_at": int|None, "evaluated": {n: pass}}""" + best, first_fail, unknown, ev = None, None, None, {} + for st in sorted(steps, key=lambda s: s["n"]): + r = evaluate(target, st["sims"]) + ok = r["pass"] + if ok and not (st.get("gates_ok", True) and st.get("stable", True)): + ok = False + ev[st["n"]] = ok + if first_fail is not None or unknown is not None: + continue + if ok: + best = st["n"] + elif ok is None: + unknown = st["n"] + else: + first_fail = st["n"] + return {"max_n": best, "first_fail": first_fail, "unknown_at": unknown, "evaluated": ev} diff --git a/src/bench/macbench/tears.py b/src/bench/macbench/tears.py new file mode 100644 index 0000000..fc98e47 --- /dev/null +++ b/src/bench/macbench/tears.py @@ -0,0 +1,58 @@ +"""Tear detector for GlassBench screenshots (ported from the private tears.py; no Pillow). + +Each card is a vertical gradient, so the strip at the card's right edge (no text) changes by about one level +a row at 3x. A band of rows taken from another frame (the scroll moved meanwhile) shows as a step there, away +from card edges, and moves the cards' spacing away from 336 px (at 3x). Calibrated for GlassBench; it needs a +screenshot of the scrolling app (it says `applicable: false` when it finds no cards).""" + + +def _row_means(img, x0, x1, y0, y1): + rows = [] + n = max(1, x1 - x0) + for y in range(img.h): + acc = [0, 0, 0] + if y0 - 80 <= y <= y1 + 80: # rows outside the checked band are not needed + for x in range(x0, x1): + r, g, b = img.px(x, y) + acc[0] += r; acc[1] += g; acc[2] += b + rows.append((acc[0] / n, acc[1] / n, acc[2] / n)) + return rows + + +def detect_tears(img): + """{"tear": bool, "applicable": bool, "odd": [...], "steps": [(row, size)], "gaps": [heights]}""" + w, h = img.w, img.h + s = w / 1206.0 + x0, x1 = int(1125 * s), max(int(1125 * s) + 1, int(1150 * s)) + y0, y1 = int(400 * s), min(int(1700 * s), h - 1) + rows = _row_means(img, x0, x1, y0, y1) + gap = [min(c) > 225 for c in rows] # the white background between cards + margin = int(75 * s) + near = [False] * h + for y in range(h): + if gap[y]: + for k in range(max(0, y - margin), min(h, y + margin + 1)): + near[k] = True + thr = 6.0 if s > 0.9 else 6.0 / s * 0.5 + steps = [] + for y in range(y0 + 1, y1): + if near[y] or near[y - 1]: + continue + d = max(abs(a - b) for a, b in zip(rows[y], rows[y - 1])) + if d > thr: + steps.append((y, round(d, 1))) + gaps, y = [], y0 + while y < y1: + if gap[y]: + st = y + while y < y1 and gap[y]: + y += 1 + gaps.append((st, y - st)) + y += 1 + gaps = [g for g in gaps if g[0] > y0 + 1 and g[0] + g[1] < y1 - 1] # whole gaps only + spacing = [b[0] - a[0] for a, b in zip(gaps, gaps[1:])] + exp, tol = 336 * s, max(2, 4 * s) + odd = [x for x in spacing if abs(x - exp) > tol] + [g[1] for g in gaps if abs(g[1] - 58 * s) > tol] + # As in the private tool, the card spacing and gap heights decide; steps in the gradient are reported only. + return {"tear": bool(odd), "applicable": len(gaps) >= 2, "odd": odd, "steps": steps[:8], + "gaps": [g[1] for g in gaps]} diff --git a/src/bench/tests/__init__.py b/src/bench/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/bench/tests/helpers.py b/src/bench/tests/helpers.py new file mode 100644 index 0000000..b178504 --- /dev/null +++ b/src/bench/tests/helpers.py @@ -0,0 +1,63 @@ +"""Fixtures for the tests: synthetic screenshots and a synthetic scale result. No VM, no network.""" +from macbench import png + + +def card_image(width=603, tear_at=None, with_cards=True): + """A screenshot like GlassBench's at scale s = width/1206: white background, cards 336*s apart whose right edge + strip (x 1125..1150 * s) is a vertical gradient (1 level per row at 3x), gaps 58*s tall. tear_at=y shifts every + row below y by 40*s rows (a band from another frame).""" + s = width / 1206.0 + h = int(2622 * s) + pitch, gap = 336 * s, 58 * s + rows = [] + for y in range(h): + yy = y + if tear_at is not None and y >= tear_at: + yy = y + int(40 * s) + k = (yy - int(300 * s)) % pitch if yy >= 300 * s else None + row = bytearray([255] * (width * 3)) + if with_cards and k is not None and gap / 2 <= k < pitch - gap / 2 and 300 * s < yy < 2300 * s: + level = int(40 + (k - gap / 2) / (pitch - gap) * 150) + for x in range(width): + if int(1125 * s) - 10 <= x < int(1150 * s) + 10: + row[x * 3:x * 3 + 3] = bytes([level, level // 2, 200 - level // 2]) + else: + row[x * 3:x * 3 + 3] = bytes([60, 60, 200]) + rows.append(bytes(row)) + return png.Image(width, h, 3, rows) + + +def strip_image(counter, width=402 * 3): + """An image with GlassBench's strip showing `counter`.""" + s = width / 402.0 + h = int(900 * s) + rows = [bytearray([128] * (width * 3)) for _ in range(h)] + + def block(i, white): + x0, x1 = int((21 + 20 * i) * s), int((21 + 20 * i + 20) * s) + for y in range(int(120 * s), int(140 * s)): + for x in range(x0, x1): + rows[y][x * 3:x * 3 + 3] = bytes([255, 255, 255] if white else [0, 0, 0]) + block(0, True) + block(1, False) + for b in range(16): + block(b + 2, bool((counter >> b) & 1)) + return png.Image(width, h, 3, [bytes(r) for r in rows]) + + +def fake_scale_result(ns=(1, 2, 3), fps=(60, 58, 40), mix="all-busy"): + steps = [] + for n, f in zip(ns, fps): + sims = [{"idx": i, "role": "busy", "fps": float(f), "p50_ms": 16.7, "p95_ms": 20.0 + (60 - f), "p99_ms": 30.0, + "max_ms": 50.0, "over2x": 2, "longest_gap_s": 0.05, "input_p95_ms": 100.0, "guest_cpu_pct": 30.0, + "guest_rss_mb": 800.0} for i in range(n)] + steps.append({"n": n, "sims": sims, "gates_ok": True, "stable": True, + "system": {"qemu_cpu_cores": 4.0 + n, "gpu_busy_pct": 30.0 * n, "gpu_ms_per_frame": 6.8, + "guest": {"mem_used_mb": 8000.0 + 900 * n, "compressed_mb": 100.0, "sims_cpu_pct": 30.0 * n, + "other_cpu_pct": 20.0, "sims_rss_mb": 800.0 * n}}, + "stability": {"ok": True}, "gates": {}}) + from macbench import analysis + summary = analysis.summarize_scale(steps, ["smooth60: fps>=55, p95<=25", "smooth30: fps>=30, p95<=50"]) + return {"schema": 1, "tool": "macbox-bench", "kind": "scale", "id": "20260101-000000", "created": "2026-01-01T00:00:00Z", + "params": {}, "env": {"build_id": "test"}, "sections": [], "gates": {}, + "data": {"mix": mix, "scale_x": 1, "ns": list(ns), "nominal_fps": 60, "steps": steps, "summary": summary}} diff --git a/src/bench/tests/test_pure.py b/src/bench/tests/test_pure.py new file mode 100644 index 0000000..7241b79 --- /dev/null +++ b/src/bench/tests/test_pure.py @@ -0,0 +1,244 @@ +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from macbench import abrule, analysis, png, stats, strip, targets, tears # noqa: E402 +from tests import helpers # noqa: E402 + + +class Stats(unittest.TestCase): + def test_percentile_matches_simfps_rule(self): + v = list(range(1, 101)) + self.assertEqual(stats.percentile(v, 0.5), 50) # sorted[int(99*0.5)] = sorted[49] + self.assertEqual(stats.percentile(v, 0.95), 95) + self.assertEqual(stats.percentile(v, 0.99), 99) + self.assertEqual(stats.percentile(v, 1.0), 100) + self.assertIsNone(stats.percentile([], 0.5)) + self.assertEqual(stats.percentile([7], 0.99), 7) + + def test_frame_stats(self): + t = [i / 60 for i in range(121)] # 2 s at 60 fps + m = stats.frame_stats(t) + self.assertAlmostEqual(m["fps"], 60.0, delta=0.1) + self.assertEqual(m["over2x"], 0) + t2 = t[:60] + [x + 0.05 for x in t[60:]] # one 66 ms gap + m2 = stats.frame_stats(t2) + self.assertEqual(m2["over2x"], 1) + self.assertAlmostEqual(m2["longest_gap_s"], 0.067, places=2) + self.assertEqual(stats.frame_stats([1.0])["fps"], 0.0) + + def test_stretches(self): + t = [0, .01, .02, .03, 1.0, 1.01, 1.02] + self.assertEqual(stats.stretches(t, 250), [(0, .03, 4), (1.0, 1.02, 3)]) + + def test_jain_and_minmax(self): + self.assertAlmostEqual(stats.jain([60, 60, 60]), 1.0) + self.assertAlmostEqual(stats.jain([60, 0, 0]), 1 / 3) + self.assertAlmostEqual(stats.jain([60, 30]), 0.9, places=6) + self.assertEqual(stats.jain([0, 0]), 1.0) + self.assertAlmostEqual(stats.min_max_ratio([30, 60]), 0.5) + self.assertIsNone(stats.jain([])) + + def test_slope(self): + b, a, r2 = stats.slope([1, 2, 3, 4], [10, 20, 30, 40]) + self.assertAlmostEqual(b, 10) + self.assertAlmostEqual(a, 0) + self.assertAlmostEqual(r2, 1) + self.assertEqual(stats.slope([1], [1]), (None, None, None)) + self.assertEqual(stats.slope([2, 2], [1, 2]), (None, None, None)) + b, _, _ = stats.slope([1, 2, 3], [5, None, 9]) # None is skipped + self.assertAlmostEqual(b, 2) + + +class Strip(unittest.TestCase): + def test_decode_every_counter_and_scale(self): + for width in (402, 1206): + for c in (0, 1, 2, 255, 21845, 43690, 65535): + self.assertEqual(strip.read_counter_image(helpers.strip_image(c, width)), c, (width, c)) + + def test_no_strip_is_unreadable(self): + img = png.Image(402, 900, 3, [bytes([128] * 402 * 3)] * 900) + self.assertEqual(strip.read_counter_image(img), -1) + + def test_png_roundtrip_and_hash(self): + img = helpers.strip_image(1234, 402) + back = png.read_png(png.write_png(img)) + self.assertEqual(back.pixel_hash(), img.pixel_hash()) + self.assertEqual(strip.read_counter_image(back), 1234) + other = helpers.strip_image(1235, 402) + self.assertNotEqual(other.pixel_hash(), img.pixel_hash()) + + def test_analyze_clean_with_repeats_and_wrap(self): + s = [(0.00, 65534), (0.01, 65534), (0.02, 65535), (0.03, 0), (0.04, 0), (0.05, 1)] + a = strip.analyze_counters(s) + self.assertEqual([c for _, c in a["unique"]], [65534, 65535, 0, 1]) + self.assertEqual((a["repeats"], a["lost"], a["backwards"], a["jumps"], a["span_ticks"]), (2, 0, 0, 0, 3)) + self.assertTrue(strip.strip_gate(a, app_ticks=3)[0]) + + def test_lost_frames_are_counted_not_failed(self): + a = strip.analyze_counters([(0, 10), (0.03, 13), (0.05, 14)]) + self.assertEqual(a["lost"], 2) + self.assertTrue(strip.strip_gate(a)[0]) + + def test_out_of_order_fails(self): + a = strip.analyze_counters([(0, 10), (0.02, 12), (0.04, 11), (0.06, 13)]) + self.assertEqual(a["backwards"], 1) + ok, why = strip.strip_gate(a) + self.assertFalse(ok) + self.assertIn("out of order", why) + + def test_jump_and_garbage_fail(self): + a = strip.analyze_counters([(0, 10), (1, 5000)]) + self.assertEqual(a["jumps"], 1) + self.assertFalse(strip.strip_gate(a)[0]) + a = strip.analyze_counters([(0, 10), (0.1, 60)]) + self.assertFalse(strip.strip_gate(a, app_ticks=10)[0]) # counter moved 50, the app says 10 ticks + self.assertTrue(strip.strip_gate(a, app_ticks=50)[0]) + + def test_unreadable_share(self): + s = [(i / 60, i) for i in range(50)] + [(1.0, -1)] * 10 + self.assertFalse(strip.strip_gate(strip.analyze_counters(s))[0]) + + +class Tears(unittest.TestCase): + def test_clean_screenshot_has_no_tear(self): + r = tears.detect_tears(helpers.card_image(603)) + self.assertTrue(r["applicable"]) + self.assertFalse(r["tear"], r) + + def test_torn_screenshot_is_found(self): + r = tears.detect_tears(helpers.card_image(603, tear_at=700)) + self.assertTrue(r["tear"], r) + + def test_no_cards_is_not_applicable(self): + r = tears.detect_tears(helpers.card_image(603, with_cards=False)) + self.assertFalse(r["applicable"]) + + +class Latency(unittest.TestCase): + def raw(self, flips, rate=60, secs=10): + """frames at `rate`; the marker (luma) flips at each time in `flips`""" + out, lum = [], 255 + for i in range(int(secs * rate)): + t = 100.0 + i / rate + if flips and t >= flips[0]: + lum = 0 if lum == 255 else 255 + flips = flips[1:] + out.append((t, i, lum)) + return out + + def test_latency_and_misses(self): + raw = self.raw([102.080, 105.250]) # answers 80 ms and 250 ms after the injections + inj = [(102.0, 102.12), (105.0, 105.14), (108.0, 108.1)] # the third is never answered + r = analysis.probe_latencies(raw, inj, timeout_s=2.0) + self.assertEqual(r["misses"], 1) + self.assertAlmostEqual(r["latencies_ms"][0], 83, delta=17) + self.assertAlmostEqual(r["latencies_ms"][1], 250, delta=17) + self.assertEqual(r["latencies_ms"][2], 2000.0) # a miss counts at the timeout + self.assertEqual(r["max_ms"], 2000.0) + self.assertAlmostEqual(r["call_p50_ms"], 120, delta=1) + + def test_static_screen_uses_the_start_line(self): + raw = [(100.0, -1, 255), (103.07, -1, 0)] # no frames between: a static screen + r = analysis.probe_latencies(raw, [(103.0, 103.1)]) + self.assertEqual(r["misses"], 0) + self.assertAlmostEqual(r["latencies_ms"][0], 70, delta=1) + + def test_sim_metrics_busy_has_frames_and_probe(self): + raw = [(100 + i / 60, i % 65536, 255) for i in range(300)] + m = analysis.sim_metrics(0, "busy", raw, [], app_ticks=300) + self.assertAlmostEqual(m["fps"], 60, delta=0.5) + self.assertTrue(m["strip_gate"]["ok"]) + s = analysis.sim_metrics(1, "static", raw, [(101.0, 101.1)]) + self.assertNotIn("fps", s) + self.assertEqual(s["input_misses"], 1) + + +class Targets(unittest.TestCase): + def sims(self, fps, p95=20.0, n=3): + return [{"idx": i, "role": "busy", "fps": f, "p95_ms": p95} for i, f in enumerate(fps)] + + def test_parse(self): + t = targets.parse_target("smooth: fps>=55, p95<=25") + self.assertEqual(t["conds"], [("fps", ">=", 55.0), ("p95", "<=", 25.0)]) + with self.assertRaises(ValueError): + targets.parse_target("x: bogus>=1") + with self.assertRaises(ValueError): + targets.parse_target("no colon") + + def test_every_sim_must_pass(self): + t = targets.parse_target("t: fps>=55") + self.assertTrue(targets.evaluate(t, self.sims([60, 58, 56]))["pass"]) + r = targets.evaluate(t, self.sims([60, 58, 40])) + self.assertFalse(r["pass"]) + self.assertEqual(r["failing"][0]["sim"], 2) + + def test_unknown_when_nothing_to_judge(self): + t = targets.parse_target("t: input_p95<=150") + self.assertIsNone(targets.evaluate(t, self.sims([60]))["pass"]) + + def test_max_sustainable_stops_at_first_fail(self): + steps = [{"n": n, "sims": self.sims([f] * n), "gates_ok": True} for n, f in ((1, 60), (2, 58), (3, 50), (4, 60))] + t = targets.parse_target("t: fps>=55") + r = targets.max_sustainable(steps, t) + self.assertEqual((r["max_n"], r["first_fail"]), (2, 3)) # 4 passes but 3 failed: not sustainable + self.assertTrue(r["evaluated"][4]) + + def test_gates_and_instability_veto(self): + steps = [{"n": 1, "sims": self.sims([60]), "gates_ok": True}, + {"n": 2, "sims": self.sims([60, 60]), "gates_ok": False}] + self.assertEqual(targets.max_sustainable(steps, targets.parse_target("t: fps>=55"))["max_n"], 1) + steps = [{"n": 1, "sims": self.sims([60]), "stable": False}] + self.assertIsNone(targets.max_sustainable(steps, targets.parse_target("t: fps>=55"))["max_n"]) + + def test_default_targets_parse(self): + for spec in targets.DEFAULT_TARGETS: + targets.parse_target(spec) + + def test_summary_and_fairness(self): + doc = helpers.fake_scale_result() + sm = doc["data"]["summary"] + self.assertEqual(sm["max_n"]["smooth60"]["max_n"], 2) + self.assertEqual(sm["max_n"]["smooth30"]["max_n"], 3) + self.assertAlmostEqual(doc["data"]["steps"][0]["fairness"]["jain"], 1.0) + sc = sm["standing_cost"] + self.assertAlmostEqual(sc["qemu_cpu_cores"]["per_sim"], 1.0, places=2) + self.assertAlmostEqual(sc["guest_mem_used_mb"]["per_sim"], 900.0, places=1) + + +class ABRule(unittest.TestCase): + def test_adopt_clear_win(self): + r = abrule.decide(100, 110, 100, 111) # +10, +11 + self.assertEqual(r["decision"], "ADOPT") + + def test_close_below_bar(self): + self.assertEqual(abrule.decide(100, 102, 100, 103)["decision"], "CLOSE") + + def test_close_when_pairs_disagree(self): + r = abrule.decide(100, 112, 100, 98) + self.assertEqual(r["decision"], "CLOSE") + self.assertTrue(any("disagree" in w for w in r["why"])) + + def test_lower_is_better(self): + self.assertEqual(abrule.decide(10, 8, 10, 8.5, higher_better=False)["decision"], "ADOPT") + self.assertEqual(abrule.decide(10, 12, 10, 12, higher_better=False)["decision"], "CLOSE") + + def test_tail_worsening_closes(self): + r = abrule.decide(100, 110, 100, 110, tails={"p95": (20, 25, 20, 26, False)}) + self.assertEqual(r["decision"], "CLOSE") + r = abrule.decide(100, 110, 100, 110, tails={"p95": (20, 20.2, 20, 20.3, False)}) + self.assertEqual(r["decision"], "ADOPT") + + def test_missing_values_close(self): + self.assertEqual(abrule.decide(0, 1, 100, 110)["decision"], "CLOSE") + + def test_decision_line(self): + r = abrule.decide(100, 110, 100, 111) + line = abrule.decision_line("knob=1", "distinct fps", ((100, 100), (110, 111)), r, 0.04) + self.assertIn("ADOPT", line) + self.assertIn("bar 4.0%", line) + + +if __name__ == "__main__": + unittest.main() From 8af434fa676e351785834c4bc171f2207c45877f Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:12:28 +0200 Subject: [PATCH 04/23] bench: the guest helper (own device set, simctl always with --set, per-simulator CPU/memory, input probes, screenshots, lifecycle) with a stub for dry runs; host probes (QEMU thread CPU split, GPU fdinfo, cgroup, PSI), gates, run safety --- src/bench/guest/__init__.py | 0 src/bench/guest/benchguest.py | 783 +++++++++++++++++++++++++++++ src/bench/guest/benchguest_stub.py | 240 +++++++++ src/bench/macbench/analysis.py | 51 ++ src/bench/macbench/gates.py | 148 ++++++ src/bench/macbench/hostprobe.py | 221 ++++++++ src/bench/macbench/runctx.py | 168 +++++++ src/bench/macbench/transport.py | 122 +++++ 8 files changed, 1733 insertions(+) create mode 100644 src/bench/guest/__init__.py create mode 100644 src/bench/guest/benchguest.py create mode 100644 src/bench/guest/benchguest_stub.py create mode 100644 src/bench/macbench/gates.py create mode 100644 src/bench/macbench/hostprobe.py create mode 100644 src/bench/macbench/runctx.py create mode 100644 src/bench/macbench/transport.py diff --git a/src/bench/guest/__init__.py b/src/bench/guest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/bench/guest/benchguest.py b/src/bench/guest/benchguest.py new file mode 100644 index 0000000..a9e1764 --- /dev/null +++ b/src/bench/guest/benchguest.py @@ -0,0 +1,783 @@ +#!/usr/bin/env python3 +"""benchguest: the bench's helper inside macOS (run by `macbox bench`; `xcrun python3`, standard library only, +Python 3.9 compatible). Every subcommand prints ONE JSON object on its last stdout line. + +Every simulator command goes through simctl() below, which always passes --set +and refuses a set without the bench's marker file: the pool and Simlock's devices are never touched. + + info | init-set | ensure-type | create | boot | prepare | settle | calibrate | measure | shots | + cycle | teardown | memcheck | build +""" +import argparse +import base64 +import json +import os +import plistlib +import re +import shutil +import subprocess +import sys +import tempfile +import threading +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.dirname(HERE)) +BUNDLE_ID = "dev.macbox.glassbench" +MARKER = ".macbox-bench-set" +PROBE_POINT = "36,175" # the input marker's centre, in points (GlassBench -probe) +SET = None +STUB = os.environ.get("BENCHGUEST_STUB") +CALLS = [] + + +# --- primitives (replaced by benchguest_stub.py under BENCHGUEST_STUB) --------------------------------- +def run(argv, timeout=120, env=None): + """(returncode, stdout, stderr, seconds). A timeout kills the process and returns rc 124.""" + t0 = time.time() + try: + p = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, env=env) + return p.returncode, p.stdout, p.stderr, time.time() - t0 + except subprocess.TimeoutExpired as e: + return 124, (e.stdout or b"").decode() if isinstance(e.stdout, bytes) else (e.stdout or ""), "timeout", time.time() - t0 + except OSError as e: + return 127, "", str(e), time.time() - t0 + + +def spawn_simfps(udid, secs, label, strip, start_at, raw_path): + """Starts simfps; returns an object whose .wait(timeout) -> (json dict or None, error or None).""" + env = dict(os.environ, SIMFPS_SET=SET, SIMFPS_START_AT="%.3f" % start_at, SIMFPS_STAMPS=raw_path, + SIMFPS_PROBE=PROBE_POINT) + argv = [os.path.join(HERE, "..", "simfps"), udid, str(secs), label] + (["--strip"] if strip else []) + p = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env) + + class H: + def wait(self, timeout): + try: + out, err = p.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + p.kill() + p.communicate() + return None, "simfps hung (killed after %ds)" % timeout + try: + return json.loads(out.strip().splitlines()[-1]), None + except (ValueError, IndexError): + return None, (err or out or "no output").strip()[-200:] + + def kill(self): + try: + p.kill() + except OSError: + pass + return H() + + +def ps_text(): + return run(["ps", "-axww", "-o", "pid=,ppid=,rss=,time=,command="], 30)[1] + + +def vmstat_text(): + return run(["vm_stat"], 15)[1] + + +def sysctl(name): + return run(["sysctl", "-n", name], 15)[1].strip() + + +def now(): + return time.time() + + +if STUB: + from guest import benchguest_stub as _stub # noqa: E402 + _stub.install(sys.modules[__name__]) + + +# --- simctl ---------------------------------------------------------------------------------------- +class GuestError(Exception): + pass + + +def check_set(): + if not SET: + raise GuestError("--set is required") + if not os.path.exists(os.path.join(SET, MARKER)): + raise GuestError("not a bench device set (no %s): refusing" % MARKER) + + +def simctl(*args, timeout=180): + """xcrun simctl --set ...; the only way this file starts simctl.""" + check_set() + argv = ["xcrun", "simctl", "--set", SET] + [str(a) for a in args] + CALLS.append(argv) + return run(argv, timeout) + + +def simctl_json(*args): + rc, out, err, _ = simctl(*args, "-j") + if rc != 0: + raise GuestError("simctl %s: %s" % (" ".join(map(str, args)), err.strip()[:200])) + return json.loads(out) + + +def devices(): + out = [] + for rt, devs in simctl_json("list", "devices").get("devices", {}).items(): + for d in devs: + d = dict(d) + d["runtime"] = rt + out.append(d) + return out + + +# --- parsers (unit-tested) ------------------------------------------------------------------------- +def parse_vm_stat(text): + """vm_stat -> {"page": bytes, "free": pages, ...}. Keys lower-cased, spaces to underscores.""" + d = {} + m = re.search(r"page size of (\d+) bytes", text) + d["page"] = int(m.group(1)) if m else 16384 + for line in text.splitlines(): + m = re.match(r"^\"?([A-Za-z ()-]+?)\"?:\s+(\d+)\.?\s*$", line) + if m: + d[re.sub(r"[^a-z0-9]+", "_", m.group(1).lower()).strip("_")] = int(m.group(2)) + return d + + +def mem_summary(vm): + """The fields a report needs, in MB, from parse_vm_stat().""" + p = vm.get("page", 16384) / 1048576.0 + g = lambda k: vm.get(k, 0) + used = (g("pages_active") + g("pages_wired_down") + g("pages_occupied_by_compressor")) * p + return {"free_mb": round((g("pages_free") + g("pages_speculative")) * p, 1), "used_mb": round(used, 1), + "compressed_mb": round(g("pages_occupied_by_compressor") * p, 1), + "swapins": g("swapins"), "swapouts": g("swapouts"), "pageins": g("pageins"), "pageouts": g("pageouts")} + + +def parse_cputime(s): + """ps TIME: [d-][h:]m:s[.cc] -> seconds""" + days = 0 + if "-" in s: + d, s = s.split("-", 1) + days = int(d) + parts = s.split(":") + secs = 0.0 + for p in parts: + secs = secs * 60 + float(p) + return days * 86400 + secs + + +def parse_ps(text): + """[{pid, ppid, rss_kb, cpu_s, cmd}]""" + out = [] + for line in text.splitlines(): + p = line.split(None, 4) + if len(p) < 5: + continue + try: + out.append({"pid": int(p[0]), "ppid": int(p[1]), "rss_kb": int(p[2]), "cpu_s": parse_cputime(p[3]), "cmd": p[4]}) + except ValueError: + pass + return out + + +def sim_trees(procs, udids): + """{udid: set of pids}: processes whose command line names the simulator's UDID (launchd_sim, SimMetalHost, + ...) and every descendant of them. A process that names no simulator is nobody's.""" + by_ppid = {} + for p in procs: + by_ppid.setdefault(p["ppid"], []).append(p["pid"]) + out = {} + for u in udids: + roots = [p["pid"] for p in procs if u in p["cmd"]] + seen, stack = set(roots), list(roots) + while stack: + for c in by_ppid.get(stack.pop(), []): + if c not in seen: + seen.add(c) + stack.append(c) + out[u] = seen + return out + + +def attribute(before, after, udids, secs): + """Per simulator and in total: CPU % of one core over the window (from the processes' CPU time deltas) and RSS MB + at the end. A process that started or ended inside the window counts from its first sample or not at all: + short-lived processes are undercounted (declared in the README).""" + b = {p["pid"]: p for p in before} + trees = sim_trees(after, udids) + per = {} + for u in udids: + cpu = sum(max(0.0, p["cpu_s"] - b[p["pid"]]["cpu_s"]) for p in after if p["pid"] in trees[u] and p["pid"] in b) + rss = sum(p["rss_kb"] for p in after if p["pid"] in trees[u]) / 1024.0 + per[u] = {"cpu_pct": round(100 * cpu / secs, 1), "rss_mb": round(rss, 1), "procs": len(trees[u])} + tot = sum(max(0.0, p["cpu_s"] - b[p["pid"]]["cpu_s"]) for p in after if p["pid"] in b) + mine = set().union(*trees.values()) if trees else set() + sims_cpu = sum(max(0.0, p["cpu_s"] - b[p["pid"]]["cpu_s"]) for p in after if p["pid"] in mine and p["pid"] in b) + return per, {"total_cpu_pct": round(100 * tot / secs, 1), "sims_cpu_pct": round(100 * sims_cpu / secs, 1), + "other_cpu_pct": round(100 * (tot - sims_cpu) / secs, 1), + "sims_rss_mb": round(sum(v["rss_mb"] for v in per.values()), 1)} + + +def pressure_level(): + s = sysctl("kern.memorystatus_vm_pressure_level") + return int(s) if s.isdigit() else None + + +def parse_pace(text): + """GlassBench's Documents/pace.json (one JSON object per line) -> [(at, counter, fps, ticks)]""" + out = [] + for line in text.splitlines(): + try: + d = json.loads(line) + out.append((float(d["at"]), int(d["counter"]), float(d.get("fps", 0)), int(d.get("ticks", 0)))) + except (ValueError, KeyError, TypeError): + pass + return out + + +def read_pace(udid): + rc, out, _, _ = simctl("get_app_container", udid, BUNDLE_ID, "data") + if rc != 0: + return [] + try: + with open(os.path.join(out.strip(), "Documents", "pace.json")) as f: + return parse_pace(f.read()) + except OSError: + return [] + + +def list_ips(): + names = set() + for d in (os.path.expanduser("~/Library/Logs/DiagnosticReports"), "/Library/Logs/DiagnosticReports"): + try: + for n in os.listdir(d): + if n.endswith((".ips", ".crash")): + names.add(n) + except OSError: + pass + return names + + +# --- injection ------------------------------------------------------------------------------------- +def inject(method, udid, state, cmd_tpl=None): + """One input event; returns the command's rc. state: a per-simulator dict for alternating methods.""" + if method == "openurl": + return simctl("openurl", udid, "glassbench://toggle", timeout=20)[0] + if method == "appearance": + state["dark"] = not state.get("dark", False) + return simctl("ui", udid, "appearance", "dark" if state["dark"] else "light", timeout=20)[0] + if method == "cmd" and cmd_tpl: + return run(["/bin/sh", "-c", cmd_tpl.replace("{udid}", udid)], 20)[0] + return 2 + + +def probe_loop(udid, method, cmd_tpl, t_first, period, t_stop, out, stop_evt): + state = {} + t = t_first + while t < t_stop and not stop_evt.is_set(): + d = t - now() + if d > 0: + stop_evt.wait(d) + if stop_evt.is_set(): + break + t0 = now() + rc = inject(method, udid, state, cmd_tpl) + out.append([t0, now(), rc]) + t += period + + +# --- actions of the agent mix ---------------------------------------------------------------------- +def agent_loop(udid, idx, n, t_go, t_stop, period, out, stop_evt): + """launch app -> scroll (the app animates by itself) -> screenshot -> terminate -> idle, staggered by sim.""" + t = t_go + (idx * period / max(n, 1)) + tmp = tempfile.mkdtemp(prefix="benchshot") + try: + while t + period <= t_stop + 0.5 and not stop_evt.is_set(): + d = t - now() + if d > 0 and stop_evt.wait(d): + break + t0 = now() + rc = simctl("launch", udid, BUNDLE_ID, "-probe", timeout=60)[0] + out.append({"op": "launch", "t0": t0, "t1": now(), "rc": rc}) + stop_evt.wait(0.45 * period) + t0 = now() + rc = simctl("io", udid, "screenshot", "--type=png", os.path.join(tmp, "s.png"), timeout=60)[0] + out.append({"op": "screenshot", "t0": t0, "t1": now(), "rc": rc}) + stop_evt.wait(0.15 * period) + t0 = now() + rc = simctl("terminate", udid, BUNDLE_ID, timeout=30)[0] + out.append({"op": "terminate", "t0": t0, "t1": now(), "rc": rc}) + t += period + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +# --- subcommands ----------------------------------------------------------------------------------- +def out_json(obj): + sys.stdout.write(json.dumps(obj) + "\n") + sys.stdout.flush() + + +def cmd_info(a): + sw = run(["sw_vers"], 10)[1] + ver = re.search(r"ProductVersion:\s*(\S+)", sw) + xc = run(["xcodebuild", "-version"], 30)[1].replace("\n", " ").strip() + rts = [] + try: + rts = [{"name": r.get("name"), "version": r.get("version"), "build": r.get("buildversion")} + for r in json.loads(run(["xcrun", "simctl", "list", "runtimes", "-j"], 30)[1]).get("runtimes", []) + if r.get("isAvailable", True)] + except ValueError: + pass + vm = parse_vm_stat(vmstat_text()) + out_json({"macos": ver.group(1) if ver else None, "xcode": xc, "runtimes": rts, "arch": run(["uname", "-m"], 5)[1].strip(), + "cpus": sysctl("hw.ncpu"), "mem_mb": int(sysctl("hw.memsize") or 0) // 1048576, "mem": mem_summary(vm), + "has_simfps": os.path.exists(os.path.join(HERE, "..", "simfps")), + "has_app": os.path.isdir(os.path.join(HERE, "..", "GlassBench.app")), "stub": bool(STUB)}) + + +def cmd_init_set(a): + global SET + os.makedirs(a.set, exist_ok=True) + with open(os.path.join(a.set, MARKER), "w") as f: + f.write("macbox bench device set\n") + SET = a.set + out_json({"ok": True, "devices": len(devices())}) + + +def stock_pro_type(): + """(runtime id, runtime version, 'iPhone NN Pro' type name, id, bundle): macbox-guest's choice of the stock type.""" + best = None + rts = json.loads(run(["xcrun", "simctl", "list", "runtimes", "-j"], 30)[1]).get("runtimes", []) + ios = [r for r in rts if r.get("platform", "iOS") == "iOS" and r.get("isAvailable", True)] + if not ios: + raise GuestError("no iOS Simulator runtime") + ios.sort(key=lambda r: [int(x) for x in re.findall(r"\d+", r.get("version", "0"))]) + rt = ios[-1] + for d in rt.get("supportedDeviceTypes", []): + m = re.match(r"^iPhone (\d+) Pro$", d["name"]) + if m and d["identifier"].startswith("com.apple.") and (best is None or int(m.group(1)) > best[0]): + best = (int(m.group(1)), d) + if not best: + raise GuestError("the runtime supports no 'iPhone NN Pro'") + d = best[1] + return rt["identifier"], rt.get("version"), d["name"], d["identifier"], d.get("bundlePath", "") + + +def cmd_ensure_type(a): + """The device type for a pixel scale: the stock iPhone NN Pro at 3x, a macbox type (the same one `macbox pool` + makes, same identifier, reused when it exists) at 1x and 2x. Making a NEW type needs CoreSimulatorService to + restart, which shuts down every simulator: done only with --allow-restart and only when none is booted.""" + rt, rtv, name, stock_id, bundle = stock_pro_type() + scale = int(a.scale) + res = {"runtime": rt, "runtime_version": rtv, "stock_name": name, "scale": scale} + if scale == 3: + res.update({"type": stock_id, "created": False, "needs_restart": False}) + return out_json(res) + tid = "dev.macbox.CoreSimulator.SimDeviceType.%s-%dx" % (stock_id.rsplit(".", 1)[-1], scale) + known = {t["identifier"] for t in json.loads(run(["xcrun", "simctl", "list", "devicetypes", "-j"], 30)[1]).get("devicetypes", [])} + if tid in known: + res.update({"type": tid, "created": False, "needs_restart": False}) + return out_json(res) + types_dir = os.path.expanduser("~/Library/Developer/CoreSimulator/Profiles/DeviceTypes") + dest = os.path.join(types_dir, "%s (%dx).simdevicetype" % (os.path.basename(bundle)[:-len(".simdevicetype")], scale)) + if not os.path.isdir(bundle): + raise GuestError("no device type bundle at the stock type's path") + os.makedirs(types_dir, exist_ok=True) + tmp = dest + ".tmp" + shutil.rmtree(tmp, ignore_errors=True) + shutil.copytree(bundle, tmp, symlinks=True) + pp = os.path.join(tmp, "Contents", "Resources", "profile.plist") + with open(pp, "rb") as f: + prof = plistlib.load(f) + s = int(prof["mainScreenScale"]) + pw, ph = int(prof["mainScreenWidth"]) // s, int(prof["mainScreenHeight"]) // s + prof["mainScreenScale"] = float(scale) if isinstance(prof["mainScreenScale"], float) else scale + prof["mainScreenWidth"], prof["mainScreenHeight"] = pw * scale, ph * scale + prof["mainScreenWidthDPI"] = (int(prof["mainScreenWidthDPI"]) * scale + s // 2) // s + prof["mainScreenHeightDPI"] = (int(prof["mainScreenHeightDPI"]) * scale + s // 2) // s + prof.pop("createByDefaultForRuntimeVersions", None) # else CoreSimulator would make and pair devices of it itself + prof.pop("pairByDefaultForRuntimeVersions", None) + with open(pp, "wb") as f: + plistlib.dump(prof, f) + ip = os.path.join(tmp, "Contents", "Info.plist") + with open(ip, "rb") as f: + info = plistlib.load(f) + info["CFBundleIdentifier"] = tid + info["CFBundleName"] = "%s (%dx)" % (name, scale) + with open(ip, "wb") as f: + plistlib.dump(info, f) + shutil.rmtree(dest, ignore_errors=True) + os.rename(tmp, dest) + res.update({"type": tid, "created": True, "needs_restart": True}) + if a.allow_restart: + booted = [d for d in devices() if d.get("state") == "Booted"] + all_booted = run(["xcrun", "simctl", "list", "devices", "booted"], 30)[1] + if booted or re.search(r"\(Booted\)", all_booted): + res["restart"] = "refused: a simulator is booted" + else: + run(["pkill", "-f", "CoreSimulatorService"], 10) + time.sleep(3) + res["restart"] = "done" + res["needs_restart"] = False + out_json(res) + + +def cmd_create(a): + made = [] + for i in range(a.start, a.start + a.n): + name = "%s-%d" % (a.prefix, i) + rc, out, err, secs = simctl("create", name, a.type, a.runtime) + if rc != 0: + raise GuestError("create %s: %s" % (name, err.strip()[:200])) + made.append({"idx": i, "udid": out.strip().splitlines()[-1], "create_s": round(secs, 2)}) + out_json({"sims": made}) + + +def _boot_one(udid, res, i): + t0 = now() + rc, _, err, _ = simctl("boot", udid, timeout=300) + t1 = now() + ok = rc == 0 or "current state: Booted" in err + rc2 = simctl("bootstatus", udid, timeout=600)[0] if ok else rc + res[i] = {"udid": udid, "ok": ok and rc2 == 0, "boot_cmd_s": round(t1 - t0, 2), "usable_s": round(now() - t0, 2), + "err": None if ok and rc2 == 0 else err.strip()[-120:]} + + +def cmd_boot(a): + us = a.udids.split(",") + res = [None] * len(us) + t0 = now() + if a.parallel: + th = [threading.Thread(target=_boot_one, args=(u, res, i)) for i, u in enumerate(us)] + [t.start() for t in th] + [t.join(700) for t in th] + else: + for i, u in enumerate(us): + _boot_one(u, res, i) + out_json({"sims": [r or {"ok": False, "err": "no result"} for r in res], "all_s": round(now() - t0, 2)}) + + +def cmd_prepare(a): + """status bar fixed, app installed, launched with the arguments of the simulator's role.""" + us, roles = a.udids.split(","), a.roles.split(",") + app = os.path.join(HERE, "..", "GlassBench.app") + res = [] + for u, role in zip(us, roles): + t0 = now() + rc_i = simctl("install", u, app, timeout=120)[0] if a.install else 0 + simctl("status_bar", u, "override", "--time", "9:41", "--batteryState", "charged", "--batteryLevel", "100", + "--cellularBars", "4", "--wifiBars", "3", "--operatorName", "", timeout=30) + if a.quiesce: + simctl("spawn", u, "launchctl", "disable", "system/com.apple.mediaanalysisd", timeout=30) + simctl("spawn", u, "launchctl", "bootout", "system/com.apple.mediaanalysisd", timeout=30) + simctl("terminate", u, BUNDLE_ID, timeout=30) + args = ["-probe"] + (["-static"] if role == "static" else []) + if role == "agent": + args = [] + rc_l = 0 if role == "agent" else simctl("launch", u, BUNDLE_ID, *args, timeout=60)[0] + res.append({"udid": u, "role": role, "install_rc": rc_i, "launch_rc": rc_l, "s": round(now() - t0, 2)}) + out_json({"sims": res}) + + +def parse_name_cpu(text): + return text + + +def bg_cpu_pct(secs=3.0): + """Everything not part of the load: CPU % of one core used by processes other than the renderers, the app and + the observer (the settle condition: the guest has stopped doing its own housekeeping).""" + skip = re.compile(r"(SimMetalHost|backboardd|GlassBench|SimRenderServer|kernel_task|simfps|WindowServer|python3|/ps$|/top$)") + a = parse_ps(ps_text()) + t0 = now() + time.sleep(secs) + b = parse_ps(ps_text()) + dt = now() - t0 + am = {p["pid"]: p for p in a} + tot = sum(max(0.0, p["cpu_s"] - am[p["pid"]]["cpu_s"]) for p in b if p["pid"] in am and not skip.search(p["cmd"].split(" ", 1)[0])) + return round(100 * tot / dt, 1) + + +def cmd_settle(a): + t0, series = now(), [] + while True: + v = bg_cpu_pct(a.step) + series.append(v) + if v < a.bg_pct or now() - t0 > a.max_s: + break + out_json({"settled": series[-1] < a.bg_pct, "bg_pct": series[-1], "waited_s": round(now() - t0, 1), "series": series[-8:]}) + + +def run_window(a, udids, roles, mix, secs, t_go, tmp, probe_method, probe_every, cmd_tpl, strip_for_all=True): + """simfps on each simulator, the probe and agent threads, and the guest's samples around the window.""" + raws, handles, stop_evt = {}, {}, threading.Event() + if mix != "idle": + for i, u in enumerate(udids): + raws[u] = os.path.join(tmp, "raw-%d.txt" % i) + handles[u] = spawn_simfps(u, secs, "bench-%d" % i, roles[i] != "static", t_go, raws[u]) + ips0, ps0, vm0, lvl0 = list_ips(), None, None, pressure_level() + d = t_go - now() + if d > 0: + time.sleep(d) + ps0, vm0 = parse_ps(ps_text()), parse_vm_stat(vmstat_text()) + t_start = now() + t_stop = t_go + secs + inj, acts, threads = {u: [] for u in udids}, {u: [] for u in udids}, [] + lvls = [] + if mix in ("all-busy", "one-busy"): + targets = [u for u, r in zip(udids, roles) if mix == "all-busy" or r == "static"] + for k, u in enumerate(targets): + t = threading.Thread(target=probe_loop, args=(u, probe_method, cmd_tpl, t_go + 1.0 + k * probe_every / max(len(targets), 1), + probe_every, t_stop - 2.2, inj[u], stop_evt)) + threads.append(t) + if mix == "agent": + for i, u in enumerate(udids): + threads.append(threading.Thread(target=agent_loop, args=(u, i, len(udids), t_go, t_stop, a.agent_period, acts[u], stop_evt))) + + def sample_pressure(): + while not stop_evt.is_set() and now() < t_stop: + lvls.append(pressure_level()) + stop_evt.wait(2.0) + threads.append(threading.Thread(target=sample_pressure)) + [t.start() for t in threads] + res = {} + try: + for u, h in handles.items(): + res[u] = h.wait(max(5, t_stop - now() + 40)) + while now() < t_stop: + time.sleep(0.2) + finally: + stop_evt.set() + [t.join(30) for t in threads] + for h in handles.values(): + h.kill() + ps1, vm1 = parse_ps(ps_text()), parse_vm_stat(vmstat_text()) + t_end = now() + per, tot = attribute(ps0, ps1, udids, max(t_end - t_start, 0.001)) + st = {d["udid"]: d.get("state") for d in devices()} + sims = [] + for i, u in enumerate(udids): + sj, err = res.get(u, (None, None)) + raw = "" + if u in raws and os.path.exists(raws[u]): + with open(raws[u]) as f: + raw = f.read() + sims.append({"udid": u, "role": roles[i], "simfps": sj, "simfps_error": err, "raw": raw, "injections": inj[u], + "actions": acts[u], "proc": per.get(u), "state_after": st.get(u), + "pace": read_pace(u) if mix != "idle" and roles[i] != "static" else []}) + m0, m1 = mem_summary(vm0), mem_summary(vm1) + return {"t_go": t_go, "t_start": t_start, "t_end": t_end, "secs": round(t_end - t_start, 2), "sims": sims, + "guest": dict(tot, mem=m1, mem_before=m0, swapins_d=m1["swapins"] - m0["swapins"], swapouts_d=m1["swapouts"] - m0["swapouts"], + pageouts_d=m1["pageouts"] - m0["pageouts"], pressure_before=lvl0, + pressure_max=max([x for x in lvls + [pressure_level()] if x] or [None]) if any(lvls) else pressure_level()), + "new_ips": sorted(list_ips() - ips0), "procs_total": len(ps1)} + + +def cmd_measure(a): + udids, roles = a.udids.split(","), a.roles.split(",") + tmp = tempfile.mkdtemp(prefix="benchwin") + try: + t_go = a.start_at or now() + 3 + len(udids) + out_json(run_window(a, udids, roles, a.mix, a.secs, t_go, tmp, a.probe_method, a.probe_every, a.probe_cmd)) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def cmd_calibrate(a): + """Which injection method is answered? 3 injections of each method on one simulator, watched by simfps.""" + from macbench import analysis + tmp = tempfile.mkdtemp(prefix="benchcal") + res = {} + try: + for method in a.methods.split(","): + t_go = now() + 3 + raw = os.path.join(tmp, "raw.txt") + h = spawn_simfps(a.udid, 9, "cal", False, t_go, raw) + inj, state = [], {} + for k in range(3): + d = t_go + 1.5 + k * 2.4 - now() + if d > 0: + time.sleep(d) + t0 = now() + rc = inject(method, a.udid, state, a.probe_cmd) + inj.append([t0, now(), rc]) + sj, err = h.wait(40) + txt = open(raw).read() if os.path.exists(raw) else "" + r = analysis.probe_latencies(analysis.parse_raw(txt), [(x[0], x[1]) for x in inj], 2.0) + res[method] = {"answered": r["n"] - r["misses"], "n": r["n"], "p50_ms": r["p50_ms"], "call_p50_ms": r["call_p50_ms"], + "error": err} + out_json({"methods": res}) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def cmd_shots(a): + """K screenshots of each simulator: pixel hash, tear check, strip counter. PNGs stay here.""" + from macbench import png, strip, tears + tmp = tempfile.mkdtemp(prefix="benchshots") + res = [] + try: + for u in a.udids.split(","): + one = {"udid": u, "shots": []} + for k in range(a.count): + p = os.path.join(tmp, "%s-%d.png" % (u[:8], k)) + rc = simctl("io", u, "screenshot", "--type=png", p, timeout=60)[0] + if rc != 0 or not os.path.exists(p): + one["shots"].append({"ok": False}) + continue + with open(p, "rb") as f: + img = png.read_png(f.read()) + t = tears.detect_tears(img) + one["shots"].append({"ok": True, "w": img.w, "h": img.h, "hash": img.pixel_hash(), + "counter": strip.read_counter_image(img), "tear": t["tear"], "tear_applicable": t["applicable"]}) + os.unlink(p) + time.sleep(0.4) + res.append(one) + out_json({"sims": res}) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def cmd_cycle(a): + """One lifecycle of a simulator: create, boot, install, launch, first app frame, shut down, resume + (boot again), first frame again, delete. Timings in seconds.""" + from macbench import analysis + tmp = tempfile.mkdtemp(prefix="benchcycle") + t = {} + udid = None + try: + name = "%s-cycle" % a.prefix + t0 = now() + udid = simctl("create", name, a.type, a.runtime)[1].strip().splitlines()[-1] + t["create_s"] = round(now() - t0, 2) + for phase in ("cold", "resume"): + if phase == "resume": + t0 = now() + simctl("shutdown", udid, timeout=120) + t["shutdown_s"] = round(now() - t0, 2) + t0 = now() + simctl("boot", udid, timeout=300) + simctl("bootstatus", udid, timeout=600) + t["%s_boot_s" % phase] = round(now() - t0, 2) + if phase == "cold": + t0 = now() + simctl("install", udid, os.path.join(HERE, "..", "GlassBench.app"), timeout=120) + t["install_s"] = round(now() - t0, 2) + raw = os.path.join(tmp, "raw-%s.txt" % phase) + h = spawn_simfps(udid, 14, "cycle", True, now() + 2, raw) + time.sleep(2.5) + t_l = now() + simctl("launch", udid, BUNDLE_ID, "-probe", timeout=60) + t["%s_launch_call_s" % phase] = round(now() - t_l, 2) + sj, err = h.wait(60) + frames = analysis.parse_raw(open(raw).read()) if os.path.exists(raw) else [] + first = next((x[0] for x in frames if x[0] >= t_l and x[1] >= 0 and x[1] != 65535), None) + t["%s_first_frame_s" % phase] = None if first is None else round(first - t_l, 2) + t0 = now() + simctl("shutdown", udid, timeout=120) + simctl("delete", udid, timeout=120) + t["delete_s"] = round(now() - t0, 2) + udid = None + out_json({"timings": t}) + finally: + if udid: + simctl("shutdown", udid, timeout=60) + simctl("delete", udid, timeout=60) + shutil.rmtree(tmp, ignore_errors=True) + + +def cmd_teardown(a): + """Shut down and delete EVERY device in the bench set (it is the bench's own, marker file checked).""" + simctl("shutdown", "all", timeout=300) + simctl("delete", "all", timeout=300) + left = [d["udid"] for d in devices()] + booted = [d["udid"] for d in devices() if d.get("state") == "Booted"] + out_json({"left": len(left), "booted": len(booted), "calls": [c[1:] for c in CALLS][-3:]}) + + +def cmd_memcheck(a): + vm = mem_summary(parse_vm_stat(vmstat_text())) + default_set = run(["xcrun", "simctl", "list", "devices", "booted"], 30)[1] # read only: the default set + out_json({"mem": vm, "pressure": pressure_level(), "booted": len([d for d in devices() if d.get("state") == "Booted"]), + "foreign_booted": len(re.findall(r"\(Booted\)", default_set)), "ips": len(list_ips())}) + + +def cmd_build(a): + """Builds the bench tools in a scratch copy: cold (a module cache that does not exist yet), then warm (the same + cache again); optionally an xcodebuild of a project given on the command line (cold: new DerivedData; warm: again).""" + root = os.path.abspath(os.path.join(HERE, "..")) + tmp = tempfile.mkdtemp(prefix="benchbuild") + res = {} + try: + for f in ("simfps.swift", "build.sh"): + shutil.copy(os.path.join(root, f), tmp) + for d in ("GlassBench", "GenieBench"): + shutil.copytree(os.path.join(root, d), os.path.join(tmp, d)) + mc = os.path.join(tmp, "modcache") + env = dict(os.environ, CLANG_MODULE_CACHE_PATH=mc) + for phase in ("cold", "warm"): + rc, _, err, secs = run(["/bin/bash", os.path.join(tmp, "build.sh")], 1800, env=env) + res["tools_%s_s" % phase] = round(secs, 1) + res["tools_%s_rc" % phase] = rc + if a.xcodebuild: + dd = os.path.join(tmp, "dd") + for phase in ("cold", "warm"): + rc, _, err, secs = run(["xcodebuild"] + a.xcodebuild.split() + ["-derivedDataPath", dd], 3600) + res["xcodebuild_%s_s" % phase] = round(secs, 1) + res["xcodebuild_%s_rc" % phase] = rc + out_json(res) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def cmd_calls(a): + out_json({"calls": CALLS}) + + +def main(argv=None): + global SET + ap = argparse.ArgumentParser(prog="benchguest") + ap.add_argument("--set", default=os.environ.get("BENCH_SET")) + sp = ap.add_subparsers(dest="cmd", required=True) + + def add(name, fn, *opts): + p = sp.add_parser(name) + p.set_defaults(fn=fn) + for o in opts: + p.add_argument(*o[0], **o[1]) + return p + O = lambda *n, **k: (n, k) + add("info", cmd_info) + add("init-set", cmd_init_set) + add("ensure-type", cmd_ensure_type, O("--scale", default="1"), O("--allow-restart", action="store_true")) + add("create", cmd_create, O("--n", type=int, required=True), O("--type", required=True), O("--runtime", required=True), + O("--prefix", default="bench"), O("--start", type=int, default=0)) + add("boot", cmd_boot, O("--udids", required=True), O("--parallel", action="store_true")) + add("prepare", cmd_prepare, O("--udids", required=True), O("--roles", required=True), O("--install", action="store_true"), + O("--quiesce", action="store_true")) + add("settle", cmd_settle, O("--max-s", type=float, default=120), O("--bg-pct", type=float, default=40), O("--step", type=float, default=3)) + add("calibrate", cmd_calibrate, O("--udid", required=True), O("--methods", default="openurl,appearance"), O("--probe-cmd")) + add("measure", cmd_measure, O("--udids", required=True), O("--roles", required=True), O("--mix", default="all-busy"), + O("--secs", type=float, default=40), O("--start-at", type=float, default=0), O("--probe-method", default="openurl"), + O("--probe-every", type=float, default=2.5), O("--probe-cmd"), O("--agent-period", type=float, default=12)) + add("shots", cmd_shots, O("--udids", required=True), O("--count", type=int, default=2)) + add("cycle", cmd_cycle, O("--type", required=True), O("--runtime", required=True), O("--prefix", default="bench")) + add("teardown", cmd_teardown) + add("build", cmd_build, O("--xcodebuild", default="")) + add("memcheck", cmd_memcheck) + add("calls", cmd_calls) + a = ap.parse_args(argv) + SET = a.set + try: + if a.cmd != "init-set": + if a.cmd not in ("info", "ensure-type"): + check_set() + a.fn(a) + except GuestError as e: + out_json({"error": str(e)}) + return 3 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/bench/guest/benchguest_stub.py b/src/bench/guest/benchguest_stub.py new file mode 100644 index 0000000..b280ca9 --- /dev/null +++ b/src/bench/guest/benchguest_stub.py @@ -0,0 +1,240 @@ +"""The guest's stand-in for DRY=1 and the tests: a fake simctl, ps, vm_stat and simfps that keep their state in a +directory ($BENCHGUEST_STATE) between calls, so `benchguest.py` runs its real control flow with no macOS. +Faults ($BENCH_STUB_FAULT, comma separated): memfloor@N (guest free memory below any floor from N booted +simulators), crash@N (a new crash report at N booted), backwards@N (a frame shown out of order at N), slow@N +(frame rate collapses from N), hang_simfps (simfps never answers), nomarker (the input marker never flips).""" +import json +import os +import random +import sys +import time +import uuid + +STATE = os.environ.get("BENCHGUEST_STATE", "/tmp/benchguest-stub") +FAULTS = [f for f in os.environ.get("BENCH_STUB_FAULT", "").split(",") if f] +T0 = float(os.environ.get("BENCH_STUB_T0", "0")) or time.time() + + +def _p(name): + return os.path.join(STATE, name) + + +def load(): + try: + with open(_p("state.json")) as f: + return json.load(f) + except (OSError, ValueError): + return {"devs": {}, "inj": [], "mode": {}, "ips": [], "pid": 1000} + + +def save(s): + os.makedirs(STATE, exist_ok=True) + with open(_p("state.json.tmp"), "w") as f: + json.dump(s, f) + os.replace(_p("state.json.tmp"), _p("state.json")) + + +def fault(name): + for f in FAULTS: + if f == name or f.startswith(name + "@"): + return int(f.split("@")[1]) if "@" in f else 0 + return None + + +def booted(s): + return [u for u, d in s["devs"].items() if d["state"] == "Booted"] + + +def _log(argv): + os.makedirs(STATE, exist_ok=True) + with open(os.environ.get("BENCHGUEST_CALLS", _p("calls.log")), "a") as f: + f.write(json.dumps(argv) + "\n") + + +RUNTIMES = {"runtimes": [{"identifier": "stub.runtime.iOS-26-5", "name": "iOS 26.5", "version": "26.5", "buildversion": "stub", + "platform": "iOS", "isAvailable": True, + "supportedDeviceTypes": [{"name": "iPhone 17 Pro", "identifier": "com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro", + "bundlePath": "/nonexistent/iPhone 17 Pro.simdevicetype"}]}]} +TYPES = {"devicetypes": [{"identifier": "com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro", "name": "iPhone 17 Pro"}, + {"identifier": "dev.macbox.CoreSimulator.SimDeviceType.iPhone-17-Pro-1x", "name": "iPhone 17 Pro (1x)"}]} + + +class _Lock: + """One stub process at a time through the state file (threads and processes alike).""" + def __enter__(self): + import fcntl + os.makedirs(STATE, exist_ok=True) + self.f = open(_p("lock"), "w") + fcntl.flock(self.f, fcntl.LOCK_EX) + + def __exit__(self, *a): + self.f.close() + + +def stub_run(argv, timeout=120, env=None): + with _Lock(): + return _stub_run(argv) + + +def _stub_run(argv): + _log(argv) + s = load() + out, rc = "", 0 + if argv[:3] == ["xcrun", "simctl", "--set"]: + cmd = argv[4:] + verb = cmd[0] + if verb == "list": + if cmd[1] == "devices": + devs = [{"udid": u, "name": d["name"], "state": d["state"], "isAvailable": True} for u, d in s["devs"].items()] + out = json.dumps({"devices": {"stub.runtime.iOS-26-5": devs}}) + else: + out = json.dumps(RUNTIMES if cmd[1] == "runtimes" else TYPES) + elif verb == "create": + u = str(uuid.uuid4()).upper() + s["devs"][u] = {"name": cmd[1], "state": "Shutdown", "type": cmd[2]} + out = u + "\n" + elif verb == "boot": + time.sleep(0.05) + s["devs"][cmd[1]]["state"] = "Booted" + elif verb == "shutdown": + for u in (list(s["devs"]) if cmd[1] == "all" else [cmd[1]]): + if u in s["devs"]: + s["devs"][u]["state"] = "Shutdown" + elif verb == "delete": + for u in (list(s["devs"]) if cmd[1] == "all" else [cmd[1]]): + s["devs"].pop(u, None) + elif verb == "launch": + s["mode"][cmd[1]] = "static" if "-static" in cmd else "busy" + elif verb == "terminate": + s["mode"].pop(cmd[1], None) + elif verb in ("openurl", "ui"): + if fault("nomarker") is None: + s["inj"].append([time.time(), cmd[1]]) + elif verb == "io": + p = cmd[-1] + from macbench import png + rows = [bytes([(x * 3 + y) % 251 for x in range(402) for _ in range(3)][:402 * 3]) for y in range(874)] + with open(p, "wb") as f: + f.write(png.write_png(rows, 402, 874, 3)) + elif verb == "get_app_container": + d = _p("container") + os.makedirs(os.path.join(d, "Documents"), exist_ok=True) + u = cmd[1] + now = time.time() + with open(os.path.join(d, "Documents", "pace.json"), "w") as f: + for k in range(8): + f.write(json.dumps({"at": now - 40 + 5 * k, "counter": int((now - 40 + 5 * k - T0) * 60) % 65536}) + "\n") + out = d + "\n" + elif argv[:3] == ["xcrun", "simctl", "list"]: + out = json.dumps(RUNTIMES if argv[3] == "runtimes" else TYPES) if argv[3] in ("runtimes", "devicetypes") else "" + elif argv[0] in ("sw_vers",): + out = "ProductName: macOS\nProductVersion: 26.0\n" + elif argv[0] == "xcodebuild": + out = "Xcode 26.0\nBuild version stub\n" + elif argv[0] == "uname": + out = "x86_64\n" + else: + out = "" + save(s) + return rc, out, "", 0.01 + + +def stub_ps(): + s = load() + now = time.time() + lines = [" 1 0 4000 %s /sbin/launchd" % _cpu(now - T0, 0.05), " 100 1 60000 %s /usr/sbin/WindowServer" % _cpu(now - T0, 0.2)] + pid = 200 + for i, u in enumerate(sorted(booted(s))): + busy = s["mode"].get(u) == "busy" + base = "/stub/set/%s/data" % u + lines.append("%5d 1 20000 %s launchd_sim %s" % (pid, _cpu(now - T0, 0.05), base)) + lines.append("%5d %5d 150000 %s SpringBoard" % (pid + 1, pid, _cpu(now - T0, 0.15 if busy else 0.03))) + lines.append("%5d %5d 300000 %s GlassBench" % (pid + 2, pid, _cpu(now - T0, 0.5 if busy else 0.01))) + lines.append("%5d 1 90000 %s SimMetalHost -d %s" % (pid + 3, _cpu(now - T0, 0.3 if busy else 0.02), u)) + pid += 10 + return "\n".join(lines) + "\n" + + +def _cpu(elapsed, frac): + secs = max(0.0, elapsed) * frac + return "%d:%05.2f" % (int(secs // 60), secs % 60) + + +def stub_vmstat(): + s = load() + n = len(booted(s)) + page = 16384 + total_pages = 16 * 1024 * 1024 * 1024 // page + used = (3000 + 1100 * n) * 1048576 // page + free = max(0, total_pages - used) + mf = fault("memfloor") + if mf is not None and n >= mf: + free = 20 * 1048576 // page + return ("Mach Virtual Memory Statistics: (page size of %d bytes)\nPages free: %d.\nPages active: %d.\nPages inactive: 1000.\n" + "Pages speculative: 500.\nPages wired down: %d.\nPages occupied by compressor: %d.\nPageins: 1000.\nPageouts: 0.\n" + "Swapins: 0.\nSwapouts: 0.\n" % (page, free, used // 2, used // 4, 5000 + 300 * n)) + + +class StubSimfps: + def __init__(self, udid, secs, strip, start_at, raw_path): + self.udid, self.secs, self.strip, self.start_at, self.raw = udid, secs, strip, start_at, raw_path + + def wait(self, timeout): + if fault("hang_simfps") is not None: + return None, "simfps hung (killed after %ds)" % int(timeout) + d = self.start_at + self.secs - time.time() + if d > 0: + time.sleep(d) + s = load() + n = len(booted(s)) + mode = s["mode"].get(self.udid, "busy") + rng = random.Random(hash(self.udid) & 0xffff) + fps = 60.0 if n <= 2 else max(8.0, 60.0 - 7.0 * (n - 2)) + sl = fault("slow") + if sl is not None and n >= sl: + fps = 14.0 + inj = sorted(t for t, u in s["inj"] if u == self.udid and self.start_at - 1 <= t <= self.start_at + self.secs) + lines, t, c, lum, k = [], self.start_at, 100, 255, 0 + back = fault("backwards") + lines.append("%.6f %d %d" % (t, c if self.strip else -1, lum)) + end = self.start_at + self.secs + delay = [0.06 + rng.random() * 0.05 for _ in inj] + while t < end: + if mode == "busy": + t += (1.0 / fps) * (0.9 + 0.2 * rng.random()) + c = (c + 1) % 65536 + if back is not None and n >= back and k == 50: + c = (c - 3) % 65536 + k += 1 + else: + t += 0.05 + while inj and t >= inj[0] + delay[0]: + lum = 0 if lum == 255 else 255 + inj.pop(0) + delay.pop(0) + if mode != "busy": + lines.append("%.6f %d %d" % (t, -1, lum)) + if mode == "busy": + lines.append("%.6f %d %d" % (t, c if self.strip else -1, lum)) + with open(self.raw, "w") as f: + f.write("\n".join(lines) + "\n") + n_frames = len(lines) + return {"fps": round(n_frames / self.secs, 1), "frames": n_frames, "secs": self.secs, "udid": self.udid}, None + + def kill(self): + pass + + +def install(mod): + mod.run = stub_run + mod.ps_text = stub_ps + mod.vmstat_text = stub_vmstat + mod.sysctl = lambda name: {"kern.memorystatus_vm_pressure_level": "1", "hw.ncpu": "8", "hw.memsize": str(16 << 30)}.get(name, "") + mod.spawn_simfps = lambda udid, secs, label, strip, start_at, raw: StubSimfps(udid, secs, strip, start_at, raw) + + def ips(): + s = load() + n = len(booted(s)) + cr = fault("crash") + return set(s["ips"]) | ({"GlassBench-stub.ips"} if cr is not None and n >= cr else set()) + mod.list_ips = ips diff --git a/src/bench/macbench/analysis.py b/src/bench/macbench/analysis.py index 97cdf5c..7704337 100644 --- a/src/bench/macbench/analysis.py +++ b/src/bench/macbench/analysis.py @@ -126,3 +126,54 @@ def summarize_scale(steps, target_specs, stopped=None): return {"max_n": mx, "stopped": stopped, "standing_cost": standing_cost(steps), "fairness_min_jain": min([s["fairness"]["jain"] for s in steps if s["fairness"]["jain"] is not None] or [None]) if any(s["fairness"]["jain"] is not None for s in steps) else None} + + +def pace_summary(records, t_first, t_last, t_start, t_end): + """GlassBench's own tick log ([(at, counter, fps, ticks)] every ~5 s, from before the window to after it): + app_tick_fps is the mean of the records written inside the window (what the app produced, before the compositor); + app_ticks brackets the frames seen: the counter's distance from the last record at or before the first frame to + the first record at or after the last one (an upper bound for the ticks between the frames: about 5 s of slack + on each side). Either is None when the log does not reach.""" + inside = [r[2] for r in records if t_start <= r[0] <= t_end + 5 and r[2] > 0] + before = [r for r in records if r[0] <= t_first] + after = [r for r in records if r[0] >= t_last] + ticks = (after[0][1] - before[-1][1]) % strip.WRAP if before and after else None + return {"app_tick_fps": stats.r1(stats.mean(inside)), "app_ticks": ticks} + + +def agent_metrics(actions, raw): + """The agent mix's actions: launch call and launch-to-first-frame, screenshot duration (ms, p50/p95).""" + out = {} + launches = [a for a in actions if a["op"] == "launch" and a.get("rc") == 0] + shots = [a for a in actions if a["op"] == "screenshot" and a.get("rc") == 0] + ttff = [] + for a in launches: + nxt = next((r[0] for r in raw if r[0] >= a["t0"] and r[1] >= 0), None) + if nxt is not None: + ttff.append((nxt - a["t0"]) * 1000.0) + for name, vals in (("launch_to_frame_ms", ttff), ("screenshot_ms", [(a["t1"] - a["t0"]) * 1000.0 for a in shots]), + ("launch_call_ms", [(a["t1"] - a["t0"]) * 1000.0 for a in launches])): + out[name + "_p50"] = stats.r1(stats.percentile(vals, 0.5)) + out[name + "_p95"] = stats.r1(stats.percentile(vals, 0.95)) + out["actions"] = len(actions) + out["action_failures"] = sum(1 for a in actions if a.get("rc") not in (0, None)) + return out + + +def motion_metrics(times, nominal_fps=60.0, still_ms=250.0): + """frame metrics over the stretches of motion only (the agent mix idles between actions): fps = frames per + second inside stretches; the intervals are the ones inside stretches; the gap that ends a stretch is not one.""" + st = stats.stretches(times, still_ms) + ivs, frames, span = [], 0, 0.0 + for t0, t1, n in st: + span += t1 - t0 + frames += n - 1 + inside = [t for t in times] + for a, b in zip(inside, inside[1:]): + if (b - a) * 1000.0 <= still_ms: + ivs.append((b - a) * 1000.0) + thr = 2000.0 / nominal_fps + return {"stretches": len(st), "fps": stats.r1(frames / span if span > 0 else 0.0), "p50_ms": stats.r1(stats.percentile(ivs, 0.5)), + "p95_ms": stats.r1(stats.percentile(ivs, 0.95)), "p99_ms": stats.r1(stats.percentile(ivs, 0.99)), + "max_ms": stats.r1(max(ivs) if ivs else None), "over2x": sum(1 for x in ivs if x > thr), + "longest_gap_s": stats.r1(max(ivs) / 1000.0, 3) if ivs else None} diff --git a/src/bench/macbench/gates.py b/src/bench/macbench/gates.py new file mode 100644 index 0000000..875e810 --- /dev/null +++ b/src/bench/macbench/gates.py @@ -0,0 +1,148 @@ +"""Correctness gates. Each returns {"status": PASS|FAIL|SKIP, "detail": str}. An unreadable gate is SKIP, shown as +such, never PASS. Coherent and kernel gates are DELTAS over a window (a mark taken before, a check after), never +absolute counts or a fixed number of surfaces.""" +import json +import os +import re + +PROCESS_GATES = ("late_copies_in_busy", "late_beside_unplanned", "plan_left_planned") + + +def gate(status, detail=""): + return {"status": status, "detail": detail} + + +def worst(gates): + """FAIL if any FAIL; else SKIP-only/mixed is still ok: a run PASSes when nothing FAILed (SKIPs stay visible).""" + return not any(g["status"] == "FAIL" for g in gates.values()) + + +class Coherent: + """Reads Coherent's census (coherent.jsonl, a report a minute per surface), QEMU's log and the GPU hang flag.""" + + def __init__(self, census_dir, qemu_log, hang_file): + self.census = os.path.join(census_dir, "coherent.jsonl") if census_dir else None + self.qemu_log, self.hang_file = qemu_log, hang_file + + def _lines(self): + out = [] + if self.census and os.path.exists(self.census): + with open(self.census, errors="replace") as f: + for l in f: + try: + out.append(json.loads(l)) + except ValueError: + pass + return out + + def _log_from(self, off): + if not (self.qemu_log and os.path.exists(self.qemu_log)): + return None + with open(self.qemu_log, "rb") as f: + f.seek(off) + return f.read().decode(errors="replace") + + def mark(self): + lines = self._lines() + proc = {k: 0 for k in PROCESS_GATES} + for d in lines: + if "process" in d: + for k in PROCESS_GATES: + proc[k] = max(proc[k], d.get(k, 0) or 0) + size = os.path.getsize(self.qemu_log) if self.qemu_log and os.path.exists(self.qemu_log) else None + return {"n": len(lines), "proc": proc, "log_off": size, "hang": bool(self.hang_file and os.path.exists(self.hang_file))} + + def check(self, mark): + """Gates since the mark: overwrite_risk, sync_errors (new surface reports), process counters that rose, + command buffer failures and GPU waits in QEMU's log, a new hang flag.""" + res = {} + lines = self._lines() + new = lines[mark["n"]:] + surf = [d for d in new if "process" not in d] + if self.census is None or not os.path.exists(self.census): + res["coherent_census"] = gate("SKIP", "no census file (Coherent off?)") + elif not surf: + res["coherent_census"] = gate("SKIP", "no census report since the mark (they come once a minute)") + else: + risk = max((d.get("overwrite_risk", 0) or 0) for d in surf) + errs = max((d.get("sync_errors", 0) or 0) for d in surf) + res["coherent_census"] = gate("FAIL" if risk or errs else "PASS", + "%d report(s): overwrite_risk max %d, sync_errors max %d" % (len(surf), risk, errs)) + proc = {k: mark["proc"][k] for k in PROCESS_GATES} + for d in new: + if "process" in d: + for k in PROCESS_GATES: + proc[k] = max(proc[k], d.get(k, 0) or 0) + rose = {k: proc[k] for k in PROCESS_GATES if proc[k] > mark["proc"][k]} + if self.census and os.path.exists(self.census): + res["coherent_process"] = gate("FAIL" if rose else "PASS", ("rose: %s" % rose) if rose else "late_copies_in_busy, late_beside_unplanned, plan_left_planned flat") + else: + res["coherent_process"] = gate("SKIP", "no census file") + txt = self._log_from(mark["log_off"]) if mark["log_off"] is not None else None + if txt is None: + res["qemu_log"] = gate("SKIP", "QEMU log not readable") + else: + cb = txt.count("command buffer failed") + waits = len(re.findall(r"waited \d+s for stamp", txt)) + res["qemu_log"] = gate("FAIL" if cb else "PASS", "command buffer failed %d, stamp waits %d" % (cb, waits)) + hang = bool(self.hang_file and os.path.exists(self.hang_file)) + if self.hang_file is None: + res["gpu_hang"] = gate("SKIP", "no hang flag path") + else: + res["gpu_hang"] = gate("FAIL" if hang and not mark["hang"] else "PASS", "hang flag " + ("set" if hang else "clear")) + return res + + +def kernel_gate(count, note): + if count is None: + return gate("SKIP", note) + return gate("FAIL" if count else "PASS", "%d GPU error line(s)%s" % (count, (": " + note) if note else "")) + + +def strip_gate_of(sims): + bad = [s for s in sims if s.get("strip_gate") and not s["strip_gate"]["ok"]] + judged = [s for s in sims if s.get("strip_gate")] + if not judged: + return gate("SKIP", "no simulator drew the strip") + if bad: + return gate("FAIL", "; ".join("sim %s: %s" % (s["idx"], s["strip_gate"]["why"]) for s in bad[:3])) + return gate("PASS", "%d simulator(s): no frame out of order" % len(judged)) + + +def shots_gates(shots_by_sim, golden_hash=None, golden_key=None): + """Tear, strip-readable and pixel-exact gates from the guest's `shots` output. + shots_by_sim: [{"idx", "role", "shots": [{ok, hash, tear, tear_applicable, counter}]}]""" + res = {} + tear_bad, tear_judged = [], 0 + for s in shots_by_sim: + if s["role"] != "busy": + continue + for sh in s["shots"]: + if sh.get("ok") and sh.get("tear_applicable"): + tear_judged += 1 + if sh["tear"]: + tear_bad.append(s["idx"]) + if not tear_judged: + res["tears"] = gate("SKIP", "no screenshot showed the scrolling cards") + else: + res["tears"] = gate("FAIL" if tear_bad else "PASS", ("torn: sim %s" % sorted(set(tear_bad))) if tear_bad else "%d screenshot(s) clean" % tear_judged) + hashes = [sh["hash"] for s in shots_by_sim if s["role"] == "static" for sh in s["shots"] if sh.get("ok")] + if not hashes: + res["pixels"] = gate("SKIP", "no static reference screen in this step") + elif len(set(hashes)) > 1: + res["pixels"] = gate("FAIL", "%d different pictures of the same static screen" % len(set(hashes))) + elif golden_hash is None: + res["pixels"] = gate("SKIP", "screens agree with each other; no golden stored for %s (bench golden record)" % golden_key) + else: + res["pixels"] = gate("PASS" if hashes[0] == golden_hash else "FAIL", + "static screen %s the golden %s" % ("matches" if hashes[0] == golden_hash else "DIFFERS from", golden_key)) + return res + + +def leak_gate(name, xs, ys, bar_per_unit, unit): + """FAIL when the least-squares slope of ys over xs exceeds the bar (e.g. MB per cycle).""" + from . import stats + b, _, r2 = stats.slope(xs, ys) + if b is None: + return gate("SKIP", "%s: not enough samples" % name) + return gate("FAIL" if b > bar_per_unit else "PASS", "%s: %+.2f %s (bar %g, R2 %.2f, %d samples)" % (name, b, unit, bar_per_unit, r2, len(xs))) diff --git a/src/bench/macbench/hostprobe.py b/src/bench/macbench/hostprobe.py new file mode 100644 index 0000000..1fc796a --- /dev/null +++ b/src/bench/macbench/hostprobe.py @@ -0,0 +1,221 @@ +"""What the Linux host says about the VM: QEMU's threads (user / guest / sys CPU), the GPU's render time from +DRM fdinfo, the VM's cgroup (memory, swap, pressure), host memory and swap, and kernel GPU error lines. +Passive reads of /proc and /sys only; no tracer, no GPU work. `root` lets the tests point at a fake tree.""" +import os +import re +import subprocess + +GPU_ERRORS = re.compile( + r"fence expir|gpu hang|context reset|i915.*\bhang\b" + r"|xe \S+: .*(timedout job|wedged|engine reset|gt\d+: (trying reset|reset (queued|started|failed))" + r"|\*error\* (tile\d+: )?gt\d|fault response|faulted address|banned|coredump has been created)", re.I) + + +def _read(root, path): + try: + with open(root + path) as f: + return f.read() + except OSError: + return None + + +def parse_task_stat(text): + """/proc//task//stat -> (comm, utime, stime, guest_time) in clock ticks. comm may hold spaces and ')'.""" + i, j = text.index("("), text.rindex(")") + comm = text[i + 1:j] + f = text[j + 2:].split() # f[0] is field 3 (state) + return comm, int(f[11]), int(f[12]), int(f[40]) if len(f) > 40 else 0 + + +def thread_class(comm): + c = re.sub(r"\d+", "N", comm) + return "vCPU" if c.startswith("CPU N/KVM") else c + + +def thread_snapshot(pid, root=""): + out = {} + base = "%s/proc/%d/task" % (root, pid) + try: + tids = os.listdir(base) + except OSError: + return out + for tid in tids: + t = _read(root, "/proc/%d/task/%s/stat" % (pid, tid)) + if t: + try: + out[tid] = parse_task_stat(t) + except (ValueError, IndexError): + pass + return out + + +def thread_delta(a, b, secs, hz=100, top=6): + """CPU of the process between two snapshots, in cores: user (host-side), guest (inside the VM), sys, total, and the + top thread classes. guest_time is part of utime in Linux, so user = utime - guest.""" + cls, tot_u = {}, [0, 0, 0] + for tid, (comm, ut, st, gt) in b.items(): + if tid not in a: + continue + du, ds, dg = ut - a[tid][1], st - a[tid][2], gt - a[tid][3] + tot_u[0] += du - dg + tot_u[1] += dg + tot_u[2] += ds + k = thread_class(comm) + e = cls.setdefault(k, [0, 0.0]) + e[0] += 1 + e[1] += (du + ds) / hz / secs + f = lambda x: round(x / hz / secs, 3) + rows = sorted(cls.items(), key=lambda kv: -kv[1][1])[:top] + return {"user": f(tot_u[0]), "guest": f(tot_u[1]), "sys": f(tot_u[2]), "cores": f(sum(tot_u)), + "top": [{"name": k, "threads": n, "cores": round(c, 3)} for k, (n, c) in rows]} + + +def parse_fdinfo(text): + """-> (client id, render ns or None). i915 states `drm-engine-render: ns`; xe states cycles (not converted: + None, so a GPU number is never made up).""" + cid = re.search(r"^drm-client-id:\s*(\S+)", text, re.M) + r = re.search(r"^drm-engine-render:\s*(\d+)", text, re.M) + if not cid: + return None, None + return cid.group(1), int(r.group(1)) if r else None + + +def gpu_render_ns(pid, root=""): + """Sum of the render-engine busy ns over the process's distinct DRM clients; None when none is readable.""" + seen, got = {}, False + base = "%s/proc/%d/fdinfo" % (root, pid) + try: + fds = os.listdir(base) + except OSError: + return None + for fd in fds: + t = _read(root, "/proc/%d/fdinfo/%s" % (pid, fd)) + if t and "drm-client-id" in t: + cid, ns = parse_fdinfo(t) + if ns is not None: + seen[cid] = ns + got = True + return sum(seen.values()) if got else None + + +def parse_kv(text): + d = {} + for line in (text or "").splitlines(): + p = line.split() + if len(p) == 2 and p[1].lstrip("-").isdigit(): + d[p[0]] = int(p[1]) + return d + + +def parse_psi(text): + """/proc/pressure/memory or memory.pressure -> {"some": {avg10,total}, "full": {...}}""" + d = {} + for line in (text or "").splitlines(): + p = line.split() + if p and p[0] in ("some", "full"): + d[p[0]] = {k: float(v) for k, v in (x.split("=") for x in p[1:])} + return d + + +def cgroup_dir(pid, root=""): + t = _read(root, "/proc/%d/cgroup" % pid) + for line in (t or "").splitlines(): + if line.startswith("0::"): + return "/sys/fs/cgroup" + line[3:] + return None + + +def mem_snapshot(pid, root=""): + """Host memory and swap counters, and the VM's cgroup's: every number is a counter or a level, deltas are taken by the caller.""" + mi = parse_kv((_read(root, "/proc/meminfo") or "").replace(":", " ")) + vs = parse_kv(_read(root, "/proc/vmstat")) + cg = cgroup_dir(pid, root) + s = {"mem_total_kb": mi.get("MemTotal"), "mem_avail_kb": mi.get("MemAvailable"), "swap_total_kb": mi.get("SwapTotal"), + "swap_free_kb": mi.get("SwapFree"), "pswpin": vs.get("pswpin"), "pswpout": vs.get("pswpout"), + "psi_some_us": parse_psi(_read(root, "/proc/pressure/memory")).get("some", {}).get("total")} + if cg: + for name, key in (("memory.current", "cg_current"), ("memory.swap.current", "cg_swap")): + t = _read(root, cg + "/" + name) + if t and t.strip().isdigit(): + s[key] = int(t.strip()) + ev = parse_kv(_read(root, cg + "/memory.events")) + s["cg_high"], s["cg_max"], s["cg_oom"] = ev.get("high"), ev.get("max"), ev.get("oom_kill") + s["cg_psi_some_us"] = parse_psi(_read(root, cg + "/memory.pressure")).get("some", {}).get("total") + st = _read(root, "/proc/%d/status" % pid) or "" + m = re.search(r"VmRSS:\s+(\d+)", st) + s["qemu_rss_kb"] = int(m.group(1)) if m else None + return s + + +def mem_delta(a, b, secs): + d = lambda k: None if a.get(k) is None or b.get(k) is None else b[k] - a[k] + out = {"mem_avail_gb": None if b.get("mem_avail_kb") is None else round(b["mem_avail_kb"] / 1048576.0, 2), + "qemu_rss_mb": None if b.get("qemu_rss_kb") is None else round(b["qemu_rss_kb"] / 1024.0, 1), + "pswpin_pages": d("pswpin"), "pswpout_pages": d("pswpout"), + "cgroup_swap_mb": None if b.get("cg_swap") is None else round(b["cg_swap"] / 1048576.0, 1), + "cgroup_swap_delta_mb": None if d("cg_swap") is None else round(d("cg_swap") / 1048576.0, 2), + "cgroup_mem_gb": None if b.get("cg_current") is None else round(b["cg_current"] / 1073741824.0, 2), + "cgroup_events": {k: d(k) for k in ("cg_high", "cg_max", "cg_oom") if d(k) is not None}, + "psi_mem_some_pct": None if d("psi_some_us") is None or secs <= 0 else round(100.0 * d("psi_some_us") / 1e6 / secs, 2), + "cgroup_psi_some_pct": None if d("cg_psi_some_us") is None or secs <= 0 else round(100.0 * d("cg_psi_some_us") / 1e6 / secs, 2)} + return out + + +def kernel_gpu_lines(since_epoch): + """Kernel GPU error lines since a time: (count or None, note). None = the journal can't be read (not a PASS).""" + try: + p = subprocess.run(["journalctl", "-k", "--since", "@%d" % int(since_epoch), "--no-pager", "-q"], + capture_output=True, text=True, timeout=30) + except (OSError, subprocess.TimeoutExpired) as e: + return None, "journalctl unavailable: %s" % e + if p.returncode != 0: + return None, "journalctl: " + (p.stderr.strip()[:100] or "failed") + lines = [l for l in p.stdout.splitlines() if GPU_ERRORS.search(l)] + return len(lines), ("; ".join(l[-80:] for l in lines[-2:]) if lines else "") + + +def read_live_conf(path): + """live.conf as {key: value} (read only).""" + out = {} + try: + with open(path) as f: + for line in f: + line = line.strip() + if line and not line.startswith("#") and "=" in line: + k, v = line.split("=", 1) + out[k.strip()] = v.strip() + except OSError: + pass + return out + + +def qemu_env_switches(pid, root=""): + """REIMS_VGPU_* names and values of the QEMU process (a directory value is shown as ).""" + t = _read(root, "/proc/%d/environ" % pid) or "" + out = {} + for kv in t.split("\0"): + if kv.startswith("REIMS_VGPU_") and "=" in kv: + k, v = kv.split("=", 1) + out[k] = "" if v.startswith("/") else v + return out + + +def host_versions(): + kernel = os.uname().release + mesa = None + for argv in (["dpkg-query", "-W", "-f=${Version}", "mesa-vulkan-drivers"], ["rpm", "-q", "--qf", "%{VERSION}", "mesa-vulkan-drivers"]): + try: + p = subprocess.run(argv, capture_output=True, text=True, timeout=10) + if p.returncode == 0 and p.stdout.strip(): + mesa = p.stdout.strip() + break + except (OSError, subprocess.TimeoutExpired): + pass + cpu = None + try: + with open("/proc/cpuinfo") as f: + m = re.search(r"model name\s*:\s*(.+)", f.read()) + cpu = m.group(1).strip() if m else None + except OSError: + pass + return {"kernel": kernel, "mesa": mesa or "unknown (pass --tag mesa=)", "cpu_model": cpu, "cpus": os.cpu_count()} diff --git a/src/bench/macbench/runctx.py b/src/bench/macbench/runctx.py new file mode 100644 index 0000000..2aa1f75 --- /dev/null +++ b/src/bench/macbench/runctx.py @@ -0,0 +1,168 @@ +"""One run's safety: a hard deadline, explicit child pids, knobs that are put back, and ONE cleanup that runs on +every way out (normal end, error, SIGTERM/SIGINT, the deadline). Nothing waits on `wait` without a pid.""" +import atexit +import os +import signal +import subprocess +import sys +import threading +import time + + +class DeadlineError(Exception): + pass + + +class Run: + def __init__(self, deadline_s, log=print): + self.t0 = time.monotonic() + self.deadline = self.t0 + deadline_s + self.log = log + self.children = [] # subprocess.Popen objects this run started + self.undo = [] # (label, fn) restoring a knob, run last-in first-out + self.teardown = [] # (label, fn): shut down what the run started (simulators) + self.on_deadline = None # fn: write the partial result + self.cleaned = False + self._lock = threading.Lock() + self.expired = threading.Event() + self.problems = [] # things cleanup found it could not restore + + # --- time + def remaining(self): + return self.deadline - time.monotonic() + + def need(self, secs, what): + """Refuse a step that would run past the deadline (the cleanup keeps its own 90 s).""" + if self.remaining() < secs + 90: + raise DeadlineError("no time for %s (%d s needed, %d s left before the deadline)" % (what, secs, max(0, self.remaining() - 90))) + + # --- children + def popen(self, argv, **kw): + p = subprocess.Popen(argv, start_new_session=True, **kw) + with self._lock: + self.children.append(p) + return p + + def kill_children(self): + for p in list(self.children): + if p.poll() is None: + try: + os.killpg(p.pid, signal.SIGTERM) + except OSError: + pass + t_end = time.monotonic() + 3 + for p in list(self.children): + try: + p.wait(max(0.1, t_end - time.monotonic())) + except subprocess.TimeoutExpired: + try: + os.killpg(p.pid, signal.SIGKILL) + except OSError: + pass + self.children.clear() + + # --- the one trap + def install(self): + atexit.register(self.cleanup) + + def on_signal(signum, frame): + self.log("signal %d: stopping" % signum) + self.expired.set() + sys.exit(128 + signum) # runs atexit -> cleanup + signal.signal(signal.SIGTERM, on_signal) + signal.signal(signal.SIGINT, on_signal) + t = threading.Thread(target=self._watchdog, daemon=True) + t.start() + + def _watchdog(self): + while time.monotonic() < self.deadline - 60: + time.sleep(1) + self.log("DEADLINE: 60 s left, stopping") + self.expired.set() + if self.on_deadline: + try: + self.on_deadline() + except Exception as e: # the partial result is best effort + self.log("partial result failed: %s" % e) + self.cleanup() + os._exit(124) + + def cleanup(self): + with self._lock: + if self.cleaned: + return + self.cleaned = True + self.expired.set() + self.kill_children() + for label, fn in reversed(self.teardown): + try: + fn() + self.log("cleanup: %s" % label) + except Exception as e: + self.problems.append("%s: %s" % (label, e)) + self.log("cleanup FAILED: %s: %s" % (label, e)) + for label, fn in reversed(self.undo): + try: + fn() + self.log("restored: %s" % label) + except Exception as e: + self.problems.append("%s: %s" % (label, e)) + self.log("restore FAILED: %s: %s" % (label, e)) + + +class LiveKnob: + """A key=value line in the VM's live switch file: A = the file as it was, B = with the line set. The file's + original text is kept and put back whole on any exit.""" + + def __init__(self, path, key, value, run): + self.path, self.key, self.value, self.run = path, key, value, run + with open(path) as f: + self.original = f.read() + run.undo.append(("live switches file", self.restore)) + + def _write(self, text): + tmp = self.path + ".bench.tmp" + with open(tmp, "w") as f: + f.write(text) + os.replace(tmp, self.path) + + def set(self): + lines = [l for l in self.original.splitlines() if not l.strip().startswith(self.key + "=")] + self._write("\n".join(lines + ["%s=%s" % (self.key, self.value)]) + "\n") + + def unset(self): + self._write(self.original) + + def restore(self): + self._write(self.original) + + def describe(self): + return "live:%s=%s" % (self.key, self.value) + + +class CmdKnob: + """Shell commands for the knob (set / unset) run on the host. The unset command also runs on cleanup.""" + + def __init__(self, set_cmd, unset_cmd, run, runner): + self.set_cmd, self.unset_cmd, self.run, self.runner = set_cmd, unset_cmd, run, runner + run.undo.append(("knob unset command", self.restore)) + + def set(self): + self.runner(self.set_cmd) + + def unset(self): + self.runner(self.unset_cmd) + + def restore(self): + self.runner(self.unset_cmd) + + def describe(self): + return "cmd" + + +def shell_runner(timeout=60): + def run_cmd(cmd): + p = subprocess.run(["/bin/sh", "-c", cmd], capture_output=True, text=True, timeout=timeout) + if p.returncode != 0: + raise RuntimeError("knob command failed (rc %d): %s" % (p.returncode, p.stderr.strip()[:120])) + return run_cmd diff --git a/src/bench/macbench/transport.py b/src/bench/macbench/transport.py new file mode 100644 index 0000000..ff6b575 --- /dev/null +++ b/src/bench/macbench/transport.py @@ -0,0 +1,122 @@ +"""Reaching the guest: ssh to the Mac (or the same helper run locally on stubs for DRY=1).""" +import json +import os +import shlex +import subprocess +import sys +import time + +from .runctx import DeadlineError + +GUEST_DIR = "~/.macbox/bench" + + +class GuestError(Exception): + pass + + +class Pending: + """A helper call running in the background: .wait() gives its JSON. The process is one of the run's children.""" + + def __init__(self, guest, p, what, timeout): + self.guest, self.p, self.what, self.timeout, self.t0 = guest, p, what, timeout, time.monotonic() + + def wait(self): + left = max(1, self.timeout - (time.monotonic() - self.t0)) + try: + out, err = self.p.communicate(timeout=left) + except subprocess.TimeoutExpired: + self.guest.run.kill_children() + raise GuestError("%s: no answer in %ds" % (self.what, self.timeout)) + return self.guest.parse(self.what, self.p.returncode, out, err) + + +class SshGuest: + def __init__(self, run, ssh_host, set_path, stubs_dir=None): + self.run, self.ssh_host, self.set_path = run, ssh_host, set_path + self.calls = 0 + self.fault_ssh_timeout = None # DRY: fail the k-th call (tests) + + def argv(self, args): + remote = "cd %s && BENCH_SET=\"$HOME/.macbox/bench/sets/%s\" xcrun python3 guest/benchguest.py %s" % ( + GUEST_DIR, self.set_path, " ".join(shlex.quote(str(a)) for a in args)) + return ["ssh", "-o", "ConnectTimeout=10", "-o", "BatchMode=yes", "-o", "ServerAliveInterval=15", self.ssh_host, remote] + + def parse(self, what, rc, out, err): + lines = [l for l in (out or "").strip().splitlines() if l.startswith("{")] + if not lines: + raise GuestError("%s: rc %s: %s" % (what, rc, (err or out or "no output").strip()[-200:])) + d = json.loads(lines[-1]) + if isinstance(d, dict) and "error" in d: + raise GuestError("%s: %s" % (what, d["error"])) + return d + + def _check(self, what, timeout): + if self.run.expired.is_set(): + raise DeadlineError("stopping: %s not started" % what) + if self.run.remaining() < 5: + raise DeadlineError("deadline: %s not started" % what) + self.calls += 1 + if self.fault_ssh_timeout and self.calls == self.fault_ssh_timeout: + raise GuestError("%s: ssh timed out (injected)" % what) + return min(timeout, max(5, int(self.run.remaining()))) + + def start(self, args, timeout=300): + what = args[0] + timeout = self._check(what, timeout) + p = self.run.popen(self.argv(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + return Pending(self, p, what, timeout) + + def call(self, args, timeout=300): + return self.start(args, timeout).wait() + + def clock_offset(self): + """Guest clock minus host clock in seconds, from the quickest of 5 round trips.""" + best = None + for _ in range(5): + t1 = time.time() + try: + p = subprocess.run(self.argv_plain("python3 -c 'import time; print(time.time())'"), capture_output=True, text=True, timeout=20) + g = float(p.stdout.strip().splitlines()[-1]) + except (OSError, ValueError, IndexError, subprocess.TimeoutExpired): + continue + t2 = time.time() + if best is None or t2 - t1 < best[0]: + best = (t2 - t1, g - (t1 + t2) / 2) + return best[1] if best else 0.0 + + def argv_plain(self, cmd): + return ["ssh", "-o", "ConnectTimeout=10", "-o", "BatchMode=yes", self.ssh_host, cmd] + + def sh(self, cmd, timeout=120): + """A plain command in the guest: (rc, stdout).""" + if self.run.expired.is_set(): + raise DeadlineError("stopping") + p = subprocess.run(self.argv_plain(cmd), capture_output=True, text=True, timeout=min(timeout, max(5, int(self.run.remaining())))) + return p.returncode, p.stdout + + +class LocalGuest(SshGuest): + """DRY=1: the same helper, run here with its stub primitives. State in a directory, so calls share one fake guest.""" + + def __init__(self, run, set_path, state_dir, bench_dir, faults=""): + super().__init__(run, "stub", set_path) + self.state_dir, self.bench_dir, self.faults = state_dir, bench_dir, faults + self.set_dir = os.path.join(state_dir, "set") + + def argv(self, args): + return [sys.executable, os.path.join(self.bench_dir, "guest", "benchguest.py"), "--set", self.set_dir] + [str(a) for a in args] + + def start(self, args, timeout=300): + what = args[0] + timeout = self._check(what, timeout) + env = dict(os.environ, BENCHGUEST_STUB="1", BENCHGUEST_STATE=self.state_dir, BENCH_STUB_FAULT=self.faults, + BENCHGUEST_CALLS=os.path.join(self.state_dir, "calls.log"), PYTHONPATH=self.bench_dir) + p = self.run.popen(self.argv(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env, cwd=self.bench_dir) + return Pending(self, p, what, timeout) + + def clock_offset(self): + return 0.0 + + def sh(self, cmd, timeout=120): + return 0, "" From d4e44ff80fe29ecb07da737b5825f31f4f910e8e Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:21:47 +0200 Subject: [PATCH 05/23] bench: the scaling sweep, idle, lifecycle, soak, build, panel plug, golden and A/B runner behind src/bench/run, with a DRY=1 stub guest --- src/bench/guest/benchguest.py | 32 +-- src/bench/guest/benchguest_stub.py | 2 +- src/bench/macbench/ab.py | 88 ++++++++ src/bench/macbench/analysis.py | 2 +- src/bench/macbench/chart.py | 8 +- src/bench/macbench/cli.py | 319 +++++++++++++++++++++++++++++ src/bench/macbench/compare.py | 6 +- src/bench/macbench/core.py | 241 ++++++++++++++++++++++ src/bench/macbench/fakes.py | 67 ++++++ src/bench/macbench/gates.py | 8 +- src/bench/macbench/hostprobe.py | 39 ++++ src/bench/macbench/others.py | 206 +++++++++++++++++++ src/bench/macbench/runctx.py | 13 +- src/bench/macbench/scale.py | 200 ++++++++++++++++++ src/bench/macbench/transport.py | 4 +- src/bench/run | 11 + src/bench/tests/test_modules.py | 289 ++++++++++++++++++++++++++ 17 files changed, 1501 insertions(+), 34 deletions(-) create mode 100644 src/bench/macbench/ab.py create mode 100644 src/bench/macbench/cli.py create mode 100644 src/bench/macbench/core.py create mode 100644 src/bench/macbench/fakes.py create mode 100644 src/bench/macbench/others.py create mode 100644 src/bench/macbench/scale.py create mode 100755 src/bench/run create mode 100644 src/bench/tests/test_modules.py diff --git a/src/bench/guest/benchguest.py b/src/bench/guest/benchguest.py index a9e1764..78263c4 100644 --- a/src/bench/guest/benchguest.py +++ b/src/bench/guest/benchguest.py @@ -332,8 +332,8 @@ def cmd_info(a): vm = parse_vm_stat(vmstat_text()) out_json({"macos": ver.group(1) if ver else None, "xcode": xc, "runtimes": rts, "arch": run(["uname", "-m"], 5)[1].strip(), "cpus": sysctl("hw.ncpu"), "mem_mb": int(sysctl("hw.memsize") or 0) // 1048576, "mem": mem_summary(vm), - "has_simfps": os.path.exists(os.path.join(HERE, "..", "simfps")), - "has_app": os.path.isdir(os.path.join(HERE, "..", "GlassBench.app")), "stub": bool(STUB)}) + "has_simfps": bool(STUB) or os.path.exists(os.path.join(HERE, "..", "simfps")), + "has_app": bool(STUB) or os.path.isdir(os.path.join(HERE, "..", "GlassBench.app")), "stub": bool(STUB)}) def cmd_init_set(a): @@ -473,9 +473,7 @@ def cmd_prepare(a): simctl("spawn", u, "launchctl", "bootout", "system/com.apple.mediaanalysisd", timeout=30) simctl("terminate", u, BUNDLE_ID, timeout=30) args = ["-probe"] + (["-static"] if role == "static" else []) - if role == "agent": - args = [] - rc_l = 0 if role == "agent" else simctl("launch", u, BUNDLE_ID, *args, timeout=60)[0] + rc_l = 0 if role in ("agent", "idle") else simctl("launch", u, BUNDLE_ID, *args, timeout=60)[0] res.append({"udid": u, "role": role, "install_rc": rc_i, "launch_rc": rc_l, "s": round(now() - t0, 2)}) out_json({"sims": res}) @@ -508,13 +506,13 @@ def cmd_settle(a): out_json({"settled": series[-1] < a.bg_pct, "bg_pct": series[-1], "waited_s": round(now() - t0, 1), "series": series[-8:]}) -def run_window(a, udids, roles, mix, secs, t_go, tmp, probe_method, probe_every, cmd_tpl, strip_for_all=True): +def run_window(a, udids, roles, mix, secs, t_go, tmp, probe_method, probe_every, cmd_tpl, no_strip=False): """simfps on each simulator, the probe and agent threads, and the guest's samples around the window.""" raws, handles, stop_evt = {}, {}, threading.Event() if mix != "idle": for i, u in enumerate(udids): raws[u] = os.path.join(tmp, "raw-%d.txt" % i) - handles[u] = spawn_simfps(u, secs, "bench-%d" % i, roles[i] != "static", t_go, raws[u]) + handles[u] = spawn_simfps(u, secs, "bench-%d" % i, roles[i] != "static" and not no_strip, t_go, raws[u]) ips0, ps0, vm0, lvl0 = list_ips(), None, None, pressure_level() d = t_go - now() if d > 0: @@ -524,7 +522,7 @@ def run_window(a, udids, roles, mix, secs, t_go, tmp, probe_method, probe_every, t_stop = t_go + secs inj, acts, threads = {u: [] for u in udids}, {u: [] for u in udids}, [] lvls = [] - if mix in ("all-busy", "one-busy"): + if mix in ("all-busy", "one-busy") and probe_method != "none": targets = [u for u, r in zip(udids, roles) if mix == "all-busy" or r == "static"] for k, u in enumerate(targets): t = threading.Thread(target=probe_loop, args=(u, probe_method, cmd_tpl, t_go + 1.0 + k * probe_every / max(len(targets), 1), @@ -574,11 +572,11 @@ def sample_pressure(): def cmd_measure(a): - udids, roles = a.udids.split(","), a.roles.split(",") + udids, roles = [u for u in a.udids.split(",") if u], [r for r in a.roles.split(",") if r] tmp = tempfile.mkdtemp(prefix="benchwin") try: t_go = a.start_at or now() + 3 + len(udids) - out_json(run_window(a, udids, roles, a.mix, a.secs, t_go, tmp, a.probe_method, a.probe_every, a.probe_cmd)) + out_json(run_window(a, udids, roles, a.mix, a.secs, t_go, tmp, a.probe_method, a.probe_every, a.probe_cmd, a.no_strip)) finally: shutil.rmtree(tmp, ignore_errors=True) @@ -590,18 +588,19 @@ def cmd_calibrate(a): res = {} try: for method in a.methods.split(","): - t_go = now() + 3 + f = a.time_scale + t_go = now() + 3 * f raw = os.path.join(tmp, "raw.txt") - h = spawn_simfps(a.udid, 9, "cal", False, t_go, raw) + h = spawn_simfps(a.udid, 9 * f, "cal", False, t_go, raw) inj, state = [], {} for k in range(3): - d = t_go + 1.5 + k * 2.4 - now() + d = t_go + (1.5 + k * 2.4) * f - now() if d > 0: time.sleep(d) t0 = now() rc = inject(method, a.udid, state, a.probe_cmd) inj.append([t0, now(), rc]) - sj, err = h.wait(40) + sj, err = h.wait(40 + 9 * f) txt = open(raw).read() if os.path.exists(raw) else "" r = analysis.probe_latencies(analysis.parse_raw(txt), [(x[0], x[1]) for x in inj], 2.0) res[method] = {"answered": r["n"] - r["misses"], "n": r["n"], "p50_ms": r["p50_ms"], "call_p50_ms": r["call_p50_ms"], @@ -756,10 +755,11 @@ def add(name, fn, *opts): add("prepare", cmd_prepare, O("--udids", required=True), O("--roles", required=True), O("--install", action="store_true"), O("--quiesce", action="store_true")) add("settle", cmd_settle, O("--max-s", type=float, default=120), O("--bg-pct", type=float, default=40), O("--step", type=float, default=3)) - add("calibrate", cmd_calibrate, O("--udid", required=True), O("--methods", default="openurl,appearance"), O("--probe-cmd")) + add("calibrate", cmd_calibrate, O("--udid", required=True), O("--methods", default="openurl,appearance"), O("--probe-cmd"), O("--time-scale", type=float, default=1.0)) add("measure", cmd_measure, O("--udids", required=True), O("--roles", required=True), O("--mix", default="all-busy"), O("--secs", type=float, default=40), O("--start-at", type=float, default=0), O("--probe-method", default="openurl"), - O("--probe-every", type=float, default=2.5), O("--probe-cmd"), O("--agent-period", type=float, default=12)) + O("--probe-every", type=float, default=2.5), O("--probe-cmd"), O("--agent-period", type=float, default=12), + O("--no-strip", action="store_true")) add("shots", cmd_shots, O("--udids", required=True), O("--count", type=int, default=2)) add("cycle", cmd_cycle, O("--type", required=True), O("--runtime", required=True), O("--prefix", default="bench")) add("teardown", cmd_teardown) diff --git a/src/bench/guest/benchguest_stub.py b/src/bench/guest/benchguest_stub.py index b280ca9..a83a6fa 100644 --- a/src/bench/guest/benchguest_stub.py +++ b/src/bench/guest/benchguest_stub.py @@ -123,7 +123,7 @@ def _stub_run(argv): now = time.time() with open(os.path.join(d, "Documents", "pace.json"), "w") as f: for k in range(8): - f.write(json.dumps({"at": now - 40 + 5 * k, "counter": int((now - 40 + 5 * k - T0) * 60) % 65536}) + "\n") + f.write(json.dumps({"at": now - 40 + 5 * k, "counter": int((now - 40 + 5 * k - T0) * 60) % 65536, "fps": 60.0, "ticks": 300}) + "\n") out = d + "\n" elif argv[:3] == ["xcrun", "simctl", "list"]: out = json.dumps(RUNTIMES if argv[3] == "runtimes" else TYPES) if argv[3] in ("runtimes", "devicetypes") else "" diff --git a/src/bench/macbench/ab.py b/src/bench/macbench/ab.py new file mode 100644 index 0000000..fe131c4 --- /dev/null +++ b/src/bench/macbench/ab.py @@ -0,0 +1,88 @@ +"""`ab`: one knob, arms interleaved A B B A inside one window, only the within-window difference judged.""" +import time + +from . import abrule, core, gates, scale, stats +from .core import Stop +from .runctx import CmdKnob, LiveKnob, shell_runner + +METRICS = { + "total_fps": (lambda st: sum(s["fps"] for s in st["sims"] if s.get("fps") is not None), True), + "min_fps": (lambda st: min([s["fps"] for s in st["sims"] if s.get("fps") is not None] or [None]), True), + "worst_p95": (lambda st: max([s["p95_ms"] for s in st["sims"] if s.get("p95_ms") is not None] or [None]), False), + "worst_p99": (lambda st: max([s["p99_ms"] for s in st["sims"] if s.get("p99_ms") is not None] or [None]), False), + "input_p95": (lambda st: max([s["input_p95_ms"] for s in st["sims"] if s.get("input_p95_ms") is not None] or [None]), False), + "gpu_ms_per_frame": (lambda st: st["system"].get("gpu_ms_per_frame"), False), + "qemu_cores": (lambda st: st["system"].get("qemu_cpu_cores"), False), +} +DEFAULT_TAILS = {"total_fps": ["worst_p95", "worst_p99"], "min_fps": ["worst_p95", "worst_p99"], "gpu_ms_per_frame": ["worst_p95"], + "qemu_cores": ["min_fps"], "worst_p95": ["min_fps"], "worst_p99": ["min_fps"], "input_p95": ["min_fps"]} + + +def make_knob(a, ctx): + if a.knob and a.knob.startswith("live:"): + k, v = a.knob[5:].split("=", 1) + if not ctx.live_conf: + raise Stop("--knob live:KEY=VALUE needs the VM's live switch file (not found)") + return LiveKnob(ctx.live_conf, k, v, ctx.run) + if a.set_cmd and a.unset_cmd: + return CmdKnob(a.set_cmd, a.unset_cmd, ctx.run, shell_runner()) + raise Stop("a knob is needed: --knob live:KEY=VALUE, or --set-cmd and --unset-cmd") + + +def run_ab(ctx): + a = ctx.a + metric = a.metric + hb = METRICS[metric][1] + tails = [t for t in (a.tails.split(",") if a.tails else DEFAULT_TAILS.get(metric, [])) if t] + rule = "AB rule (fixed before the run): knob %s; metric %s (%s better); ADOPT only if the mean effect of the 2 pairs exceeds %.1f%% and both pairs agree in sign; tails %s may not worsen by more than %.1f%%; order A B B A; no re-run" % ( + a.knob or "cmd", metric, "higher" if hb else "lower", a.threshold * 100, ",".join(tails) or "none", a.tail_tol * 100) + ctx.log(rule) + ctx.doc["params"]["rule"] = rule + knob = make_knob(a, ctx) + info = core.setup_guest(ctx) + core.capture_env(ctx, info) + ctx.marks = {"coh": ctx.coh.mark(), "t0": time.time()} + n, mix = a.n, a.mix + reason = core.guard(ctx, n, "before the A/B") + if reason: + raise Stop(reason) + ctx.host.n = n + core.grow(ctx, n, core.roles_for(mix, n)) + ctx.probe_method = scale.calibrate(ctx, ctx.sims[0])["chosen"] if mix in ("all-busy", "one-busy") and metric == "input_p95" else "none" + core.settle(ctx) + ctx.doc["data"] = {"knob": knob.describe(), "metric": metric, "arms": [], "n": n, "mix": mix} + vals = {"A": [], "B": []} + steps = {"A": [], "B": []} + bad = None + ctx.run.on_deadline = lambda: (_save(ctx), None)[1] + for arm in abrule.abba_order(): + (knob.set if arm == "B" else knob.unset)() + time.sleep(a.arm_settle) + g, s0, s1, t_go = core.window(ctx, a.secs, mix) + step = core.build_step(ctx, n, mix, g, s0, s1, t_go) + core.step_gates(ctx, step) + steps[arm].append(step) + v = {k: f(step) for k, (f, _) in METRICS.items()} + vals[arm].append(v) + ctx.doc["data"]["arms"].append({"arm": arm, "values": v, "gates_ok": step["gates_ok"], "stable": step["stable"]}) + _save(ctx) + if not (step["gates_ok"] and step["stable"]) and bad is None: + bad = "arm %s failed its gates or stability: %s" % (arm, [k for k, g in step["gates"].items() if g["status"] == "FAIL"] or step["stability"]) + knob.unset() + a1, a2 = vals["A"][0][metric], vals["A"][1][metric] + b1, b2 = vals["B"][0][metric], vals["B"][1][metric] + tl = {t: (vals["A"][0][t], vals["B"][0][t], vals["A"][1][t], vals["B"][1][t], METRICS[t][1]) for t in tails} + res = abrule.decide(a1, b1, a2, b2, hb, a.threshold, tl, a.tail_tol) + if bad: + res["decision"] = "CLOSE" + res["why"].append(bad) + line = abrule.decision_line(knob.describe() if a.knob else "cmd", metric, ((a1, a2), (b1, b2)), res, a.threshold) + ctx.doc["data"]["decision"] = res + ctx.doc["data"]["line"] = line + ctx.doc["data"]["pairs"] = [{"pair": 1, "A": a1, "B": b1}, {"pair": 2, "A": a2, "B": b2}] + ctx.section("ab", "PASS" if not bad else "FAIL", line) + ctx.save() + + +def _save(ctx): + ctx.save() diff --git a/src/bench/macbench/analysis.py b/src/bench/macbench/analysis.py index 7704337..9c6af3b 100644 --- a/src/bench/macbench/analysis.py +++ b/src/bench/macbench/analysis.py @@ -79,7 +79,7 @@ def sim_metrics(idx, role, raw, injections, nominal_fps=60.0, timeout_s=2.0, app def fairness(sims): """Jain's index and min/max over the distinct fps of the simulators that were meant to animate.""" - f = [s["fps"] for s in sims if s.get("role") == "busy" and s.get("fps") is not None] + f = [s["fps"] for s in sims if s.get("role") in ("busy", "agent") and s.get("fps") is not None] return {"jain": stats.r1(stats.jain(f), 3), "min_max": stats.r1(stats.min_max_ratio(f), 3), "n": len(f)} diff --git a/src/bench/macbench/chart.py b/src/bench/macbench/chart.py index 55bac10..e6c3475 100644 --- a/src/bench/macbench/chart.py +++ b/src/bench/macbench/chart.py @@ -58,19 +58,19 @@ def py(y): return "\n".join(o) -def _per_sim(steps, key, role="busy"): +def _per_sim(steps, key, role=("busy", "agent")): pts = [] for st in steps: for s in st["sims"]: - if s.get("role") == role and s.get(key) is not None: + if s.get("role") in role and s.get(key) is not None: pts.append((st["n"], s[key])) return pts -def _agg(steps, fn, key, role="busy"): +def _agg(steps, fn, key, role=("busy", "agent")): out = [] for st in steps: - v = [s[key] for s in st["sims"] if s.get("role") == role and s.get(key) is not None] + v = [s[key] for s in st["sims"] if s.get("role") in role and s.get(key) is not None] out.append((st["n"], fn(v) if v else None)) return out diff --git a/src/bench/macbench/cli.py b/src/bench/macbench/cli.py new file mode 100644 index 0000000..869be10 --- /dev/null +++ b/src/bench/macbench/cli.py @@ -0,0 +1,319 @@ +"""`macbox bench `: argument parsing and the wiring of a run.""" +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time + +from . import abrule, ab, chart, compare, core, fakes, hostprobe, others, scale, schema, targets +from .runctx import DeadlineError, Run +from .transport import GuestError, LocalGuest, SshGuest + +BENCH_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def add_common(p): + p.add_argument("--out", default=os.environ.get("BENCH_OUT", "bench-results"), help="directory for the result file") + p.add_argument("--deadline", type=int, default=5400, help="hard deadline in seconds: cleanup runs and the tool exits (default 5400)") + p.add_argument("--tag", action="append", default=[], help="key=value recorded in the result (for example mesa=26.2.4)") + p.add_argument("--build-id", default=os.environ.get("BENCH_BUILD_ID"), help="the VM's build id (macbox bench fills it in)") + p.add_argument("--min-guest-free-gib", type=float, default=1.5, help="refuse to boot more simulators below this guest free memory") + p.add_argument("--min-host-free-gib", type=float, default=6.0, help="refuse to boot more simulators below this host available memory") + p.add_argument("--min-disk-gib", type=float, default=12.0, help="refuse below this free disk space (the VM's disk, host side)") + p.add_argument("--disk-path", default=os.environ.get("BENCH_HOME"), help=argparse.SUPPRESS) + p.add_argument("--allow-foreign", action="store_true", help="go on although simulators outside the bench are booted (recorded)") + p.add_argument("--allow-restart", action="store_true", help="let a NEW device type restart CoreSimulatorService (only when nothing is booted)") + p.add_argument("--scale-x", dest="scale_x", type=int, choices=(1, 2, 3), default=1, help="device pixel scale: 1 = the macbox 1x type, 3 = the stock iPhone (default 1)") + p.add_argument("--secs", type=float, default=40, help="length of a measurement window, seconds") + p.add_argument("--nominal-fps", type=float, default=60.0) + p.add_argument("--probe-method", default="auto", choices=("auto", "openurl", "appearance", "cmd", "none"), help="input injection (auto: calibrated at the first N)") + p.add_argument("--probe-cmd", default=None, help="for --probe-method cmd: a shell command with {udid}; it must flip GlassBench's marker (a tap)") + p.add_argument("--probe-every", type=float, default=2.5, help="seconds between input probes of one simulator") + p.add_argument("--probe-timeout", type=float, default=2.0, help="an input probe not answered in this time is a miss (counted at this value)") + p.add_argument("--agent-period", type=float, default=12.0) + p.add_argument("--settle-max", type=float, default=120) + p.add_argument("--lead", type=float, default=None, help=argparse.SUPPRESS) + p.add_argument("--settle-step", type=float, default=3.0, help=argparse.SUPPRESS) + p.add_argument("--settle-bg", type=float, default=40, help="the guest counts as settled below this background CPU %% (of one core)") + p.add_argument("--quiesce", action="store_true", help="disable mediaanalysisd in each simulator (recorded; production does not)") + p.add_argument("--fail-on-swap", action="store_true", help="a guest swap-out in a window fails the memory gate (default: reported, host swap fails)") + p.add_argument("--golden-file", default=os.path.join(BENCH_DIR, "baselines", "golden.json")) + + +def parser(): + ap = argparse.ArgumentParser(prog="macbox bench", description="scaling, regression and A/B suite for the macbox simulator stack (src/bench/README.md)") + sp = ap.add_subparsers(dest="cmd", required=True) + s = sp.add_parser("scale", help="N = 1, 2, 3... simulators, per-simulator numbers, the maximum sustainable N per quality target") + add_common(s) + s.add_argument("--n", dest="ns", default="1,2,3,4,5,6", help="the N list, e.g. 1,2,3,4,6,8") + s.add_argument("--mix", choices=("all-busy", "one-busy", "agent", "idle"), default="all-busy") + s.add_argument("--targets", action="append", default=None, help="'name: fps>=55, p95<=25' (repeatable; default: smooth60, smooth30, responsive)") + s.add_argument("--floor", default="fps>=20", help="stop the sweep when a simulator breaches this (default fps>=20)") + s.add_argument("--shots", type=int, default=2, help="screenshots per simulator per step for the tear gate (0: none)") + s.add_argument("--no-observer-check", dest="observer_check", action="store_false", help="skip the strip-read cost window at the first N") + s.add_argument("--no-golden", dest="golden_phase", action="store_false", help="skip the static reference screen at the first N") + for name, hlp in (("idle", "the VM with no simulators: the standing cost"), ("lifecycle", "create, boot, install, launch, shut down, resume, delete; leak per cycle; boot storm"), + ("soak", "N busy simulators for minutes: memory growth, frame rate drift"), ("build", "build timing: the bench tools, optionally an xcodebuild"), + ("panel", "the Device panel's stream (optional, needs Simlock/T3 Code)"), ("golden", "record the static screen's pixel hash for this setup"), + ("ab", "A/B one knob, A B B A inside one window, ADOPT or CLOSE")): + p = sp.add_parser(name, help=hlp) + add_common(p) + if name == "idle": + p.add_argument("--bar-qemu-cores", type=float, default=0.15, help="PASS bar for QEMU's CPU in cores") + if name == "lifecycle": + p.add_argument("--cycles", type=int, default=3) + p.add_argument("--storm", type=int, default=4, help="boot this many at once (0: skip)") + p.add_argument("--storm-bar-s", type=float, default=300) + p.add_argument("--leak-rss-mb", type=float, default=20, help="FAIL above this QEMU RSS growth per cycle (MB)") + p.add_argument("--leak-guest-mb", type=float, default=30, help="FAIL above this guest memory growth per cycle (MB)") + if name == "soak": + p.add_argument("--soak-n", type=int, default=3) + p.add_argument("--soak-minutes", type=float, default=15) + p.add_argument("--soak-every", type=float, default=30) + p.add_argument("--soak-rss-mb-min", type=float, default=5, help="FAIL above this QEMU RSS growth (MB/min)") + p.add_argument("--soak-guest-mb-min", type=float, default=10, help="FAIL above this guest memory growth (MB/min)") + if name == "build": + p.add_argument("--xcodebuild", default="", help="arguments for xcodebuild (a project or workspace and scheme) to time cold and warm") + if name == "panel": + p.add_argument("--panel-cmd", default=None, help="command printing {\"frames\": [[t, counter], ...]}") + p.add_argument("--panel-fps", type=float, default=55.0) + p.add_argument("--panel-p95", type=float, default=25.0) + if name == "ab": + p.add_argument("--knob", help="live:KEY=VALUE (a line in the VM's live switch file), or use --set-cmd/--unset-cmd") + p.add_argument("--set-cmd") + p.add_argument("--unset-cmd") + p.add_argument("--n", type=int, default=3, help="simulators") + p.add_argument("--mix", choices=("all-busy", "one-busy", "agent"), default="all-busy") + p.add_argument("--metric", choices=sorted(ab.METRICS), default="total_fps") + p.add_argument("--tails", default=None, help="comma list of metrics that may not worsen (default per metric)") + p.add_argument("--threshold", type=float, default=abrule.DEFAULT_THRESHOLD, help="effect needed to ADOPT, fixed BEFORE the run (default 0.04 = the run-to-run noise)") + p.add_argument("--tail-tol", type=float, default=abrule.DEFAULT_THRESHOLD) + p.add_argument("--arm-settle", type=float, default=10) + c = sp.add_parser("compare", help="two result files") + c.add_argument("a") + c.add_argument("b") + c.add_argument("--noise", type=float, default=compare.NOISE) + h = sp.add_parser("chart", help="an HTML/SVG scaling chart from a scale result") + h.add_argument("result") + h.add_argument("-o", "--output") + r = sp.add_parser("report", help="the PASS/FAIL lines of a result") + r.add_argument("result") + t = sp.add_parser("tools", help="copy the bench to the Mac and build simfps and GlassBench there") + t.add_argument("--no-build", action="store_true") + sp.add_parser("check", help="what this host and the VM offer the bench") + return ap + + +def log(msg): + print("%s %s" % (time.strftime("%H:%M:%S"), msg), flush=True) + + +def qemu_pid_from_docker(): + cont = os.environ.get("MACBOX_CONTAINER", "macbox") + try: + out = subprocess.run([os.environ.get("MACBOX_DOCKER", "docker"), "top", cont, "-eo", "pid,args"], capture_output=True, text=True, timeout=15).stdout + except (OSError, subprocess.TimeoutExpired): + return None + for line in out.splitlines(): + if "qemu-system-x86_64" in line: + return int(line.split()[0]) + return None + + +class Golden: + def __init__(self, path): + self.path = path + try: + with open(path) as f: + self.d = json.load(f) + except (OSError, ValueError): + self.d = {} + + def key(self, ctx, w, h): + g = ctx.doc["env"].get("guest", {}) + return "%s|runtime %s|%s|%dx%d" % (g.get("macos"), g.get("runtime_used"), g.get("device_type"), w, h) + + def record(self, key, h): + self.d[key] = h + os.makedirs(os.path.dirname(self.path), exist_ok=True) + with open(self.path, "w") as f: + json.dump(self.d, f, indent=1, sort_keys=True) + f.write("\n") + + +def build_ctx(a, kind): + dry = os.environ.get("DRY") == "1" + run = Run(a.deadline, log, margin=float(os.environ.get("BENCH_DEADLINE_MARGIN", "60"))) + names = [os.environ.get("MACBOX_SSH_HOST", ""), os.environ.get("USER", ""), os.environ.get("MACBOX_GUEST_USER", "")] + clean = schema.scrubber([n for n in names if n]) + stamp = schema.stamp() + set_name = "bench-%s" % stamp + faults = os.environ.get("BENCH_STUB_FAULT", "") + if dry: + tmp = os.environ.get("BENCH_DRY_KEEP") or tempfile.mkdtemp(prefix="macbench-dry-") + os.makedirs(tmp, exist_ok=True) + if not os.environ.get("BENCH_DRY_KEEP"): + run.teardown.insert(0, ("dry state directory", lambda: shutil.rmtree(tmp, ignore_errors=True))) + a.lead, a.settle_step = 1.0, 0.3 + host = fakes.FakeHost(faults) + coh = fakes.FakeCoherent(host, faults) + guest = LocalGuest(run, set_name, os.path.join(tmp, "guest"), BENCH_DIR, faults) + guest.fault_ssh_timeout = int(os.environ.get("BENCH_STUB_SSH_TIMEOUT", "0")) or None + a.build_id = a.build_id or "dry-run" + live = os.path.join(tmp, "live.conf") + with open(live, "w") as f: + f.write("surface_coherent=1\n") + a.disk_path = tmp + else: + pid = int(os.environ.get("BENCH_QEMU_PID") or qemu_pid_from_docker() or 0) + if not pid: + raise SystemExit("macbox bench: no running QEMU (macbox up first)") + runp = os.environ.get("BENCH_RUN") or "" + live = os.path.join(runp, "live.conf") if runp and os.path.exists(os.path.join(runp, "live.conf")) else None + host = hostprobe.Host(pid, live_conf=live) + from .gates import Coherent + coh = Coherent(os.path.join(runp, "coherent") if runp else None, os.environ.get("BENCH_QEMU_LOG"), + os.path.join(os.environ.get("BENCH_HOME", ""), "home/.cache/m2v-gpu/hang") if os.environ.get("BENCH_HOME") else None) + guest = SshGuest(run, os.environ.get("BENCH_SSH") or os.environ.get("MACBOX_SSH_HOST") or "macbox", set_name) + os.makedirs(a.out, exist_ok=True) + params = {k: v for k, v in vars(a).items() if k not in ("cmd", "out", "golden_file", "disk_path", "tag") and not callable(v)} + doc = schema.new_result(kind if kind in schema.KINDS else "scale", params, {"host": {}, "guest": {}}, stamp) + if kind == "scale": + doc["data"] = {"mix": a.mix, "scale_x": a.scale_x, "steps": [], "summary": {"max_n": {}}} + path = os.path.join(a.out, "%s-%s%s.json" % (kind, stamp, "-dry" if dry else "")) + ctx = core.Ctx(a, run, guest, host, coh, doc, path, clean, log) + ctx.live_conf = live + gold = Golden(a.golden_file) + ctx.golden_store = gold + ctx.golden = lambda w, h: (gold.d.get(gold.key(ctx, w, h)), gold.key(ctx, w, h)) + ctx.golden_key_for = lambda w, h: gold.key(ctx, w, h) + ctx.record_golden = gold.record + if dry: + ctx.dry_dir = tmp + return ctx + + +def execute(a, kind, body): + ctx = build_ctx(a, kind) + run = ctx.run + run.install() + if ctx.host.dry: + log("DRY=1: stub guest, fake host; nothing real is touched") + rc = 0 + try: + body(ctx) + except core.Stop as e: + ctx.section("stopped", "FAIL", "STOPPED: %s" % e) + rc = 2 + except (GuestError, DeadlineError) as e: + ctx.section("error", "FAIL", "ERROR: %s: %s" % (type(e).__name__, e)) + rc = 2 + finally: + run.cleanup() + info = ctx.cleanup_info + if info or any(l for l in run.teardown): + left_ok = (not info) or (info.get("booted", 0) == 0 and info.get("left", 0) == 0) + sec = "PASS" if left_ok and not run.problems else "FAIL" + ctx.section("cleanup", sec, "CLEANUP %s: bench simulators left booted %s, devices left %s, knobs restored %s%s" % ( + sec, info.get("booted", "n/a"), info.get("left", "n/a"), "yes" if not run.problems else "NO", + ("; " + "; ".join(run.problems)) if run.problems else "")) + problems = ctx.save() + fails = [s for s in ctx.doc["sections"] if s["status"] == "FAIL"] + print("result: %s%s" % (os.path.basename(ctx.out_path), "" if not problems else " (NOT publishable as is: %s)" % "; ".join(problems[:3]))) + if fails: + return max(rc, 1) + return rc + + +def main(argv=None): + a = parser().parse_args(argv) + c = a.cmd + if c in ("compare", "report", "chart"): + return offline(a) + if c == "tools": + return tools(a) + if c == "check": + return check(a) + if c == "scale": + a.ns = [int(x) for x in a.ns.split(",")] + a.targets = a.targets or targets.DEFAULT_TARGETS + for t in a.targets + ["floor: " + a.floor]: + targets.parse_target(t) + return execute(a, "scale", scale.run_scale) + body = {"idle": others.run_idle, "lifecycle": others.run_lifecycle, "soak": others.run_soak, "build": others.run_build, + "panel": others.run_panel, "golden": others.run_golden, "ab": ab.run_ab}[c] + for k, v in (("ns", [1]), ("mix", "all-busy"), ("targets", targets.DEFAULT_TARGETS), ("floor", "fps>=20"), ("shots", 2), + ("observer_check", False), ("golden_phase", False)): + if not hasattr(a, k): + setattr(a, k, v) + return execute(a, {"golden": "gates"}.get(c, c), body) + + +def offline(a): + if a.cmd == "compare": + text, worse = compare.compare(schema.load(a.a), schema.load(a.b), a.noise) + print(text) + return 1 if worse else 0 + doc = schema.load(a.result) + if a.cmd == "report": + print("%s %s %s (build %s)" % (doc["kind"], doc["id"], doc["created"], doc["env"].get("build_id"))) + for s in doc["sections"]: + print("%-4s %s" % (s["status"], s["line"])) + return 1 if any(s["status"] == "FAIL" for s in doc["sections"]) else 0 + if doc["kind"] != "scale": + print("chart: only scale results have a scaling chart", file=sys.stderr) + return 2 + out = a.output or os.path.splitext(a.result)[0] + ".html" + with open(out, "w") as f: + f.write(chart.scale_chart(doc)) + print(out) + return 0 + + +def tools(a): + ssh = os.environ.get("BENCH_SSH") or os.environ.get("MACBOX_SSH_HOST") or "macbox" + if os.environ.get("DRY") == "1": + print("DRY=1: would copy %s to the guest's ~/.macbox/bench and run build.sh" % BENCH_DIR) + return 0 + tar = subprocess.Popen(["tar", "-C", BENCH_DIR, "--exclude=__pycache__", "--exclude=tests", "--exclude=baselines", "-cf", "-", "."], stdout=subprocess.PIPE) + r = subprocess.run(["ssh", ssh, "mkdir -p ~/.macbox/bench && tar -C ~/.macbox/bench -xf -"], stdin=tar.stdout) + tar.wait() + if r.returncode: + return 2 + if a.no_build: + return 0 + return subprocess.run(["ssh", ssh, "cd ~/.macbox/bench && ./build.sh"]).returncode + + +def check(a): + pid = int(os.environ.get("BENCH_QEMU_PID") or qemu_pid_from_docker() or 0) + ok = True + + def line(good, text): + nonlocal ok + ok &= good + print("%s %s" % ("ok " if good else "MISS", text)) + line(bool(pid), "QEMU running (pid known)") + if pid: + hp = hostprobe.Host(pid, live_conf=os.path.join(os.environ.get("BENCH_RUN", ""), "live.conf")) + s = hp.snap() + line(bool(s["threads"]), "QEMU thread CPU readable (%d threads)" % len(s["threads"])) + line(s["gpu"] is not None, "GPU render time readable from DRM fdinfo (i915 drm-engine-render)") + line(s["mem"].get("cg_current") is not None, "the VM's cgroup memory readable") + n, note = hostprobe.kernel_gpu_lines(time.time() - 60) + line(n is not None, "kernel journal readable for GPU error lines" + ("" if n is not None else " (%s)" % note)) + line(hp.mem_avail_gb() is not None, "host available memory readable") + ssh = os.environ.get("BENCH_SSH") or os.environ.get("MACBOX_SSH_HOST") or "macbox" + try: + out = subprocess.run(["ssh", "-o", "ConnectTimeout=8", "-o", "BatchMode=yes", ssh, "cd ~/.macbox/bench && xcrun python3 guest/benchguest.py info"], + capture_output=True, text=True, timeout=60).stdout + d = json.loads(out.strip().splitlines()[-1]) + line(True, "guest answers: macOS %s, %s" % (d.get("macos"), d.get("xcode"))) + line(d.get("has_simfps"), "simfps built in the guest (macbox bench tools)") + line(d.get("has_app"), "GlassBench.app built in the guest (macbox bench tools)") + except (OSError, ValueError, IndexError, subprocess.TimeoutExpired): + line(False, "the guest's bench helper (macbox bench tools)") + return 0 if ok else 1 diff --git a/src/bench/macbench/compare.py b/src/bench/macbench/compare.py index f162cf0..68f9d17 100644 --- a/src/bench/macbench/compare.py +++ b/src/bench/macbench/compare.py @@ -5,9 +5,9 @@ # (label, getter on a step, higher is better) STEP_ROWS = [ - ("min distinct fps", lambda st: _min([s.get("fps") for s in st["sims"] if s.get("role") == "busy"]), True), - ("worst p95 ms", lambda st: _max([s.get("p95_ms") for s in st["sims"] if s.get("role") == "busy"]), False), - ("worst p99 ms", lambda st: _max([s.get("p99_ms") for s in st["sims"] if s.get("role") == "busy"]), False), + ("min distinct fps", lambda st: _min([s.get("fps") for s in st["sims"] if s.get("role") in ("busy", "agent")]), True), + ("worst p95 ms", lambda st: _max([s.get("p95_ms") for s in st["sims"] if s.get("role") in ("busy", "agent")]), False), + ("worst p99 ms", lambda st: _max([s.get("p99_ms") for s in st["sims"] if s.get("role") in ("busy", "agent")]), False), ("longest gap s", lambda st: _max([s.get("longest_gap_s") for s in st["sims"]]), False), ("input p95 ms", lambda st: _max([s.get("input_p95_ms") for s in st["sims"]]), False), ("Jain index", lambda st: (st.get("fairness") or {}).get("jain"), True), diff --git a/src/bench/macbench/core.py b/src/bench/macbench/core.py new file mode 100644 index 0000000..0adfee3 --- /dev/null +++ b/src/bench/macbench/core.py @@ -0,0 +1,241 @@ +"""The parts every scenario shares: the context, the guard, simulator preparation, one measurement window and the +step record built from it.""" +import time + +from . import analysis, gates, hostprobe, schema, stats, targets + + +class Stop(Exception): + """A reason to end the sweep early; the result so far is kept.""" + + +class Ctx: + def __init__(self, args, run, guest, host, coherent, doc, out_path, clean, log=print): + self.a, self.run, self.guest, self.host, self.coh = args, run, guest, host, coherent + self.doc, self.out_path, self.clean, self.log = doc, out_path, clean, log + self.offset = 0.0 + self.sims = [] # [{"udid", "idx"}] booted and prepared, in order + self.type_id = self.runtime = None + self.probe_method = None + self.cleanup_info = {} + self.live_conf = None + self.est_sim_mb = 1100.0 # a simulator's guest memory, until measured + self.marks = {} + self.golden = None # fn(w, h) -> (hash or None, key) + self.pixel_gate = None + + def save(self): + return schema.write(self.doc, self.out_path, self.clean) + + def section(self, name, status, line, detail=None): + schema.add_section(self.doc, name, status, line, detail) + self.log("%s %s" % (status, line)) + + +def capture_env(ctx, info): + host, sw, qenv = ctx.host.env() + g = {k: info.get(k) for k in ("macos", "xcode", "arch", "cpus", "mem_mb")} + rts = info.get("runtimes") or [] + g["runtimes"] = ["%s %s" % (r.get("name"), r.get("build")) for r in rts] + ctx.doc["env"].update({"build_id": ctx.a.build_id or "unknown", "host": host, "guest": g, + "switches": sw, "qemu_env": qenv, "tags": dict(kv.split("=", 1) for kv in (ctx.a.tag or []) if "=" in kv), + "dry": bool(ctx.host.dry)}) + + +def guard(ctx, k_new, label): + """Refuses to boot k_new more simulators when the guest or the host would run short: returns a reason or None. + The sweep stops there; it never swaps the VM to death.""" + a = ctx.a + hm = ctx.host.mem_avail_gb() + if hm is not None and hm - k_new * 0.5 < a.min_host_free_gib: + return "host free memory %.1f GiB below the %.1f GiB floor %s" % (hm, a.min_host_free_gib, label) + df = ctx.host.disk_free_gb(a.disk_path or "/") + if df is not None and df < a.min_disk_gib: + return "free disk %.0f GiB below the %.0f GiB floor %s" % (df, a.min_disk_gib, label) + m = ctx.guest.call(["memcheck"], 60) + free_gib = (m["mem"]["free_mb"] - k_new * ctx.est_sim_mb) / 1024.0 + if free_gib < a.min_guest_free_gib: + return "guest free memory would be %.1f GiB (%.1f now, %d x ~%d MB to boot), below the %.1f GiB floor %s" % ( + free_gib, m["mem"]["free_mb"] / 1024.0, k_new, ctx.est_sim_mb, a.min_guest_free_gib, label) + if (m.get("pressure") or 1) >= 2: + return "guest memory pressure level %s %s" % (m.get("pressure"), label) + if m.get("foreign_booted", 0) > 0 and not a.allow_foreign: + return "%d simulator(s) outside the bench are booted (the pool or someone's lease): not a clean measurement; --allow-foreign to go on" % m["foreign_booted"] + return None + + +def roles_for(mix, n): + if mix == "all-busy": + return ["busy"] * n + if mix == "one-busy": + return ["busy"] + ["static"] * (n - 1) + if mix == "agent": + return ["agent"] * n + if mix == "idle": + return ["idle"] * n + raise ValueError("unknown mix %r" % mix) + + +def setup_guest(ctx): + """Device set, device type, tools check. Returns the guest info.""" + a = ctx.a + info = ctx.guest.call(["info"], 120) + if not info.get("has_simfps") or not info.get("has_app"): + raise Stop("the bench tools are not built in the guest: macbox bench tools") + ctx.guest.call(["init-set"], 60) + ctx.run.teardown.append(("bench simulators", lambda: ctx.cleanup_info.update(ctx.guest.call(["teardown"], 240)))) + t = ctx.guest.call(["ensure-type", "--scale", str(a.scale_x)] + (["--allow-restart"] if a.allow_restart else []), 180) + if t.get("needs_restart"): + raise Stop("device type %s was just created and CoreSimulatorService must restart to load it, which would shut " + "down every booted simulator: stop them, then run again with --allow-restart (or `macbox pool create`)" % t["type"]) + ctx.type_id, ctx.runtime = t["type"], t["runtime"] + ctx.doc["env"]["guest"]["runtime_used"] = t.get("runtime_version") + ctx.doc["env"]["guest"]["device_type"] = "stock %s" % t["stock_name"] if a.scale_x == 3 else "macbox %dx of %s" % (a.scale_x, t["stock_name"]) + ctx.offset = ctx.guest.clock_offset() + return info + + +def grow(ctx, n, roles, install=True): + """Make and boot simulators up to n (the ones already running stay), launch each with its role. Returns boot timings.""" + have = len(ctx.sims) + new = n - have + boot = {} + if new > 0: + r = ctx.guest.call(["create", "--n", new, "--type", ctx.type_id, "--runtime", ctx.runtime, "--start", have], 240) + fresh = r["sims"] + us = ",".join(s["udid"] for s in fresh) + b = ctx.guest.call(["boot", "--udids", us] + (["--parallel"] if new > 1 else []), 900) + bad = [s for s in b["sims"] if not s.get("ok")] + if bad: + raise Stop("a simulator did not boot: %s" % bad[0].get("err")) + for s, bs in zip(fresh, b["sims"]): + ctx.sims.append({"idx": s["idx"], "udid": s["udid"]}) + boot = {"new": new, "all_usable_s": b["all_s"], "each_usable_s": [s["usable_s"] for s in b["sims"]]} + # roles may change with N (one-busy: only the first animates); (re)launch every simulator whose role changed or is new + for i, s in enumerate(ctx.sims): + want = roles[i] + if s.get("role") != want: + ctx.guest.call(["prepare", "--udids", s["udid"], "--roles", want] + (["--install"] if not s.get("installed") else []) + + (["--quiesce"] if ctx.a.quiesce else []), 300) + s["role"], s["installed"] = want, True + return boot + + +def settle(ctx): + r = ctx.guest.call(["settle", "--max-s", ctx.a.settle_max, "--bg-pct", ctx.a.settle_bg, "--step", ctx.a.settle_step], ctx.a.settle_max + 120) + return r + + +def window(ctx, secs, mix, no_strip=False): + """One measurement window over every simulator of ctx.sims: the guest measures, the host samples the same window. + Returns (guest result, host system dict, window start/end in guest time).""" + a = ctx.a + udids = ",".join(s["udid"] for s in ctx.sims) + roles = ",".join(s["role"] for s in ctx.sims) + lead = a.lead if a.lead is not None else 8 + 2 * len(ctx.sims) + t_go = time.time() + ctx.offset + lead + ctx.run.need(secs + lead + 60, "a %d s window" % secs) + args = ["measure", "--udids", udids, "--roles", roles, "--mix", mix, "--secs", secs, "--start-at", "%.3f" % t_go, + "--probe-method", ctx.probe_method or "openurl", "--probe-every", a.probe_every, "--agent-period", a.agent_period] + if a.probe_cmd: + args += ["--probe-cmd", a.probe_cmd] + pend = ctx.guest.start(args, secs + lead + 240) + host_go = t_go - ctx.offset + time.sleep(max(0, host_go - time.time())) + s0 = ctx.host.snap() + time.sleep(max(0, secs - (time.time() - host_go))) + s1 = ctx.host.snap() + g = pend.wait() + return g, s0, s1, t_go + + +def system_dict(ctx, g, s0, s1, frames_total): + dt = max(s1["mono"] - s0["mono"], 0.001) + th = hostprobe.thread_delta(s0["threads"], s1["threads"], dt, getattr(ctx.host, "hz", 100)) + gpu_ns = None if s0["gpu"] is None or s1["gpu"] is None else s1["gpu"] - s0["gpu"] + gu = g["guest"] + return { + "qemu_cpu_cores": th["cores"], "qemu_user_cores": th["user"], "qemu_guest_cores": th["guest"], "qemu_sys_cores": th["sys"], + "qemu_top_threads": th["top"], + "gpu_busy_pct": None if gpu_ns is None else round(100.0 * gpu_ns / 1e9 / dt, 1), + "gpu_ms_per_frame": None if gpu_ns is None or not frames_total else round(gpu_ns / 1e6 / frames_total, 2), + "window_s": round(dt, 2), + "guest": {"mem_used_mb": gu["mem"]["used_mb"], "mem_free_mb": gu["mem"]["free_mb"], "compressed_mb": gu["mem"]["compressed_mb"], + "swapins_d": gu["swapins_d"], "swapouts_d": gu["swapouts_d"], "pageouts_d": gu["pageouts_d"], + "pressure_max": gu["pressure_max"], "total_cpu_pct": gu["total_cpu_pct"], "sims_cpu_pct": gu["sims_cpu_pct"], + "other_cpu_pct": gu["other_cpu_pct"], "sims_rss_mb": gu["sims_rss_mb"]}, + "host": hostprobe.mem_delta(s0["mem"], s1["mem"], dt), + } + + +def build_step(ctx, n, mix, g, s0, s1, t_go, boot=None): + """The step record: per simulator metrics, system numbers, stability.""" + a = ctx.a + secs = g["secs"] + t_end = t_go + a.secs + sims, frames_total = [], 0 + for i, s in enumerate(g["sims"]): + role = s["role"] + raw = [r for r in analysis.parse_raw(s["raw"]) if r[0] >= t_go - 0.001 and r[0] <= t_end + 0.5] or analysis.parse_raw(s["raw"])[:1] + inj = [(x[0], x[1]) for x in s["injections"] if x[2] == 0 or True] + if role == "agent": + sm = {"idx": i, "role": role} + a_ = analysis.strip.analyze_counters([(t, c) for t, c, _ in raw]) + sm.update(analysis.motion_metrics([t for t, _ in a_["unique"]], a.nominal_fps)) + sm["strip"] = {k: a_[k] for k in ("repeats", "lost", "backwards", "jumps", "unreadable", "span_ticks")} + ok, why = analysis.strip.strip_gate(a_) + sm["strip_gate"] = {"ok": ok, "why": why} + sm.update(analysis.agent_metrics(s["actions"], raw)) + else: + ticks = None + if role == "busy" and s["pace"] and raw: + ps = analysis.pace_summary(s["pace"], raw[0][0], raw[-1][0], t_go, t_end) + ticks = ps["app_ticks"] + sm = analysis.sim_metrics(i, role, raw, inj, a.nominal_fps, a.probe_timeout, ticks) + if role == "busy" and s["pace"] and raw: + sm["app_tick_fps"] = ps["app_tick_fps"] + if role in ("busy", "agent") and sm.get("frames"): + frames_total += sm["frames"] + if s.get("simfps"): + sm["surface_fps"] = s["simfps"].get("fps") + if s.get("simfps_error"): + sm["observer_error"] = s["simfps_error"] + p = s.get("proc") or {} + sm["guest_cpu_pct"], sm["guest_rss_mb"] = p.get("cpu_pct"), p.get("rss_mb") + if sm.get("longest_gap_s") is not None and role == "static": + sm["longest_gap_s"] = None + sims.append(sm) + system = system_dict(ctx, g, s0, s1, frames_total) + died = [s["idx"] for s in sims if g["sims"][s["idx"]].get("state_after") != "Booted"] + kern, knote = ctx.host.kernel_gpu(s0["wall"]) + stab = {"new_ips": len(g["new_ips"]), "ips": g["new_ips"][:5], "sims_died": died, "kernel_gpu_lines": kern, "kernel_note": knote, + "qemu_same": ctx.host.same_qemu(), "observer_errors": [s["idx"] for s in sims if s.get("observer_error")]} + stab["ok"] = not (stab["new_ips"] or died or kern or not stab["qemu_same"]) + return {"n": n, "mix": mix, "sims": sims, "system": system, "stability": stab, "boot": boot or {}} + + +def step_gates(ctx, step, shots=None): + gs = {"strip": gates.strip_gate_of(step["sims"]), "kernel_gpu": gates.kernel_gate(step["stability"]["kernel_gpu_lines"], step["stability"]["kernel_note"])} + gs.update(ctx.coh.check(ctx.marks["coh"])) + if shots: + gs.update(shots) + swapped = step["system"]["guest"]["swapouts_d"] or 0 + hs = step["system"]["host"] + host_swap = (hs.get("cgroup_swap_delta_mb") or 0) > 50 or (hs.get("pswpout_pages") or 0) > 25600 + gs["memory"] = gates.gate("FAIL" if swapped > 0 and ctx.a.fail_on_swap or host_swap else "PASS", + "guest swapouts +%s, pressure max %s; host cgroup swap %s MB, pswpout %s pages" % ( + swapped, step["system"]["guest"]["pressure_max"], hs.get("cgroup_swap_delta_mb"), hs.get("pswpout_pages"))) + step["gates"] = gs + step["gates_ok"] = gates.worst(gs) + step["stable"] = step["stability"]["ok"] + return gs + + +def take_shots(ctx, count=2): + """Screenshots of every simulator, through the guest, judged by the screenshot gates.""" + udids = ",".join(s["udid"] for s in ctx.sims) + r = ctx.guest.call(["shots", "--udids", udids, "--count", count], 60 + 30 * len(ctx.sims)) + by = [] + for s, one in zip(ctx.sims, r["sims"]): + by.append({"idx": s["idx"], "role": s["role"], "shots": one["shots"]}) + return gates.shots_gates(by, ctx.golden), by diff --git a/src/bench/macbench/fakes.py b/src/bench/macbench/fakes.py new file mode 100644 index 0000000..23b71cb --- /dev/null +++ b/src/bench/macbench/fakes.py @@ -0,0 +1,67 @@ +"""Stand-ins for DRY=1 and the tests: a fake Linux host (QEMU CPU, GPU, memory) and fake Coherent gates. The fake +numbers follow the number of booted simulators so every code path that reads them runs; they mean nothing.""" +import os +import time + + +class FakeHost: + dry = True + + def __init__(self, faults=""): + self.n = 0 + self.faults = faults.split(",") + self.t_last = time.monotonic() + self.cpu_ticks = {"vcpu": 0, "sc": 0} + self.gpu = 0 + self.pid = 4242 + self.kernel_lines = 0 + + def qemu_pid(self): + return self.pid + + def snap(self): + now = time.monotonic() + dt, self.t_last = now - self.t_last, now + self.cpu_ticks["vcpu"] += int(dt * 100 * (2.0 + 0.8 * self.n)) + self.cpu_ticks["sc"] += int(dt * 100 * (0.1 + 0.05 * self.n)) + self.gpu += int(dt * 1e9 * min(0.95, 0.15 * self.n)) + v, s = self.cpu_ticks["vcpu"], self.cpu_ticks["sc"] + thr = {"1": ("CPU 0/KVM", v - v // 5, v // 10, v // 2), "2": ("metal2vulkan-sc", s, s // 4, 0)} + return {"mono": now, "wall": time.time(), "pid": self.pid, "threads": thr, "gpu": self.gpu, + "mem": {"mem_avail_kb": int((40 - 0.5 * self.n) * 1048576), "qemu_rss_kb": int((9 + 0.4 * self.n) * 1048576), + "pswpin": 0, "pswpout": 0, "cg_swap": 0, "psi_some_us": 0}} + + def mem_avail_gb(self): + v = 40 - 0.5 * self.n + mf = [f for f in self.faults if f.startswith("hostmem@")] + if mf and self.n >= int(mf[0].split("@")[1]): + return 0.5 + return v + + def disk_free_gb(self, path): + return 500.0 + + def kernel_gpu(self, since): + if any(f.startswith("kernelgpu@") and self.n >= int(f.split("@")[1]) for f in self.faults): + return 2, "fence expired (fake)" + return 0, "" + + def same_qemu(self): + return True + + def env(self): + return {"kernel": "fake", "mesa": "fake", "cpu_model": "fake", "cpus": 20}, {"surface_coherent": "1"}, {"REIMS_VGPU_COHERENT": "1"} + + +class FakeCoherent: + def __init__(self, host, faults=""): + self.host, self.faults = host, faults.split(",") + + def mark(self): + return {"n": 0} + + def check(self, mark): + bad = any(f.startswith("coherent@") and self.host.n >= int(f.split("@")[1]) for f in self.faults) + ok = lambda d: {"status": "PASS", "detail": d} + return {"coherent_census": {"status": "FAIL", "detail": "overwrite_risk 1 (fake)"} if bad else ok("fake"), + "coherent_process": ok("fake"), "qemu_log": ok("fake"), "gpu_hang": ok("fake")} diff --git a/src/bench/macbench/gates.py b/src/bench/macbench/gates.py index 875e810..8fb1054 100644 --- a/src/bench/macbench/gates.py +++ b/src/bench/macbench/gates.py @@ -109,7 +109,7 @@ def strip_gate_of(sims): return gate("PASS", "%d simulator(s): no frame out of order" % len(judged)) -def shots_gates(shots_by_sim, golden_hash=None, golden_key=None): +def shots_gates(shots_by_sim, golden=None): """Tear, strip-readable and pixel-exact gates from the guest's `shots` output. shots_by_sim: [{"idx", "role", "shots": [{ok, hash, tear, tear_applicable, counter}]}]""" res = {} @@ -126,13 +126,15 @@ def shots_gates(shots_by_sim, golden_hash=None, golden_key=None): res["tears"] = gate("SKIP", "no screenshot showed the scrolling cards") else: res["tears"] = gate("FAIL" if tear_bad else "PASS", ("torn: sim %s" % sorted(set(tear_bad))) if tear_bad else "%d screenshot(s) clean" % tear_judged) - hashes = [sh["hash"] for s in shots_by_sim if s["role"] == "static" for sh in s["shots"] if sh.get("ok")] + statics = [sh for s in shots_by_sim if s["role"] == "static" for sh in s["shots"] if sh.get("ok")] + hashes = [sh["hash"] for sh in statics] + golden_hash, golden_key = golden(statics[0]["w"], statics[0]["h"]) if (golden and statics) else (None, None) if not hashes: res["pixels"] = gate("SKIP", "no static reference screen in this step") elif len(set(hashes)) > 1: res["pixels"] = gate("FAIL", "%d different pictures of the same static screen" % len(set(hashes))) elif golden_hash is None: - res["pixels"] = gate("SKIP", "screens agree with each other; no golden stored for %s (bench golden record)" % golden_key) + res["pixels"] = gate("SKIP", "screens agree with each other; no golden stored for %s (bench golden record)" % (golden_key or "this setup")) else: res["pixels"] = gate("PASS" if hashes[0] == golden_hash else "FAIL", "static screen %s the golden %s" % ("matches" if hashes[0] == golden_hash else "DIFFERS from", golden_key)) diff --git a/src/bench/macbench/hostprobe.py b/src/bench/macbench/hostprobe.py index 1fc796a..5535c75 100644 --- a/src/bench/macbench/hostprobe.py +++ b/src/bench/macbench/hostprobe.py @@ -219,3 +219,42 @@ def host_versions(): except OSError: pass return {"kernel": kernel, "mesa": mesa or "unknown (pass --tag mesa=)", "cpu_model": cpu, "cpus": os.cpu_count()} + + +class Host: + """The real host side for one QEMU process. `root` is "" in production (tests point it at a fake tree).""" + dry = False + + def __init__(self, pid, root="", live_conf=None, hz=None): + self.pid, self.root, self.live_conf = pid, root, live_conf + self.hz = hz or os.sysconf("SC_CLK_TCK") + self.n = 0 + + def qemu_pid(self): + return self.pid + + def snap(self): + import time + return {"mono": time.monotonic(), "wall": time.time(), "pid": self.pid, "threads": thread_snapshot(self.pid, self.root), + "gpu": gpu_render_ns(self.pid, self.root), "mem": mem_snapshot(self.pid, self.root)} + + def mem_avail_gb(self): + v = mem_snapshot(self.pid, self.root).get("mem_avail_kb") + return None if v is None else v / 1048576.0 + + def disk_free_gb(self, path): + try: + st = os.statvfs(path) + return st.f_bavail * st.f_frsize / 1073741824.0 + except OSError: + return None + + def kernel_gpu(self, since): + return kernel_gpu_lines(since) + + def same_qemu(self): + return os.path.exists("%s/proc/%d" % (self.root, self.pid)) + + def env(self): + sw = read_live_conf(self.live_conf) if self.live_conf else {} + return host_versions(), sw, qemu_env_switches(self.pid, self.root) diff --git a/src/bench/macbench/others.py b/src/bench/macbench/others.py new file mode 100644 index 0000000..2efba23 --- /dev/null +++ b/src/bench/macbench/others.py @@ -0,0 +1,206 @@ +"""The other scenarios: idle VM, lifecycle (+ leak per cycle, boot storm), soak, build timing, the Device panel plug, +the golden recorder.""" +import json +import os +import subprocess +import time + +from . import analysis, core, gates, scale, stats, strip +from .core import Stop + + +def _fmt(x, unit=""): + return "n/a" if x is None else "%g%s" % (x, unit) + + +def run_idle(ctx): + a = ctx.a + info = core.setup_guest(ctx) + core.capture_env(ctx, info) + ctx.marks = {"coh": ctx.coh.mark(), "t0": time.time()} + m = ctx.guest.call(["memcheck"], 60) + if (m["booted"] or m.get("foreign_booted")) and not a.allow_foreign: + raise Stop("simulators are booted (%d of the bench, %d outside it): an idle VM has none; --allow-foreign to measure anyway" % (m["booted"], m.get("foreign_booted", 0))) + core.settle(ctx) + g, s0, s1, t_go = core.window(ctx, a.secs, "idle") + sysd = core.system_dict(ctx, g, s0, s1, 0) + kern, note = ctx.host.kernel_gpu(s0["wall"]) + gs = dict(ctx.coh.check(ctx.marks["coh"])) + gs["kernel_gpu"] = gates.kernel_gate(kern, note) + ctx.doc["data"] = {"window_s": a.secs, "system": sysd} + ctx.doc["gates"] = gs + ok = sysd["qemu_cpu_cores"] <= a.bar_qemu_cores + ctx.section("idle", "PASS" if ok and gates.worst(gs) else "FAIL", + "IDLE %s: QEMU %.3f cores (bar <= %g; user %.3f guest %.3f sys %.3f), GPU busy %s%%, guest CPU %s%%, guest used %s MB, host cgroup swap %s MB; gates %s" % ( + "PASS" if ok and gates.worst(gs) else "FAIL", sysd["qemu_cpu_cores"], a.bar_qemu_cores, sysd["qemu_user_cores"], sysd["qemu_guest_cores"], + sysd["qemu_sys_cores"], sysd["gpu_busy_pct"], sysd["guest"]["total_cpu_pct"], sysd["guest"]["mem_used_mb"], + sysd["host"].get("cgroup_swap_mb"), "ok" if gates.worst(gs) else [k for k, v in gs.items() if v["status"] == "FAIL"])) + ctx.save() + + +def _median(v): + return stats.r1(stats.percentile(v, 0.5), 2) + + +def run_lifecycle(ctx): + a = ctx.a + info = core.setup_guest(ctx) + core.capture_env(ctx, info) + ctx.marks = {"coh": ctx.coh.mark(), "t0": time.time()} + cyc, xs, rss, used, comp = [], [], [], [], [] + base = ctx.host.snap()["mem"].get("qemu_rss_kb") + mem0 = ctx.guest.call(["memcheck"], 60)["mem"] + data = ctx.doc["data"] = {"cycles": cyc, "storm": None} + for i in range(a.cycles): + r = ctx.guest.call(["cycle", "--type", ctx.type_id, "--runtime", ctx.runtime], 900) + cyc.append(r["timings"]) + m = ctx.guest.call(["memcheck"], 60) + xs.append(i + 1) + used.append(m["mem"]["used_mb"] - mem0["used_mb"]) + comp.append(m["mem"]["compressed_mb"]) + sn = ctx.host.snap()["mem"].get("qemu_rss_kb") + rss.append(None if sn is None or base is None else (sn - base) / 1024.0) + ctx.save() + keys = sorted({k for c in cyc for k in c}) + med = {k: _median([c[k] for c in cyc if c.get(k) is not None]) for k in keys} + data["median_s"] = med + ok = all(v is not None for v in med.values()) and bool(cyc) + ctx.section("lifecycle", "PASS" if ok else "FAIL", "LIFECYCLE %s (%d cycles, medians, s): create %s, cold boot %s, install %s, launch call %s, first app frame %s, shutdown %s, resume boot %s, first frame after resume %s, delete %s" % ( + "PASS" if ok else "FAIL", len(cyc), med.get("create_s"), med.get("cold_boot_s"), med.get("install_s"), med.get("cold_launch_call_s"), + med.get("cold_first_frame_s"), med.get("shutdown_s"), med.get("resume_boot_s"), med.get("resume_first_frame_s"), med.get("delete_s"))) + leaks = {"qemu_rss": gates.leak_gate("QEMU RSS", xs, rss, a.leak_rss_mb, "MB/cycle"), + "guest_used": gates.leak_gate("guest used memory", xs, used, a.leak_guest_mb, "MB/cycle")} + data["leak"] = {"qemu_rss_mb": rss, "guest_used_mb": used} + for k, g in leaks.items(): + ctx.doc["gates"]["leak_" + k] = g + ctx.section("leak:" + k, g["status"], "LEAK %s: %s" % (g["status"], g["detail"])) + if a.storm: + n = a.storm + reason = core.guard(ctx, n, "before the boot storm of %d" % n) + if reason: + ctx.section("storm", "SKIP", "BOOT STORM SKIP: " + reason) + else: + t0 = time.time() + boot = core.grow(ctx, n, ["busy"] * n) + usable = time.time() - t0 + data["storm"] = {"n": n, "boot_all_s": boot.get("all_usable_s"), "each_s": boot.get("each_usable_s"), "boot_and_launch_s": round(usable, 1)} + ok = usable <= a.storm_bar_s + ctx.section("storm", "PASS" if ok else "FAIL", "BOOT STORM %s: %d simulators booted at once: all booted in %s s (slowest %s s), booted and app launched in %.0f s (bar <= %g)" % ( + "PASS" if ok else "FAIL", n, boot.get("all_usable_s"), max(boot.get("each_usable_s") or [0]), usable, a.storm_bar_s)) + ctx.doc["gates"].update(ctx.coh.check(ctx.marks["coh"])) + ctx.save() + + +def run_soak(ctx): + a = ctx.a + info = core.setup_guest(ctx) + core.capture_env(ctx, info) + ctx.marks = {"coh": ctx.coh.mark(), "t0": time.time()} + n = a.soak_n + reason = core.guard(ctx, n, "before the soak") + if reason: + raise Stop(reason) + ctx.host.n = n + core.grow(ctx, n, ["busy"] * n) + core.settle(ctx) + res = {} + for label in ("start", "end"): + if label == "end": + t_end = time.time() + a.soak_minutes * 60 + xs, rss, used = [], [], [] + while time.time() < t_end and not ctx.run.expired.is_set(): + ctx.run.need(60, "a soak sample") + time.sleep(min(a.soak_every, max(0, t_end - time.time()))) + m = ctx.guest.call(["memcheck"], 60) + sn = ctx.host.snap()["mem"].get("qemu_rss_kb") + xs.append((time.time() - ctx.marks["t0"]) / 60.0) + used.append(m["mem"]["used_mb"]) + rss.append(None if sn is None else sn / 1024.0) + ctx.doc["data"]["soak"] = {"minutes": xs, "qemu_rss_mb": rss, "guest_used_mb": used} + g, s0, s1, t_go = core.window(ctx, a.secs, "all-busy") + res[label] = core.build_step(ctx, n, "all-busy", g, s0, s1, t_go) + ctx.doc["data"]["steps"] = [res["start"], res["end"]] + for k, gname, ys, bar in (("QEMU RSS", "leak_qemu_rss", rss, a.soak_rss_mb_min), ("guest used memory", "leak_guest_used", used, a.soak_guest_mb_min)): + gt = gates.leak_gate(k, xs, ys, bar, "MB/min") + ctx.doc["gates"][gname] = gt + ctx.section(gname, gt["status"], "SOAK %s: %s" % (gt["status"], gt["detail"])) + f0 = min([s["fps"] for s in res["start"]["sims"] if s.get("fps")] or [0]) + f1 = min([s["fps"] for s in res["end"]["sims"] if s.get("fps")] or [0]) + ok = f0 > 0 and f1 >= 0.95 * f0 + ctx.section("soak_fps", "PASS" if ok else "FAIL", "SOAK %s: slowest simulator %.1f fps at the start, %.1f after %g min (bar: not below 95%%)" % ("PASS" if ok else "FAIL", f0, f1, a.soak_minutes)) + ctx.doc["gates"].update(ctx.coh.check(ctx.marks["coh"])) + ctx.doc["gates"]["strip"] = gates.strip_gate_of(res["end"]["sims"]) + ctx.save() + + +def run_build(ctx): + a = ctx.a + info = core.setup_guest(ctx) + core.capture_env(ctx, info) + r = ctx.guest.call(["build"] + (["--xcodebuild", a.xcodebuild] if a.xcodebuild else []), 3900) + ctx.doc["data"] = r + ok = r.get("tools_cold_rc") == 0 and r.get("tools_warm_rc") == 0 + ctx.section("build", "PASS" if ok else "FAIL", "BUILD %s: bench tools (swiftc, GlassBench + simfps + GenieBench) cold %s s, warm %s s" % ("PASS" if ok else "FAIL", r.get("tools_cold_s"), r.get("tools_warm_s"))) + if a.xcodebuild: + ok = r.get("xcodebuild_cold_rc") == 0 and r.get("xcodebuild_warm_rc") == 0 + ctx.section("xcodebuild", "PASS" if ok else "FAIL", "XCODEBUILD %s: cold %s s, warm %s s" % ("PASS" if ok else "FAIL", r.get("xcodebuild_cold_s"), r.get("xcodebuild_warm_s"))) + else: + ctx.section("xcodebuild", "SKIP", "XCODEBUILD SKIP: GlassBench is not an Xcode project; pass --xcodebuild '' to time yours (cold: new DerivedData, warm: again)") + ctx.save() + + +def run_panel(ctx): + """The Device panel's stream, measured outside this tool: --panel-cmd runs a command that prints one JSON object, + {"frames": [[t_seconds, counter], ...]}: each frame the viewer received and the counter GlassBench's strip showed in it.""" + a = ctx.a + ctx.doc["env"].update({"build_id": a.build_id or "unknown"}) + cmd = a.panel_cmd or os.environ.get("MACBOX_BENCH_PANEL_CMD") + if not cmd: + ctx.section("panel", "SKIP", "PANEL SKIP: no --panel-cmd (needs Simlock and T3 Code's Device panel; see README, 'The panel section')") + ctx.save() + return + p = ctx.run.popen(["/bin/sh", "-c", cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + env=dict(os.environ, BENCH_PANEL_SECS=str(a.secs))) + try: + out, err = p.communicate(timeout=a.secs + 300) + except subprocess.TimeoutExpired: + ctx.run.kill_children() + ctx.section("panel", "FAIL", "PANEL FAIL: the command did not finish") + ctx.save() + return + try: + frames = json.loads(out.strip().splitlines()[-1])["frames"] + except (ValueError, KeyError, IndexError): + ctx.section("panel", "FAIL", "PANEL FAIL: no JSON with frames on the command's last line (%s)" % err.strip()[-100:]) + ctx.save() + return + an = strip.analyze_counters([(t, c) for t, c in frames]) + m = stats.frame_stats([t for t, _ in an["unique"]], a.nominal_fps) + ok_s, why = strip.strip_gate(an) + from . import targets + tg = targets.parse_target("panel: fps>=%g, p95<=%g" % (a.panel_fps, a.panel_p95)) + ev = targets.evaluate(tg, [m]) + ctx.doc["data"] = {"metrics": m, "strip": {k: an[k] for k in ("repeats", "lost", "backwards", "jumps", "unreadable")}} + ctx.doc["gates"]["strip"] = gates.gate("PASS" if ok_s else "FAIL", why) + ok = bool(ev["pass"]) and ok_s + ctx.section("panel", "PASS" if ok else "FAIL", "PANEL %s: %s distinct/s, interval p50 %s p95 %s p99 %s max %s ms, %d over 2x nominal, longest gap %s s (bars: >= %g/s, p95 <= %g ms); strip %s" % ( + "PASS" if ok else "FAIL", m["fps"], m["p50_ms"], m["p95_ms"], m["p99_ms"], m["max_ms"], m["over2x"], m["longest_gap_s"], a.panel_fps, a.panel_p95, why)) + ctx.save() + + +def run_golden(ctx): + """Records the pixel hash of GlassBench -static for this setup (macOS runtime, device type, scale, size).""" + a = ctx.a + info = core.setup_guest(ctx) + core.capture_env(ctx, info) + core.grow(ctx, 1, ["static"]) + core.settle(ctx) + scale.golden_phase(ctx) + seen = getattr(ctx, "golden_seen", None) + if not seen: + ctx.section("golden", "FAIL", "GOLDEN FAIL: the static screen gave different pictures, or none: nothing recorded") + else: + key = ctx.golden_key_for(seen["w"], seen["h"]) + ctx.record_golden(key, seen["hash"]) + ctx.section("golden", "PASS", "GOLDEN recorded for %s: %s" % (key, seen["hash"][:16])) + ctx.save() diff --git a/src/bench/macbench/runctx.py b/src/bench/macbench/runctx.py index 2aa1f75..23be89c 100644 --- a/src/bench/macbench/runctx.py +++ b/src/bench/macbench/runctx.py @@ -14,15 +14,17 @@ class DeadlineError(Exception): class Run: - def __init__(self, deadline_s, log=print): + def __init__(self, deadline_s, log=print, margin=60.0): self.t0 = time.monotonic() self.deadline = self.t0 + deadline_s + self.margin = margin # the watchdog acts this long before the deadline self.log = log self.children = [] # subprocess.Popen objects this run started self.undo = [] # (label, fn) restoring a knob, run last-in first-out self.teardown = [] # (label, fn): shut down what the run started (simulators) self.on_deadline = None # fn: write the partial result self.cleaned = False + self.in_cleanup = False self._lock = threading.Lock() self.expired = threading.Event() self.problems = [] # things cleanup found it could not restore @@ -33,8 +35,8 @@ def remaining(self): def need(self, secs, what): """Refuse a step that would run past the deadline (the cleanup keeps its own 90 s).""" - if self.remaining() < secs + 90: - raise DeadlineError("no time for %s (%d s needed, %d s left before the deadline)" % (what, secs, max(0, self.remaining() - 90))) + if self.remaining() < secs + self.margin + 30: + raise DeadlineError("no time for %s (%d s needed, %d s left before the deadline)" % (what, secs, max(0, self.remaining() - self.margin - 30))) # --- children def popen(self, argv, **kw): @@ -75,9 +77,9 @@ def on_signal(signum, frame): t.start() def _watchdog(self): - while time.monotonic() < self.deadline - 60: + while time.monotonic() < self.deadline - self.margin: time.sleep(1) - self.log("DEADLINE: 60 s left, stopping") + self.log("DEADLINE: %d s left, stopping" % self.margin) self.expired.set() if self.on_deadline: try: @@ -93,6 +95,7 @@ def cleanup(self): return self.cleaned = True self.expired.set() + self.in_cleanup = True self.kill_children() for label, fn in reversed(self.teardown): try: diff --git a/src/bench/macbench/scale.py b/src/bench/macbench/scale.py new file mode 100644 index 0000000..802f9e4 --- /dev/null +++ b/src/bench/macbench/scale.py @@ -0,0 +1,200 @@ +"""The scaling sweep: N = 1, 2, 3, ... simulators, per-simulator numbers at each N, the maximum sustainable N per +quality target.""" +import time + +from . import analysis, core, gates, stats, targets +from .core import Stop +from .runctx import DeadlineError +from .transport import GuestError + +STEP_GATES = ("strip", "kernel_gpu", "coherent_census", "coherent_process", "qemu_log", "gpu_hang", "tears", "pixels", "memory") + + +def calibrate(ctx, sim): + """Pick the input injection that is answered (or 'none'): 3 tries of each method on one simulator.""" + a = ctx.a + if a.probe_method != "auto": + ctx.probe_method = a.probe_method + return {"chosen": a.probe_method, "calibrated": False} + methods = "openurl,appearance" + r = ctx.guest.call(["calibrate", "--udid", sim["udid"], "--methods", methods, "--time-scale", 0.3 if ctx.host.dry else 1.0] + (["--probe-cmd", a.probe_cmd] if a.probe_cmd else []), 200) + chosen = next((m for m in methods.split(",") if r["methods"].get(m, {}).get("answered", 0) >= 2), "none") + ctx.probe_method = chosen + return {"chosen": chosen, "calibrated": True, "methods": r["methods"]} + + +def golden_phase(ctx, record=False): + """Pixel-exact reference: the first simulator shows GlassBench -static; its screenshots must agree and match + the stored golden. Returns the pixels gate. Costs a few seconds.""" + s0 = ctx.sims[0] + keep = s0["role"] + ctx.guest.call(["prepare", "--udids", s0["udid"], "--roles", "static"] + (["--quiesce"] if ctx.a.quiesce else []), 120) + s0["role"] = "static" + time.sleep(0.3 if ctx.host.dry else 3) + r = ctx.guest.call(["shots", "--udids", s0["udid"], "--count", 3], 120) + by = [{"idx": 0, "role": "static", "shots": r["sims"][0]["shots"]}] + g = gates.shots_gates(by, ctx.golden)["pixels"] + ok = [x for x in by[0]["shots"] if x.get("ok")] + ctx.golden_seen = {"hash": ok[0]["hash"], "w": ok[0]["w"], "h": ok[0]["h"]} if ok and len({x["hash"] for x in ok}) == 1 else None + s0["role"] = None # forces grow() to launch it with its real role + return g + + +def run_scale(ctx): + a = ctx.a + doc, mix = ctx.doc, a.mix + tg_specs = a.targets + floor = targets.parse_target("floor: " + a.floor) + info = core.setup_guest(ctx) + core.capture_env(ctx, info) + data = doc["data"] = {"mix": mix, "scale_x": a.scale_x, "device": doc["env"]["guest"].get("device_type"), "ns": a.ns, + "nominal_fps": a.nominal_fps, "window_s": a.secs, "targets": tg_specs, "floor": a.floor, "steps": [], + "summary": {}, "calibration": None, "observer": None} + ctx.marks = {"coh": ctx.coh.mark(), "t0": time.time()} + ctx.run.on_deadline = lambda: (_finish(ctx, "deadline"), None)[1] + stopped, first = None, True + try: + for n in a.ns: + roles = core.roles_for(mix, n) + reason = core.guard(ctx, n - len(ctx.sims), "before booting simulator %d" % n) + if reason: + stopped = reason + break + ctx.host.n = n + boot = core.grow(ctx, n, roles) + if first: + if mix in ("all-busy", "one-busy"): + data["calibration"] = calibrate(ctx, ctx.sims[0]) + ctx.guest.call(["settle", "--max-s", 20, "--bg-pct", 90, "--step", ctx.a.settle_step], 60) + if a.golden_phase: + ctx.pixel_gate = golden_phase(ctx) + core.grow(ctx, n, roles) + core.settle(ctx) + g, s0, s1, t_go = core.window(ctx, a.secs, mix) + step = core.build_step(ctx, n, mix, g, s0, s1, t_go, boot) + shots, by = core.take_shots(ctx, a.shots) if a.shots and mix != "idle" else ({}, []) + if first and ctx.pixel_gate and "pixels" not in shots: + shots["pixels"] = ctx.pixel_gate + core.step_gates(ctx, step, shots) + step["window"] = {"start_offset_s": round(t_go - ctx.marks["t0"] - ctx.offset, 1)} + data["steps"].append(step) + if first and a.observer_check and mix in ("all-busy", "one-busy", "agent"): + data["observer"] = observer_check(ctx, step, mix) + _est(ctx, data["steps"]) + _finish(ctx, None, partial=True) + first = False + if not step["gates_ok"]: + bad = [k for k, v in step["gates"].items() if v["status"] == "FAIL"] + stopped = "gate failed at N=%d: %s" % (n, ", ".join(bad)) + break + if not step["stable"]: + stopped = "instability at N=%d: %s" % (n, _why_unstable(step)) + break + fl = targets.evaluate(floor, step["sims"]) + if fl["pass"] is False: + stopped = "floor %s breached at N=%d (sim %s: %s = %s)" % (a.floor, n, fl["failing"][0]["sim"], fl["failing"][0]["cond"], fl["failing"][0]["value"]) + break + except Stop as e: + stopped = str(e) + except (DeadlineError, GuestError) as e: + stopped = "%s: %s" % (type(e).__name__, e) + return _finish(ctx, stopped) + + +def _why_unstable(step): + s = step["stability"] + out = [] + if s["new_ips"]: + out.append("%d new crash report(s)" % s["new_ips"]) + if s["sims_died"]: + out.append("simulator(s) %s no longer booted" % s["sims_died"]) + if s["kernel_gpu_lines"]: + out.append("%d kernel GPU error line(s)" % s["kernel_gpu_lines"]) + if not s["qemu_same"]: + out.append("QEMU restarted") + return "; ".join(out) + + +def _est(ctx, steps): + """A simulator's guest memory from the steps so far (for the guard).""" + if len(steps) >= 2: + d = [(b["system"]["guest"]["mem_used_mb"] - a["system"]["guest"]["mem_used_mb"]) / max(1, b["n"] - a["n"]) for a, b in zip(steps, steps[1:])] + ctx.est_sim_mb = max(300.0, max(d)) + elif steps: + ctx.est_sim_mb = max(ctx.est_sim_mb, 1100.0) + + +def observer_check(ctx, step, mix): + """Decision: if reading the strip moves the surface frame rate by more than the noise (4%), every distinct-frame + number in this result carries that observer cost. A second window of the same N reads no strip.""" + g, s0, s1, t_go = core.window(ctx, ctx.a.secs, mix, no_strip=True) + with_strip = [s["surface_fps"] for s in step["sims"] if s.get("role") in ("busy", "agent") and s.get("surface_fps")] + without = [s["simfps"]["fps"] for s in g["sims"] if s["role"] in ("busy", "agent") and s.get("simfps")] + ws, wo = stats.mean(with_strip), stats.mean(without) + d = stats.rel_diff(wo, ws) + return {"n": step["n"], "surface_fps_with_strip": stats.r1(ws), "surface_fps_without_strip": stats.r1(wo), + "delta": None if d is None else round(d, 4), "ok": d is not None and abs(d) <= 0.04} + + +def _finish(ctx, stopped, partial=False): + """Summary, sections and the result file. Also the deadline's partial write (partial=True keeps the doc open).""" + a, doc = ctx.a, ctx.doc + data = doc["data"] + steps = data["steps"] + if not steps and not stopped: + return None + data["summary"] = analysis.summarize_scale(steps, a.targets, stopped) if steps else {"max_n": {}, "stopped": stopped, "standing_cost": {}} + doc["data"]["summary"]["stopped"] = stopped + doc["gates"] = _gate_rollup(steps, ctx) + if not partial: + doc["sections"] = [s for s in doc["sections"] if s["name"] not in ("scale", "gates", "stability", "observer", "input", "targets", "cost")] + _sections(ctx, data, stopped) + ctx.save() + return data["summary"] + + +def _gate_rollup(steps, ctx): + roll = {} + for name in STEP_GATES: + rows = [(st["n"], st["gates"].get(name)) for st in steps if st.get("gates", {}).get(name)] + if not rows: + roll[name] = gates.gate("SKIP", "not run") + continue + bad = [(n, g) for n, g in rows if g["status"] == "FAIL"] + if bad: + roll[name] = gates.gate("FAIL", "N=%d: %s" % (bad[0][0], bad[0][1]["detail"])) + elif all(g["status"] == "SKIP" for _, g in rows): + roll[name] = gates.gate("SKIP", rows[-1][1]["detail"]) + else: + roll[name] = gates.gate("PASS", "%d step(s); %s" % (len(rows), rows[-1][1]["detail"])) + return roll + + +def _sections(ctx, data, stopped): + a = ctx.a + steps = data["steps"] + sm = data["summary"] + gate_fail = [k for k, v in ctx.doc["gates"].items() if v["status"] == "FAIL"] + unstable = any(not s["stable"] for s in steps) + mx = ", ".join("%s=%s" % (k, v["max_n"] if v["max_n"] is not None else "none") for k, v in sm["max_n"].items()) + status = "FAIL" if gate_fail or unstable else "PASS" + ctx.section("scale", status, "SCALE %s %s %dx %s: max sustainable N: %s | steps %s%s | min Jain %s" % ( + status, a.mix, a.scale_x, "(DRY)" if ctx.host.dry else "", mx, [s["n"] for s in steps], + (" | stopped: %s" % stopped) if stopped else " | all requested N done", sm.get("fairness_min_jain"))) + for name, v in ctx.doc["gates"].items(): + ctx.section("gate:" + name, v["status"] if v["status"] != "PASS" else "PASS", "GATE %s %s: %s" % (v["status"], name, v["detail"])) + st = [s for s in steps] + ctx.section("stability", "FAIL" if unstable else "PASS", "STABILITY %s: %s" % ( + "FAIL" if unstable else "PASS", "; ".join("N=%d %s" % (s["n"], _why_unstable(s)) for s in st if not s["stable"]) or "no crash report, no dead simulator, no kernel GPU line, one QEMU")) + cal = data.get("calibration") + if cal: + ctx.section("input", "INFO", "INPUT PROXY method %s %s" % (cal["chosen"], "(calibrated: %s)" % {k: "%d/%d" % (v["answered"], v["n"]) for k, v in cal.get("methods", {}).items()} if cal.get("calibrated") else "(given)")) + ob = data.get("observer") + if ob: + ctx.section("observer", "PASS" if ob["ok"] else "FAIL", "OBSERVER %s: surface fps %s with the strip read, %s without (%s): %s" % ( + "PASS" if ob["ok"] else "FAIL", ob["surface_fps_with_strip"], ob["surface_fps_without_strip"], + "n/a" if ob["delta"] is None else "%+.1f%%" % (ob["delta"] * 100), "within the noise" if ob["ok"] else "reading the strip changes the system: distinct-fps numbers are lower bounds")) + sc = sm.get("standing_cost") or {} + if sc: + parts = ["%s %+.2f/sim" % (k, v["per_sim"]) for k, v in sc.items() if k in ("qemu_cpu_cores", "gpu_busy_pct", "guest_mem_used_mb", "guest_sims_cpu_pct", "guest_sims_rss_mb")] + ctx.section("cost", "INFO", "STANDING COST per simulator (slope over N): " + ", ".join(parts)) diff --git a/src/bench/macbench/transport.py b/src/bench/macbench/transport.py index ff6b575..3cc49e1 100644 --- a/src/bench/macbench/transport.py +++ b/src/bench/macbench/transport.py @@ -52,6 +52,8 @@ def parse(self, what, rc, out, err): return d def _check(self, what, timeout): + if self.run.in_cleanup: + return min(timeout, 240) if self.run.expired.is_set(): raise DeadlineError("stopping: %s not started" % what) if self.run.remaining() < 5: @@ -90,7 +92,7 @@ def argv_plain(self, cmd): def sh(self, cmd, timeout=120): """A plain command in the guest: (rc, stdout).""" - if self.run.expired.is_set(): + if self.run.expired.is_set() and not self.run.in_cleanup: raise DeadlineError("stopping") p = subprocess.run(self.argv_plain(cmd), capture_output=True, text=True, timeout=min(timeout, max(5, int(self.run.remaining())))) return p.returncode, p.stdout diff --git a/src/bench/run b/src/bench/run new file mode 100755 index 0000000..4090e53 --- /dev/null +++ b/src/bench/run @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +"""macbox bench: the scaling, regression and A/B suite. `macbox bench `, or this file directly. +See README.md here, and `run --help`.""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from macbench.cli import main # noqa: E402 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/bench/tests/test_modules.py b/src/bench/tests/test_modules.py new file mode 100644 index 0000000..29a5cf6 --- /dev/null +++ b/src/bench/tests/test_modules.py @@ -0,0 +1,289 @@ +import importlib.util +import json +import os +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..")) +from macbench import analysis, chart, compare, gates, hostprobe, schema # noqa: E402 +from tests import helpers # noqa: E402 + +spec = importlib.util.spec_from_file_location("benchguest", os.path.join(HERE, "..", "guest", "benchguest.py")) +bg = importlib.util.module_from_spec(spec) +spec.loader.exec_module(bg) + +VM_STAT = """Mach Virtual Memory Statistics: (page size of 16384 bytes) +Pages free: 573102. +Pages active: 1833909. +Pages inactive: 100000. +Pages speculative: 10000. +Pages wired down: 946936. +Pages occupied by compressor: 52949. +Pageins: 123456. +Pageouts: 12. +Swapins: 0. +Swapouts: 7. +""" +PS = """ 1 0 4000 0:01.50 /sbin/launchd + 200 1 20000 2:03.25 launchd_sim /x/set/AAAA-1/data + 201 200 150000 1-00:00:10.00 SpringBoard + 300 1 90000 0:10.00 SimMetalHost -d BBBB-2 + 301 300 1000 0:00.10 helper + 400 1 1000 0:00.50 unrelated AAAA-1x +""" + + +def wr(path, mode, text): + with open(path, mode) as f: + f.write(text) + + +class GuestParsers(unittest.TestCase): + def test_vm_stat(self): + m = bg.mem_summary(bg.parse_vm_stat(VM_STAT)) + self.assertEqual(m["swapouts"], 7) + self.assertAlmostEqual(m["free_mb"], (573102 + 10000) * 16384 / 1048576.0, places=1) + self.assertAlmostEqual(m["compressed_mb"], 52949 * 16384 / 1048576.0, places=1) + self.assertEqual(bg.parse_vm_stat(VM_STAT)["page"], 16384) + + def test_cputime(self): + self.assertAlmostEqual(bg.parse_cputime("0:01.50"), 1.5) + self.assertAlmostEqual(bg.parse_cputime("2:03.25"), 123.25) + self.assertAlmostEqual(bg.parse_cputime("1-00:00:10.00"), 86410.0) + + def test_process_trees_and_attribution(self): + procs = bg.parse_ps(PS) + trees = bg.sim_trees(procs, ["AAAA-1", "BBBB-2"]) + self.assertEqual(trees["AAAA-1"], {200, 201, 400}) # a command line that names it, and children + self.assertEqual(trees["BBBB-2"], {300, 301}) + later = [dict(p, cpu_s=p["cpu_s"] + 2.0) for p in procs] + per, tot = bg.attribute(procs, later, ["AAAA-1", "BBBB-2"], 4.0) + self.assertEqual(per["BBBB-2"]["procs"], 2) + self.assertAlmostEqual(per["BBBB-2"]["cpu_pct"], 100.0, places=1) # 2 processes x 2 s over 4 s + self.assertAlmostEqual(tot["total_cpu_pct"], 6 * 2.0 / 4 * 100, places=1) + + def test_pace(self): + recs = bg.parse_pace('{"at": 100.5, "counter": 300, "ticks": 300, "fps": 59.9}\nnot json\n{"at": 105.5, "counter": 600, "ticks": 300, "fps": 60.0}\n') + self.assertEqual(recs, [(100.5, 300, 59.9, 300), (105.5, 600, 60.0, 300)]) + ps = analysis.pace_summary(recs + [(110.5, 900, 60.1, 300)], 102.0, 108.0, 101.0, 109.0) + self.assertEqual(ps["app_ticks"], 600) # records at or before the first frame (300) to the first after the last (900) + self.assertAlmostEqual(ps["app_tick_fps"], 60.0, delta=0.2) + + +class HostParsers(unittest.TestCase): + def test_task_stat_with_awkward_names(self): + fields = ["S", "1", "1", "0", "0", "0", "0", "0", "0", "0", "0", "300", "50", "0", "0", "20", "0", "1", "0", "100", "0", "0"] + ["0"] * 19 + ["120", "0"] + text = "123 (CPU 3/KVM (x)) " + " ".join(fields) + comm, ut, st, gt = hostprobe.parse_task_stat(text) + self.assertEqual((comm, ut, st), ("CPU 3/KVM (x)", 300, 50)) + self.assertEqual(gt, int(fields[40])) + + def test_thread_delta_splits_user_guest_sys(self): + a = {"1": ("CPU 0/KVM", 1000, 100, 800), "2": ("CPU 1/KVM", 1000, 100, 800), "3": ("metal2vulkan-sc", 100, 10, 0)} + b = {"1": ("CPU 0/KVM", 1400, 120, 1100), "2": ("CPU 1/KVM", 1400, 120, 1100), "3": ("metal2vulkan-sc", 150, 30, 0)} + d = hostprobe.thread_delta(a, b, 2.0, hz=100) + # user (host side) = (400-300) + (400-300) + 50 = 250 ticks; guest = 600; sys = 20+20+20 = 60: over 2 s at 100 Hz + self.assertAlmostEqual(d["user"], 1.25) + self.assertAlmostEqual(d["guest"], 3.0) + self.assertAlmostEqual(d["sys"], 0.3) + self.assertAlmostEqual(d["cores"], 4.55) + self.assertEqual(d["top"][0]["name"], "vCPU") + self.assertEqual(d["top"][0]["threads"], 2) + + def test_fdinfo_and_fake_proc_tree(self): + with tempfile.TemporaryDirectory() as root: + def put(rel, text): + p = root + rel + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "w") as f: + f.write(text) + put("/proc/7/fdinfo/10", "pos: 0\ndrm-client-id:\t5\ndrm-engine-render:\t1000 ns\n") + put("/proc/7/fdinfo/11", "pos: 0\ndrm-client-id:\t5\ndrm-engine-render:\t1000 ns\n") # a dup of the same client + put("/proc/7/fdinfo/12", "pos: 0\ndrm-client-id:\t6\ndrm-engine-render:\t500 ns\n") + put("/proc/7/fdinfo/13", "pos: 0\nflags: 02\n") + self.assertEqual(hostprobe.gpu_render_ns(7, root), 1500) + self.assertIsNone(hostprobe.gpu_render_ns(8, root)) + put("/proc/7/cgroup", "0::/machine/vm.scope\n") + put("/sys/fs/cgroup/machine/vm.scope/memory.current", "4294967296\n") + put("/sys/fs/cgroup/machine/vm.scope/memory.swap.current", "1048576\n") + put("/sys/fs/cgroup/machine/vm.scope/memory.events", "low 0\nhigh 0\nmax 2\noom 0\noom_kill 0\n") + put("/proc/meminfo", "MemTotal: 1000000 kB\nMemAvailable: 500000 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n") + put("/proc/vmstat", "pswpin 10\npswpout 20\n") + put("/proc/7/status", "VmRSS:\t 2048 kB\n") + a = hostprobe.mem_snapshot(7, root) + self.assertEqual(a["cg_current"], 4294967296) + self.assertEqual(a["cg_swap"], 1048576) + self.assertEqual(a["qemu_rss_kb"], 2048) + put("/proc/vmstat", "pswpin 15\npswpout 120\n") + put("/sys/fs/cgroup/machine/vm.scope/memory.swap.current", "3145728\n") + b = hostprobe.mem_snapshot(7, root) + d = hostprobe.mem_delta(a, b, 10) + self.assertEqual((d["pswpin_pages"], d["pswpout_pages"]), (5, 100)) + self.assertAlmostEqual(d["cgroup_swap_delta_mb"], 2.0) + put("/proc/7/environ", "A=1\0REIMS_VGPU_CENSUS=/some/dir\0REIMS_VGPU_COHERENT=1\0") + self.assertEqual(hostprobe.qemu_env_switches(7, root), {"REIMS_VGPU_CENSUS": "", "REIMS_VGPU_COHERENT": "1"}) + + def test_gpu_error_regex(self): + self.assertTrue(hostprobe.GPU_ERRORS.search("i915 0000:00:02.0: [drm] GPU HANG: ecode 12")) + self.assertTrue(hostprobe.GPU_ERRORS.search("Fence expired on rcs0")) + self.assertFalse(hostprobe.GPU_ERRORS.search("i915: frequency changed to 1200")) + + +class Gates(unittest.TestCase): + def setUp(self): + self.d = tempfile.mkdtemp() + self.census = os.path.join(self.d, "coherent.jsonl") + self.log = os.path.join(self.d, "qemu.log") + self.hang = os.path.join(self.d, "hang") + wr(self.log, "w", "old line\ncommand buffer failed (before the mark)\n") + with open(self.census, "w") as f: + f.write(json.dumps({"id": 1, "overwrite_risk": 3, "sync_errors": 0}) + "\n") # history: before the mark + f.write(json.dumps({"process": 1, "late_copies_in_busy": 0}) + "\n") + + def test_deltas_only(self): + c = gates.Coherent(self.d, self.log, self.hang) + m = c.mark() + r = c.check(m) + self.assertEqual(r["coherent_census"]["status"], "SKIP") # nothing new: not a pass + self.assertEqual(r["qemu_log"]["status"], "PASS") # the old failure is before the mark + with open(self.census, "a") as f: + f.write(json.dumps({"id": 1, "overwrite_risk": 0, "sync_errors": 0}) + "\n") + self.assertEqual(c.check(m)["coherent_census"]["status"], "PASS") + with open(self.census, "a") as f: + f.write(json.dumps({"id": 1, "overwrite_risk": 1, "sync_errors": 0}) + "\n") + f.write(json.dumps({"process": 1, "late_copies_in_busy": 2}) + "\n") + r = c.check(m) + self.assertEqual(r["coherent_census"]["status"], "FAIL") + self.assertEqual(r["coherent_process"]["status"], "FAIL") + wr(self.log, "a", "m2v: command buffer failed\n") + wr(self.hang, "w", "x") + r = c.check(m) + self.assertEqual(r["qemu_log"]["status"], "FAIL") + self.assertEqual(r["gpu_hang"]["status"], "FAIL") + + def test_unreadable_is_skip_never_pass(self): + r = gates.Coherent(None, None, None).check({"n": 0, "proc": {k: 0 for k in gates.PROCESS_GATES}, "log_off": None, "hang": False}) + self.assertTrue(all(g["status"] == "SKIP" for g in r.values()), r) + self.assertEqual(gates.kernel_gate(None, "no journal")["status"], "SKIP") + self.assertEqual(gates.kernel_gate(0, "")["status"], "PASS") + self.assertEqual(gates.kernel_gate(1, "x")["status"], "FAIL") + + def test_screen_gates(self): + ok = {"ok": True, "hash": "aa", "tear": False, "tear_applicable": True, "w": 402, "h": 874} + by = [{"idx": 0, "role": "busy", "shots": [ok, dict(ok, tear=True)]}, {"idx": 1, "role": "static", "shots": [ok, ok]}] + r = gates.shots_gates(by, lambda w, h: ("aa", "k")) + self.assertEqual(r["tears"]["status"], "FAIL") + self.assertEqual(r["pixels"]["status"], "PASS") + self.assertEqual(gates.shots_gates(by, lambda w, h: ("bb", "k"))["pixels"]["status"], "FAIL") + self.assertEqual(gates.shots_gates(by, lambda w, h: (None, "k"))["pixels"]["status"], "SKIP") + by[1]["shots"][1] = dict(ok, hash="cc") + self.assertEqual(gates.shots_gates(by, None)["pixels"]["status"], "FAIL") # the same screen, two pictures + by2 = [{"idx": 0, "role": "busy", "shots": [dict(ok, tear_applicable=False)]}] + self.assertEqual(gates.shots_gates(by2, None)["tears"]["status"], "SKIP") + + def test_leak_gate(self): + self.assertEqual(gates.leak_gate("x", [1, 2, 3, 4], [0, 50, 100, 150], 20, "MB")["status"], "FAIL") + self.assertEqual(gates.leak_gate("x", [1, 2, 3, 4], [0, 1, 2, 3], 20, "MB")["status"], "PASS") + self.assertEqual(gates.leak_gate("x", [1], [0], 20, "MB")["status"], "SKIP") + + +class Schema(unittest.TestCase): + def test_fake_result_validates_and_is_private(self): + doc = helpers.fake_scale_result() + self.assertEqual(schema.validate(doc), []) + + def test_privacy_catches_paths_ips_hosts(self): + doc = helpers.fake_scale_result() + doc["env"]["x"] = "/srv/data/something/live.conf" + self.assertTrue(any("path" in p for p in schema.validate(doc))) + doc["env"]["x"] = "connected to 192.168.1.20" + self.assertTrue(any("ip" in p for p in schema.validate(doc))) + import socket + doc["env"]["x"] = "ran on %s" % socket.gethostname() + self.assertTrue(any("host" in p for p in schema.validate(doc))) + + def test_scrub_then_publishable(self): + import socket + doc = helpers.fake_scale_result() + doc["env"]["x"] = "ssh %s from %s at 10.0.0.1" % (socket.gethostname(), os.path.expanduser("~") + "/work") + clean = schema.scrubber(["my-vm-alias"]) + doc["env"]["y"] = "my-vm-alias" + d = tempfile.mkdtemp() + p = os.path.join(d, "r.json") + problems = schema.write(doc, p, clean) + self.assertEqual(problems, []) + back = schema.load(p) + self.assertTrue(back["publishable"]) + self.assertNotIn(socket.gethostname(), json.dumps(back)) + self.assertEqual(back["env"]["y"], "") + + def test_write_keeps_data_even_when_not_publishable(self): + doc = helpers.fake_scale_result() + doc["env"]["x"] = "/srv/private/path" + p = os.path.join(tempfile.mkdtemp(), "r.json") + problems = schema.write(doc, p) + self.assertTrue(problems) + back = schema.load(p) + self.assertFalse(back["publishable"]) + self.assertEqual(len(back["data"]["steps"]), 3) + + def test_validate_rejects_bad_shapes(self): + self.assertTrue(schema.validate({"schema": 99})) + doc = helpers.fake_scale_result() + del doc["data"]["steps"][0]["system"] + self.assertTrue(any("system" in p for p in schema.validate(doc))) + + +class CompareAndChart(unittest.TestCase): + def test_compare_flags_regressions_beyond_noise(self): + a = helpers.fake_scale_result(fps=(60, 58, 56)) + b = helpers.fake_scale_result(fps=(60, 58, 40)) + text, worse = compare.compare(a, b) + self.assertGreater(worse, 0) + self.assertIn("WORSE", text) + same, w0 = compare.compare(a, helpers.fake_scale_result(fps=(60, 58, 56))) + self.assertEqual(w0, 0, same) + c = helpers.fake_scale_result(fps=(61, 58.5, 56.5)) # +1.7%, +0.9%: inside the 4% noise + self.assertEqual(compare.compare(a, c)[1], 0) + + def test_compare_max_n(self): + a = helpers.fake_scale_result(fps=(60, 58, 56)) + b = helpers.fake_scale_result(fps=(60, 58, 40)) + text, _ = compare.compare(a, b) + self.assertIn("max sustainable N smooth60", text) + + def test_chart_is_self_contained(self): + html = chart.scale_chart(helpers.fake_scale_result(ns=(1, 2, 3, 4), fps=(60, 59, 55, 30))) + self.assertIn("")[0] + "") + + +class AgentAndMotion(unittest.TestCase): + def test_motion_metrics_ignore_idle_gaps(self): + t = [i / 60 for i in range(60)] + [5 + i / 60 for i in range(60)] + m = analysis.motion_metrics(t) + self.assertEqual(m["stretches"], 2) + self.assertAlmostEqual(m["fps"], 60.0, delta=0.5) + self.assertEqual(m["over2x"], 0) + + def test_agent_metrics(self): + raw = [(10.0 + i / 60, i, 255) for i in range(300)] + acts = [{"op": "launch", "t0": 10.5, "t1": 10.9, "rc": 0}, {"op": "screenshot", "t0": 12.0, "t1": 12.4, "rc": 0}, + {"op": "terminate", "t0": 13, "t1": 13.1, "rc": 0}] + m = analysis.agent_metrics(acts, raw) + self.assertAlmostEqual(m["launch_to_frame_ms_p50"], 17, delta=17) + self.assertAlmostEqual(m["screenshot_ms_p50"], 400, delta=1) + + +if __name__ == "__main__": + unittest.main() From 67dc4fc5fd276345faabf71de544f514ba0197c2 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:34:22 +0200 Subject: [PATCH 06/23] bench: README (every command and metric defined), baselines format, macbox bench entry point, dry-run fault tests --- .gitignore | 1 + macbox | 12 ++ src/bench/README.md | 225 ++++++++++++++++++++++++- src/bench/baselines/README.md | 24 +++ src/bench/guest/benchguest.py | 17 +- src/bench/guest/benchguest_stub.py | 7 +- src/bench/macbench/cli.py | 4 +- src/bench/macbench/core.py | 2 +- src/bench/macbench/fakes.py | 4 +- src/bench/macbench/hostprobe.py | 29 +++- src/bench/macbench/scale.py | 16 +- src/bench/macbench/targets.py | 11 +- src/bench/tests/test_dry.py | 253 +++++++++++++++++++++++++++++ src/bench/tests/test_modules.py | 9 +- src/bench/tests/test_pure.py | 3 + src/bench/tests/test_runctx.py | 89 ++++++++++ 16 files changed, 669 insertions(+), 37 deletions(-) create mode 100644 src/bench/baselines/README.md create mode 100644 src/bench/tests/test_dry.py create mode 100644 src/bench/tests/test_runctx.py diff --git a/.gitignore b/.gitignore index b4fbec5..d59a8c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.bak* __pycache__/ /shots/ +/bench-results/ diff --git a/macbox b/macbox index 5cb6dd3..861f8e9 100755 --- a/macbox +++ b/macbox @@ -662,6 +662,15 @@ cmd_coherent() { # Coherent's crash guard (docs/performance.md) esac } +cmd_bench() { # the scaling, regression and A/B suite (src/bench/README.md) + case "${1:-}" in + compare|chart|report|tools|check|"") ;; + *) [ "${DRY:-0}" = 1 ] || { vm_running || die "the VM is not running: macbox up first (DRY=1 runs the suite against stubs)"; ssh_ok || die "ssh $MACBOX_SSH_HOST doesn't answer"; } ;; + esac + BENCH_QEMU_PID="$(qemu_pid)" BENCH_BUILD_ID="$(build_id)" BENCH_RUN="$RUN" BENCH_QEMU_LOG="$VMDIR/logs/qemu.log" \ + BENCH_SSH="$MACBOX_SSH_HOST" BENCH_HOME="$MACBOX_HOME" exec "$REPO/src/bench/run" "$@" +} + cmd_backup() { local name=${1:?usage: macbox backup } b="$MACBOX_HOME/backups/$1" disk vm_running && die "the VM is running: macbox down first" @@ -707,6 +716,8 @@ Day to day: macbox coherent [status|reset] Coherent's crash guard (MACBOX_COHERENT=auto|on|off) macbox video-kexts reims-util's video kexts into OpenCore (for MACBOX_VIDEO=on) macbox backup + macbox bench scale|idle|lifecycle|soak|build|panel|ab|golden|compare|chart|report|tools|check + the scaling, regression and A/B suite (src/bench/README.md) EOF } @@ -761,6 +772,7 @@ case "$sub" in coherent) cmd_coherent "$@" ;; video-kexts) cmd_video_kexts "$@" ;; backup) cmd_backup "$@" ;; + bench) cmd_bench "$@" ;; help|-h|--help) cmd_help ;; *) cmd_help >&2; exit 2 ;; esac diff --git a/src/bench/README.md b/src/bench/README.md index 13cd5ac..68ec100 100644 --- a/src/bench/README.md +++ b/src/bench/README.md @@ -1,7 +1,224 @@ # Benchmarks -The tools behind the numbers in [docs/performance.md](../../docs/performance.md), for measuring your -own GPU, or a change, before and after. They run inside macOS; build them there once: +How many simulators does this host run at once, each one smooth and responsive? `macbox bench` answers that +and guards every change against it: one suite, one result file per run, one comparison between runs. The older +single-purpose tools it builds on (`simfps`, GlassBench, GenieBench) are described [at the end](#the-tools). + +```sh +macbox bench tools # once: copy the bench to the Mac, build simfps and GlassBench there +macbox bench check # what this host and the VM offer +macbox bench scale --n 1,2,3,4,5,6 # the scaling sweep (the centrepiece) +macbox bench compare baseline.json new.json +macbox bench chart new.json # an HTML/SVG chart next to it +DRY=1 macbox bench scale --n 1,2,3 # the whole control flow against stubs: no VM, no simulator, no GPU +``` + +Requirements: Python 3.8+ on the host (standard library only; no Pillow, no numpy), the VM running, `ssh` to it. +Unit tests, no VM needed: `cd src/bench && python3 -m unittest discover -s tests -t .` (the pure parts, 55 cases) +and `python3 -m unittest tests.test_dry` (the dry runs with injected faults, a couple of minutes). + +## What a run does, and what it never touches + +The bench has a **device set of its own** (`simctl --set`, a directory with a marker file), so it never sees the +pool's simulators or anyone's lease; every `simctl` the guest helper starts goes through one function that adds +`--set` and refuses a set without the marker. It creates its own devices, and at the end shuts down and deletes +all of them. The 3x device is the stock iPhone Pro; the **1x type** is the macbox 1x device type `macbox pool` +makes (same identifier, reused when it exists; made by the bench when it does not, and then CoreSimulatorService +must restart to load it, which closes every booted simulator: the bench refuses that unless nothing is booted +and `--allow-restart` is given). It starts no tracer. It reads `/proc`, `/sys`, the kernel journal and the VM's +census and log; it writes only its result file, and, with `ab --knob live:KEY=VALUE`, the one line in the VM's live +switch file (restored byte for byte on every way out). + +Safety, in every run: the **hard deadline** (`--deadline`, default 90 minutes) stops the run 60 s early, writes the +result so far and runs the cleanup; **one exit trap** (normal end, error, SIGTERM/SIGINT, the deadline) kills the +helpers it started (explicit pids, never a bare `wait`), shuts down and deletes the bench's simulators and +restores every knob; a **guard** refuses to boot more simulators when the guest's free memory would fall under +`--min-guest-free-gib` (default 1.5, after the estimated cost of the new simulators), the guest reports memory +pressure level 2 or more, the host's available memory is under `--min-host-free-gib` (6), or the disk under +`--min-disk-gib` (12): the sweep stops there, it does not swap the VM to death. Simulators booted outside the bench +refuse the run (`--allow-foreign` to measure anyway, recorded). + +## Commands + +| | | +|---|---| +| `scale` | N = 1, 2, 3... simulators (`--n 1,2,3,4,6,8`, `--scale-x 1\|3`, `--mix`), per simulator numbers, the maximum sustainable N per quality target. Stops early when a floor is breached (`--floor`, default `fps>=20`), a gate fails or the system turns unstable. | +| `idle` | The VM with no simulator: QEMU, GPU and memory standing cost against `--bar-qemu-cores` (0.15). | +| `lifecycle` | `--cycles` times: create, boot, install, launch, first app frame, shut down, resume, first frame, delete (medians); a per-cycle leak gate; `--storm N`: N simulators booted at once. | +| `soak` | `--soak-n` busy simulators for `--soak-minutes`: slope of QEMU's RSS and of the guest's memory, and the frame rate at the start against the end. | +| `build` | Build timing: the bench tools (swiftc) cold and warm; `--xcodebuild ''` times your own project cold and warm (GlassBench is not an Xcode project, so the section is SKIP without it). | +| `panel` | The Device panel's stream: pluggable (see [The panel section](#the-panel-section)); SKIP without `--panel-cmd`. | +| `ab` | A B B A inside one window for one knob: ADOPT or CLOSE against a threshold fixed before the run. | +| `golden` | Records the pixel hash of the static reference screen for this setup in `baselines/golden.json`. | +| `compare A.json B.json` | What moved, per N, beyond the noise (`--noise 0.04`); exit status 1 on a regression. | +| `chart R.json [-o out.html]` | A self-contained HTML page of SVG charts: per-simulator fps and p95 against N, input proxy, QEMU CPU and GPU, guest CPU and memory, GPU ms per frame, fairness. No script, no external file. | +| `report R.json` | The PASS/FAIL lines of a result. | +| `tools [--no-build]` / `check` | Copy the bench to the guest and build it / what is there and what is missing. | + +Common options: `--out DIR` (results, default `bench-results/`), `--secs` (window length, default 40), +`--nominal-fps 60`, `--tag key=value` (recorded, for example `mesa=26.2.4` when Mesa is not a package), +`--probe-method auto|openurl|appearance|cmd|none`, `--quiesce` (disables `mediaanalysisd` in each simulator, as +some old harnesses did; recorded, off by default), `--no-observer-check`, `--no-golden`, `--shots N`. +Every run prints one PASS/FAIL/SKIP/INFO line per section and exits 0, 1 (a section failed) or 2 (the run could not +go on). The JSON has the same lines under `sections`. + +### Workload mixes (`scale --mix`) + +Agents' simulators are not all animating. + +| mix | what each simulator runs | what it measures | +|---|---|---| +| `all-busy` (default) | GlassBench scrolling, input marker on | every simulator at full rate; frames and input of each | +| `one-busy` | simulator 0 animates; the others show GlassBench `-static` and get an input probe every `--probe-every` s | whether a busy simulator makes the others sluggish (their input proxy), and what an idle-but-ready simulator costs | +| `agent` | per simulator, staggered: launch the app, let it scroll, screenshot, terminate, idle (`--agent-period`, 12 s) | launch to first frame, screenshot time, and the frame rate inside the motion stretches | +| `idle` | booted, nothing launched, no observer | the pure standing cost of a booted simulator: the slope of each resource over N | + +## What is measured: exact definitions + +Everything is per simulator first; system numbers come with it, never instead of it. A result never holds only an +average. + +**Frames** (per animating simulator; GlassBench draws a 16-bit counter strip, see below). `simfps --strip` polls the +simulator's framebuffer surface at about 2 kHz and reads the counter from every new surface. +- *distinct frames*: surfaces whose counter differs from the previous distinct one. `fps` = (n - 1) / (last - first). +- `p50_ms`, `p95_ms`, `p99_ms`, `max_ms`: percentiles of the intervals between consecutive distinct frames + (`sorted[min(n-1, int((n-1)*p))]`, the rule of simfps and GlassBench itself). `over2x`: intervals longer than + 2 / `--nominal-fps`. `longest_gap_s`: the longest interval, seconds (the "stall"). +- `strip`: `repeats` (same counter on another surface: normal, the compositor makes more surfaces than the app + ticks), `lost` (ticks that never reached the screen), `backwards` (an older counter after a newer one), + `jumps` (forward by more than 1000), `unreadable` (references not found), `span_ticks` (ticks the counter moved). +- `app_tick_fps`: GlassBench's own tick rate from its `pace.json` (the app, before the compositor), `surface_fps`: + surface changes per second as simfps counts them. Three layers, so a loss can be placed. +- The agent mix reports the same over the stretches of motion only (a gap above 250 ms ends a stretch). + +**Input latency proxy** (`input_p50_ms`, `input_p95_ms`, `input_max_ms`, `input_misses`, `input_call_p50_ms`). +It is a proxy and says so. Without Simlock or agent-device, the guest injects an event with `simctl`: by default +`simctl openurl glassbench://toggle`, which flips a 30 pt marker (white/black) in GlassBench `-probe`; the fallback is +`simctl ui appearance dark|light`, which flips the same marker; `--probe-cmd` may run any command that flips it (a +real tap through your own tool). The first N calibrates which method is answered (3 tries each) and the result +records it. For each injection: *t0* = the wall clock just before the command is started in the guest; the answer +is the first new surface after t0 whose marker (luma above or below 127 at a fixed point) differs from the last +surface before t0; latency = that surface's time - t0. A probe not answered in `--probe-timeout` (2 s) is a **miss +and counts at the timeout** in p50/p95, never dropped. The number includes starting `simctl` (its own duration is +`input_call_p50_ms`), CoreSimulator, the app's handling and one composite. It excludes the Device panel's stream, the +viewer and a real finger's HID path. Read it as a trend over N and between builds, not as an absolute touch latency; +the default `responsive` bar (150 ms) wants one calibration at N=1 on your hardware before you trust it. p95 over +few probes is coarse: a 40 s window gives 16 probes per simulator. + +**Guest CPU and memory of a simulator** (`guest_cpu_pct`, `guest_rss_mb`). From `ps` snapshots in the guest at the +window's start and end: the CPU time of every process in the simulator's tree over the window, in % of one core +(CPU time has a 10 ms resolution; a process that started or ended inside the window is counted only if it +was in both snapshots, so very short-lived processes are undercounted), and the RSS of the tree at the end. The +tree is every process whose command line names the simulator's UDID (`launchd_sim`, the per-device helpers) and +all their descendants. RSS counts shared pages in each process: it overstates what leaving a simulator out would +free. `system.guest`: `total_cpu_pct`, `sims_cpu_pct` (the trees), `other_cpu_pct` (everything else), `sims_rss_mb`. + +**Guest memory** (`system.guest`): `mem_used_mb` = active + wired + compressor pages; `mem_free_mb` = free + +speculative; `compressed_mb`; `swapins_d`, `swapouts_d`, `pageouts_d` = the counters' change over the window; +`pressure_max` = the highest `kern.memorystatus_vm_pressure_level` sampled every 2 s (1 normal, 2 warning, 4 +critical). + +**QEMU CPU** (`system`): from `/proc//task/*/stat` over the window, in cores: `qemu_user_cores` = host-side +user time (utime - guest_time), `qemu_guest_cores` = time in guest mode (the vCPUs running the guest), +`qemu_sys_cores` = kernel time; `qemu_cpu_cores` their sum; `qemu_top_threads` the six busiest thread classes (digits +folded: `CPU N/KVM` is `vCPU`). + +**GPU** (`system`): `gpu_busy_pct` = the change of `drm-engine-render` summed over QEMU's DRM clients (fdinfo, +each client once) divided by the wall time; `gpu_ms_per_frame` = that GPU time divided by the distinct frames of all +animating simulators in the window. i915 only (xe states cycles, not ns: the field is null, never converted by a guess). + +**Host memory and swap** (`system.host`): `mem_avail_gb`, `qemu_rss_mb`, `cgroup_mem_gb`, `cgroup_swap_mb` and its +change over the window, `cgroup_events` (high/max/oom_kill deltas), `pswpin_pages`/`pswpout_pages` (system wide), +`psi_mem_some_pct` (system) and `cgroup_psi_some_pct` (the VM's cgroup). + +**Stability** (`stability`): new `.ips`/`.crash` files in the guest's DiagnosticReports during the window, +simulators no longer Booted at its end, kernel GPU error lines (`journalctl -k`: GPU hang, fence expired, context +reset, xe job timeouts and faults), QEMU still the same process. Any one makes the step unstable and ends the sweep. + +**Fairness** (`fairness`, per N): Jain's index over the distinct fps of the animating simulators (1.0 = equal, 1/n = +one gets everything) and min/max. + +**Maximum sustainable N** (`summary.max_n`, one per target). A target is `name: fps>=55, p95<=25` over per +simulator metrics (`fps p50 p95 p99 max over2x stall input_p50 input_p95 cpu rss`). N passes when every simulator +that has a value for each condition meets it, with its gates ok and the system stable. The maximum is the largest N +such that it and **every smaller measured N** passed: the first failure ends it (a pass above a failure is luck, not +sustainability). A condition no simulator has a value for makes the target unknown (`null`), never a pass. +Defaults: `smooth60: fps>=55, p95<=25`, `smooth30: fps>=30, p95<=50`, `responsive: input_p95<=150, stall<0.3`. + +**Standing cost per simulator** (`summary.standing_cost`): the least-squares slope of each system number over N, with +its intercept and R². Use `--mix idle` for the pure cost of a booted simulator. + +## Correctness gates + +They run with every scenario; a FAIL fails the run and stops a sweep, but the result with every smaller N is +written first. A gate that cannot be read is SKIP, shown as such, never PASS. + +| gate | PASS when | +|---|---| +| `strip` | no frame out of order, no counter jump, the counter advanced no more than the app's own tick log says, under 1% of samples unreadable. Lost ticks are the fps metric, not a gate failure. | +| `tears` | no screenshot of the scrolling GlassBench shows a torn band (card spacing and gap heights, the old `tears.py` rule, standard library PNG decoding); SKIP when no screenshot shows cards | +| `pixels` | GlassBench `-static` screenshots agree with each other and match the stored golden (`baselines/golden.json`, key: macOS, runtime, device type, size); SKIP with the pictures agreeing when no golden is stored: `macbox bench golden` records it | +| `coherent_census` / `coherent_process` | in the census reports written since the run began: `overwrite_risk` and `sync_errors` stayed 0, and `late_copies_in_busy`, `late_beside_unplanned`, `plan_left_planned` did not rise. Deltas, not absolute counts. Reports come once a minute, so a short step may have none: SKIP. | +| `qemu_log` | no new `command buffer failed` line in QEMU's log (stamp waits are counted in the detail) | +| `gpu_hang` | the hang flag did not appear | +| `kernel_gpu` | no kernel GPU error line since the step began (SKIP when the journal is not readable) | +| `memory` | no host swapping (cgroup swap +50 MB, or 100 MB of system swap-outs); guest swap-outs fail it with `--fail-on-swap` | +| `leak_*` | `lifecycle`: QEMU RSS and guest memory grow less than `--leak-rss-mb` / `--leak-guest-mb` per create/boot/delete cycle (least-squares slope); `soak`: less than `--soak-rss-mb-min` / `--soak-guest-mb-min` per minute | +| `observer` | reading the strip does not move the surface frame rate by more than 4% (a second window of the first N without the strip read); otherwise the distinct-frame numbers are lower bounds | + +## A/B discipline + +`ab` runs one comparison per lever and nothing else. + +```sh +macbox bench ab --knob live:group_batches=16 --n 3 --metric total_fps --threshold 0.04 +macbox bench ab --set-cmd 'macbox x on' --unset-cmd 'macbox x off' --n 3 --metric worst_p95 +``` + +The rule is printed and stored **before** the run. Simulators boot once; then the arms run interleaved A B B A in +one window (`--arm-settle` between them, `--secs` each). Pair 1 = the first A with the first B, pair 2 = the second +B with the second A. **ADOPT** only if the mean effect of the two pairs exceeds the threshold (default 4%, the +run-to-run noise on this stack; fixed on the command line before the run), **both pairs agree in sign**, and no tail +metric (default `worst_p95`, `worst_p99` for the fps metrics) worsens by more than `--tail-tol` (4%). Anything else, +including a gate failure or instability in any arm, is **CLOSE**, with no re-run. Metrics: `total_fps` (sum of the +simulators' distinct fps), `min_fps`, `worst_p95`, `worst_p99`, `input_p95`, `gpu_ms_per_frame`, `qemu_cores`. The +decision line (`AB : ADOPT|CLOSE | metric A a1/a2 B b1/b2 | effect, pairs, bar, reasons`) is a section of the result +file. A knob that needs a VM restart cannot be A/B'd here (the arms share one boot, by design). +`--knob live:KEY=VALUE` edits one line of the VM's live switch file and puts the file back whole on every exit. + +## The panel section + +The Device panel needs Simlock and T3 Code, which the bench does not depend on. `panel --panel-cmd 'cmd'` runs your +command with `BENCH_PANEL_SECS` set and reads one JSON object from its last stdout line: +`{"frames": [[t_seconds, counter], ...]}`: every frame a viewer received through the panel and the counter +GlassBench's strip showed in it (the strip survives H.264). The bench then applies the same distinct-frame metrics and +strip gate and judges `--panel-fps` / `--panel-p95` (55 / 25). Without a command the section is SKIP. + +## The result file + +`/-.json`, `schema` 1: `kind`, `id`, `created`, `params` (the command line), `env` (`build_id`, +`switches` read from the live switch file, `qemu_env` (REIMS_VGPU_* names and values), `guest` macOS / Xcode / +runtime / device type, `host` kernel / Mesa / CPU, free-form `tags`), `sections` (the PASS/FAIL lines), `gates` (the +run's roll-up), `data` (for `scale`: `mix`, `scale_x`, `steps[]` with `sims[]`, `system`, `stability`, `gates`, +`targets`, `fairness`, and `summary`). The file is written after every step, so a run that dies keeps its data. Nothing in it +names a host, an account or a path: strings are scrubbed on write (home paths to `~`, host and ssh names to ``, +IPv4 addresses to ``), and `publishable` says whether anything slipped through (`problems` lists it). + +`baselines/` holds results worth comparing against (see its README); `golden.json` holds the pixel hashes. + +## Limits of the measurements + +- The observer is not free: `simfps` polls at 2 kHz and `--strip` reads each new surface with the CPU, which also + touches Coherent's surfaces. `observer` measures that at the first N; the guest CPU of the simulators includes it + only for the helper's own process, which is not in a simulator's tree. +- Probes run `simctl` in the guest every 2.5 s per simulator, a small steady load that agents also make. +- Windows are 40 s: slow drifts need `soak`; a stall rarer than once per window is invisible. +- GPU numbers need DRM fdinfo of QEMU's process to be readable by the user running the bench. + +## The tools + +The tools behind the numbers in [docs/performance.md](../../docs/performance.md). `macbox bench tools` copies and builds +them; by hand, inside macOS, once: ```sh tar -C bench -cf - . | ssh macbox 'mkdir -p ~/.macbox/bench && tar -C ~/.macbox/bench -xf -' @@ -10,8 +227,8 @@ ssh macbox 'cd ~/.macbox/bench && ./build.sh' | | | |---|---| -| `simfps [label] [--strip]` | counts the frames a simulator's screen produces, straight from CoreSimulator's framebuffer (no encoder in the way); one JSON line with fps and frame intervals. `--strip` also reads GlassBench's frame counter from each new picture (its synced reads slow the 2 kHz poll, so count frames without it) | -| `GlassBench.app` | the iOS load: a table scrolling by itself under Liquid Glass bars and a floating glass toolbar. It draws a **frame strip** at the top left: a 16-bit counter that advances every display-link tick, as 18 squares of 20 pt from (21, 120) pt (white and black references, then bits 0-15), large enough to survive H.264, so any viewer can tell unique frames from repeats and count the ones lost on the way. Its own tick intervals go to `Documents/pace.json` in its container every 5 s. `-nostrip` leaves the strip out | +| `simfps [label] [--strip]` | counts the frames a simulator's screen produces, straight from CoreSimulator's framebuffer (no encoder in the way); one JSON line with fps and frame intervals. `--strip` also reads GlassBench's frame counter from each new picture (its synced reads slow the 2 kHz poll, so count frames without it). Environment, used by the bench: `SIMFPS_SET` (a device set of its own), `SIMFPS_START_AT` (attach, then count from this unix second, so several simfps share one window), `SIMFPS_PROBE=x,y` (read this point's luma, in points of a 402 pt screen), `SIMFPS_STAMPS=path` (one line per new surface: unix seconds, counter or -1, probe luma or -1; the first line is the screen as it stands at the start) | +| `GlassBench.app` | the iOS load: a table scrolling by itself under Liquid Glass bars and a floating glass toolbar. It draws a **frame strip** at the top left: a 16-bit counter that advances every display-link tick, as 18 squares of 20 pt from (21, 120) pt (white and black references, then bits 0-15), large enough to survive H.264, so any viewer can tell unique frames from repeats and count the ones lost on the way. Its own tick intervals go to `Documents/pace.json` in its container every 5 s. `-nostrip` leaves the strip out, `-static` animates nothing (a fixed picture), `-probe` adds the input marker: a 30 pt square at (21, 160) pt that flips white/black on a tap or a `glassbench://toggle` URL | | `GenieBench.app` | the desktop load: a window that minimizes and restores, for counting the Genie animation's frames | | `scroll-local.html` | a page for Safari that scrolls itself under a blurred floating bar, with no network | diff --git a/src/bench/baselines/README.md b/src/bench/baselines/README.md new file mode 100644 index 0000000..ddc8322 --- /dev/null +++ b/src/bench/baselines/README.md @@ -0,0 +1,24 @@ +# Baselines + +Results worth comparing against, committed here so that a change can be held to them: + +```sh +macbox bench scale --n 1,2,3,4,5,6 --scale-x 1 --mix all-busy --out /tmp/r +macbox bench compare src/bench/baselines/.json /tmp/r/scale-.json +``` + +Format: a result file of the bench as it wrote it (`schema` 1, `"publishable": true`), nothing edited by hand. +Name it `--x-.json` (for example `scale-all-busy-1x-.json`), so a +comparison says what it compares. Only results with `"publishable": true` belong here: the bench scrubs host +names, accounts, addresses and paths on write, and `publishable` says whether anything slipped through. + +A baseline is only a baseline for the setup that made it (`env`: macOS and runtime, device type, host kernel and +Mesa, the live switches). `compare` prints the mix and scale of both and warns when they differ; it does not +compare across setups for you. Keep one baseline per (kind, mix, scale), replace it when the setup changes on +purpose, and say why in the commit. + +`golden.json` is different: the pixel hashes of GlassBench's static screen per (macOS, runtime, device type, size), +written by `macbox bench golden` and read by the `pixels` gate. Re-record it after a macOS or runtime update that +changes how text or glass renders; a hash that changed without such an update is a rendering regression. + +There are none yet: the first ones come from a run on the VM that has the build you want to hold. diff --git a/src/bench/guest/benchguest.py b/src/bench/guest/benchguest.py index 78263c4..746c8b5 100644 --- a/src/bench/guest/benchguest.py +++ b/src/bench/guest/benchguest.py @@ -88,10 +88,6 @@ def now(): return time.time() -if STUB: - from guest import benchguest_stub as _stub # noqa: E402 - _stub.install(sys.modules[__name__]) - # --- simctl ---------------------------------------------------------------------------------------- class GuestError(Exception): @@ -506,7 +502,7 @@ def cmd_settle(a): out_json({"settled": series[-1] < a.bg_pct, "bg_pct": series[-1], "waited_s": round(now() - t0, 1), "series": series[-8:]}) -def run_window(a, udids, roles, mix, secs, t_go, tmp, probe_method, probe_every, cmd_tpl, no_strip=False): +def run_window(a, udids, roles, mix, secs, t_go, tmp, probe_method, probe_every, cmd_tpl, no_strip=False, probe_timeout=2.0): """simfps on each simulator, the probe and agent threads, and the guest's samples around the window.""" raws, handles, stop_evt = {}, {}, threading.Event() if mix != "idle": @@ -525,8 +521,8 @@ def run_window(a, udids, roles, mix, secs, t_go, tmp, probe_method, probe_every, if mix in ("all-busy", "one-busy") and probe_method != "none": targets = [u for u, r in zip(udids, roles) if mix == "all-busy" or r == "static"] for k, u in enumerate(targets): - t = threading.Thread(target=probe_loop, args=(u, probe_method, cmd_tpl, t_go + 1.0 + k * probe_every / max(len(targets), 1), - probe_every, t_stop - 2.2, inj[u], stop_evt)) + t = threading.Thread(target=probe_loop, args=(u, probe_method, cmd_tpl, t_go + min(1.0, secs * 0.15) + k * probe_every / max(len(targets), 1), + probe_every, t_stop - probe_timeout - 0.2, inj[u], stop_evt)) threads.append(t) if mix == "agent": for i, u in enumerate(udids): @@ -576,7 +572,7 @@ def cmd_measure(a): tmp = tempfile.mkdtemp(prefix="benchwin") try: t_go = a.start_at or now() + 3 + len(udids) - out_json(run_window(a, udids, roles, a.mix, a.secs, t_go, tmp, a.probe_method, a.probe_every, a.probe_cmd, a.no_strip)) + out_json(run_window(a, udids, roles, a.mix, a.secs, t_go, tmp, a.probe_method, a.probe_every, a.probe_cmd, a.no_strip, a.probe_timeout)) finally: shutil.rmtree(tmp, ignore_errors=True) @@ -735,6 +731,9 @@ def cmd_calls(a): def main(argv=None): global SET + if STUB: # after every function is defined: the stub replaces the primitives (run, ps_text, list_ips...) + from guest import benchguest_stub + benchguest_stub.install(sys.modules[__name__]) ap = argparse.ArgumentParser(prog="benchguest") ap.add_argument("--set", default=os.environ.get("BENCH_SET")) sp = ap.add_subparsers(dest="cmd", required=True) @@ -759,7 +758,7 @@ def add(name, fn, *opts): add("measure", cmd_measure, O("--udids", required=True), O("--roles", required=True), O("--mix", default="all-busy"), O("--secs", type=float, default=40), O("--start-at", type=float, default=0), O("--probe-method", default="openurl"), O("--probe-every", type=float, default=2.5), O("--probe-cmd"), O("--agent-period", type=float, default=12), - O("--no-strip", action="store_true")) + O("--no-strip", action="store_true"), O("--probe-timeout", type=float, default=2.0)) add("shots", cmd_shots, O("--udids", required=True), O("--count", type=int, default=2)) add("cycle", cmd_cycle, O("--type", required=True), O("--runtime", required=True), O("--prefix", default="bench")) add("teardown", cmd_teardown) diff --git a/src/bench/guest/benchguest_stub.py b/src/bench/guest/benchguest_stub.py index a83a6fa..3a7ca34 100644 --- a/src/bench/guest/benchguest_stub.py +++ b/src/bench/guest/benchguest_stub.py @@ -232,9 +232,12 @@ def install(mod): mod.sysctl = lambda name: {"kern.memorystatus_vm_pressure_level": "1", "hw.ncpu": "8", "hw.memsize": str(16 << 30)}.get(name, "") mod.spawn_simfps = lambda udid, secs, label, strip, start_at, raw: StubSimfps(udid, secs, strip, start_at, raw) + calls = [0] + def ips(): + # a crash report appears during a measurement (the second look of a window), not before it s = load() - n = len(booted(s)) + calls[0] += 1 cr = fault("crash") - return set(s["ips"]) | ({"GlassBench-stub.ips"} if cr is not None and n >= cr else set()) + return set(s["ips"]) | ({"GlassBench-stub.ips"} if cr is not None and len(booted(s)) >= cr and calls[0] >= 2 else set()) mod.list_ips = ips diff --git a/src/bench/macbench/cli.py b/src/bench/macbench/cli.py index 869be10..f844f04 100644 --- a/src/bench/macbench/cli.py +++ b/src/bench/macbench/cli.py @@ -147,8 +147,8 @@ def record(self, key, h): def build_ctx(a, kind): dry = os.environ.get("DRY") == "1" run = Run(a.deadline, log, margin=float(os.environ.get("BENCH_DEADLINE_MARGIN", "60"))) - names = [os.environ.get("MACBOX_SSH_HOST", ""), os.environ.get("USER", ""), os.environ.get("MACBOX_GUEST_USER", "")] - clean = schema.scrubber([n for n in names if n]) + alias = os.environ.get("MACBOX_SSH_HOST", "") + clean = schema.scrubber([alias] if alias and alias != "macbox" else []) # not user names or the default alias: ordinary words stamp = schema.stamp() set_name = "bench-%s" % stamp faults = os.environ.get("BENCH_STUB_FAULT", "") diff --git a/src/bench/macbench/core.py b/src/bench/macbench/core.py index 0adfee3..37d0dec 100644 --- a/src/bench/macbench/core.py +++ b/src/bench/macbench/core.py @@ -136,7 +136,7 @@ def window(ctx, secs, mix, no_strip=False): t_go = time.time() + ctx.offset + lead ctx.run.need(secs + lead + 60, "a %d s window" % secs) args = ["measure", "--udids", udids, "--roles", roles, "--mix", mix, "--secs", secs, "--start-at", "%.3f" % t_go, - "--probe-method", ctx.probe_method or "openurl", "--probe-every", a.probe_every, "--agent-period", a.agent_period] + "--probe-method", ctx.probe_method or "openurl", "--probe-every", a.probe_every, "--agent-period", a.agent_period, "--probe-timeout", a.probe_timeout] if a.probe_cmd: args += ["--probe-cmd", a.probe_cmd] pend = ctx.guest.start(args, secs + lead + 240) diff --git a/src/bench/macbench/fakes.py b/src/bench/macbench/fakes.py index 23b71cb..692ccd1 100644 --- a/src/bench/macbench/fakes.py +++ b/src/bench/macbench/fakes.py @@ -22,8 +22,8 @@ def qemu_pid(self): def snap(self): now = time.monotonic() dt, self.t_last = now - self.t_last, now - self.cpu_ticks["vcpu"] += int(dt * 100 * (2.0 + 0.8 * self.n)) - self.cpu_ticks["sc"] += int(dt * 100 * (0.1 + 0.05 * self.n)) + self.cpu_ticks["vcpu"] += int(dt * 100 * (0.05 + 0.8 * self.n)) + self.cpu_ticks["sc"] += int(dt * 100 * (0.01 + 0.05 * self.n)) self.gpu += int(dt * 1e9 * min(0.95, 0.15 * self.n)) v, s = self.cpu_ticks["vcpu"], self.cpu_ticks["sc"] thr = {"1": ("CPU 0/KVM", v - v // 5, v // 10, v // 2), "2": ("metal2vulkan-sc", s, s // 4, 0)} diff --git a/src/bench/macbench/hostprobe.py b/src/bench/macbench/hostprobe.py index 5535c75..34ac23f 100644 --- a/src/bench/macbench/hostprobe.py +++ b/src/bench/macbench/hostprobe.py @@ -107,6 +107,15 @@ def parse_kv(text): return d +def parse_meminfo(text): + d = {} + for line in (text or "").splitlines(): + p = line.replace(":", " ").split() + if len(p) >= 2 and p[1].isdigit(): + d[p[0]] = int(p[1]) + return d + + def parse_psi(text): """/proc/pressure/memory or memory.pressure -> {"some": {avg10,total}, "full": {...}}""" d = {} @@ -127,7 +136,7 @@ def cgroup_dir(pid, root=""): def mem_snapshot(pid, root=""): """Host memory and swap counters, and the VM's cgroup's: every number is a counter or a level, deltas are taken by the caller.""" - mi = parse_kv((_read(root, "/proc/meminfo") or "").replace(":", " ")) + mi = parse_meminfo(_read(root, "/proc/meminfo")) vs = parse_kv(_read(root, "/proc/vmstat")) cg = cgroup_dir(pid, root) s = {"mem_total_kb": mi.get("MemTotal"), "mem_avail_kb": mi.get("MemAvailable"), "swap_total_kb": mi.get("SwapTotal"), @@ -200,7 +209,14 @@ def qemu_env_switches(pid, root=""): return out -def host_versions(): +def mesa_is_custom(pid, root=""): + """True when QEMU maps an ANV driver from outside the system library directories (its own Mesa build).""" + t = _read(root, "/proc/%d/maps" % pid) or "" + paths = {l.split()[-1] for l in t.splitlines() if "libvulkan_intel" in l and l.split()[-1].startswith("/")} + return bool(paths) and not all(p.startswith(("/usr/lib", "/lib")) for p in paths) + + +def host_versions(pid=None): kernel = os.uname().release mesa = None for argv in (["dpkg-query", "-W", "-f=${Version}", "mesa-vulkan-drivers"], ["rpm", "-q", "--qf", "%{VERSION}", "mesa-vulkan-drivers"]): @@ -218,7 +234,12 @@ def host_versions(): cpu = m.group(1).strip() if m else None except OSError: pass - return {"kernel": kernel, "mesa": mesa or "unknown (pass --tag mesa=)", "cpu_model": cpu, "cpus": os.cpu_count()} + note = "unknown" + if mesa: + note = mesa + " (system package)" + if pid and mesa_is_custom(pid): + note = "custom build in QEMU (system package is %s): pass --tag mesa=" % (mesa or "n/a") + return {"kernel": kernel, "mesa": note, "cpu_model": cpu, "cpus": os.cpu_count()} class Host: @@ -257,4 +278,4 @@ def same_qemu(self): def env(self): sw = read_live_conf(self.live_conf) if self.live_conf else {} - return host_versions(), sw, qemu_env_switches(self.pid, self.root) + return host_versions(self.pid), sw, qemu_env_switches(self.pid, self.root) diff --git a/src/bench/macbench/scale.py b/src/bench/macbench/scale.py index 802f9e4..ec6a6af 100644 --- a/src/bench/macbench/scale.py +++ b/src/bench/macbench/scale.py @@ -52,13 +52,13 @@ def run_scale(ctx): "summary": {}, "calibration": None, "observer": None} ctx.marks = {"coh": ctx.coh.mark(), "t0": time.time()} ctx.run.on_deadline = lambda: (_finish(ctx, "deadline"), None)[1] - stopped, first = None, True + stopped, first, kind = None, True, None try: for n in a.ns: roles = core.roles_for(mix, n) reason = core.guard(ctx, n - len(ctx.sims), "before booting simulator %d" % n) if reason: - stopped = reason + stopped, kind = reason, "guard" break ctx.host.n = n boot = core.grow(ctx, n, roles) @@ -85,19 +85,21 @@ def run_scale(ctx): first = False if not step["gates_ok"]: bad = [k for k, v in step["gates"].items() if v["status"] == "FAIL"] - stopped = "gate failed at N=%d: %s" % (n, ", ".join(bad)) + stopped, kind = "gate failed at N=%d: %s" % (n, ", ".join(bad)), "gate" break if not step["stable"]: - stopped = "instability at N=%d: %s" % (n, _why_unstable(step)) + stopped, kind = "instability at N=%d: %s" % (n, _why_unstable(step)), "instability" break fl = targets.evaluate(floor, step["sims"]) if fl["pass"] is False: + kind = "floor" stopped = "floor %s breached at N=%d (sim %s: %s = %s)" % (a.floor, n, fl["failing"][0]["sim"], fl["failing"][0]["cond"], fl["failing"][0]["value"]) break except Stop as e: - stopped = str(e) + stopped, kind = str(e), "error" except (DeadlineError, GuestError) as e: - stopped = "%s: %s" % (type(e).__name__, e) + stopped, kind = "%s: %s" % (type(e).__name__, e), "error" + data["stopped_kind"] = kind return _finish(ctx, stopped) @@ -177,7 +179,7 @@ def _sections(ctx, data, stopped): gate_fail = [k for k, v in ctx.doc["gates"].items() if v["status"] == "FAIL"] unstable = any(not s["stable"] for s in steps) mx = ", ".join("%s=%s" % (k, v["max_n"] if v["max_n"] is not None else "none") for k, v in sm["max_n"].items()) - status = "FAIL" if gate_fail or unstable else "PASS" + status = "FAIL" if gate_fail or unstable or data.get("stopped_kind") == "error" else "PASS" ctx.section("scale", status, "SCALE %s %s %dx %s: max sustainable N: %s | steps %s%s | min Jain %s" % ( status, a.mix, a.scale_x, "(DRY)" if ctx.host.dry else "", mx, [s["n"] for s in steps], (" | stopped: %s" % stopped) if stopped else " | all requested N done", sm.get("fairness_min_jain"))) diff --git a/src/bench/macbench/targets.py b/src/bench/macbench/targets.py index f18c39e..5fc1616 100644 --- a/src/bench/macbench/targets.py +++ b/src/bench/macbench/targets.py @@ -4,8 +4,8 @@ Metrics (per simulator): fps (distinct app frames/s), p50, p95, p99, max (ms between distinct frames), over2x, stall (longest gap between distinct frames, s), input_p50, input_p95 (ms, proxy; misses count at the timeout), cpu (guest CPU % of one core), rss (MB). A metric a simulator has no value for (fps of an idle simulator, -the input proxy where no probe ran) is not judged for that simulator; a target no simulator of an N has any -value for is `None` (unknown), never a pass.""" +the input proxy where no probe ran) is not judged for that simulator; a target with a condition that NO simulator of +an N has a value for is `None` (unknown), never a pass (unless another condition already failed it).""" import re METRICS = {"fps": "fps", "p50": "p50_ms", "p95": "p95_ms", "p99": "p99_ms", "max": "max_ms", "over2x": "over2x", @@ -40,16 +40,19 @@ def parse_target(spec): def evaluate(target, sims): """target against one N's per-simulator dicts: {"pass": True|False|None, "failing": [...], "judged": int}""" failing, judged = [], 0 + unjudged = set(c[0] for c in target["conds"]) for sim in sims: for metric, op, val in target["conds"]: v = sim.get(METRICS[metric]) if v is None: continue judged += 1 + unjudged.discard(metric) if not OPS[op](v, val): failing.append({"sim": sim.get("idx"), "cond": "%s%s%g" % (metric, op, val), "value": v}) - if judged == 0: - return {"pass": None, "failing": [], "judged": 0} + if judged == 0 or (unjudged and not failing): + # a condition no simulator has a value for is unknown, and so is the target (never a pass by default) + return {"pass": None, "failing": failing, "judged": judged, "unjudged": sorted(unjudged)} return {"pass": not failing, "failing": failing, "judged": judged} diff --git a/src/bench/tests/test_dry.py b/src/bench/tests/test_dry.py new file mode 100644 index 0000000..7fdecda --- /dev/null +++ b/src/bench/tests/test_dry.py @@ -0,0 +1,253 @@ +"""DRY=1 runs of the whole suite against the stub guest and a fake host: control flow, faults, safety, outputs. +Each case is a real `run` process (its own signal handlers, its own exit trap). They run in parallel.""" +import concurrent.futures +import json +import os +import signal +import subprocess +import sys +import tempfile +import time +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +RUN = os.path.join(HERE, "..", "run") +FAST = ["--secs", "3", "--probe-every", "0.7", "--probe-timeout", "1", "--settle-max", "2", "--no-observer-check"] + + +def run_case(args, faults="", extra_env=None, kill_after=None, keep=True, timeout=240): + d = tempfile.mkdtemp(prefix="dry-case-") + env = dict(os.environ, DRY="1", BENCH_STUB_FAULT=faults, BENCH_DRY_KEEP=os.path.join(d, "state") if keep else "", + BENCH_DEADLINE_MARGIN="3", **(extra_env or {})) + if not keep: + env.pop("BENCH_DRY_KEEP") + p = subprocess.Popen([sys.executable, RUN] + args + ["--out", os.path.join(d, "out")], env=env, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True) + if kill_after: + time.sleep(kill_after) + p.send_signal(signal.SIGTERM) + out, _ = p.communicate(timeout=timeout) + res = None + od = os.path.join(d, "out") + if os.path.isdir(od): + for f in sorted(os.listdir(od)): + if f.endswith(".json"): + with open(os.path.join(od, f)) as fh: + res = json.load(fh) + return {"rc": p.returncode, "out": out, "doc": res, "state": os.path.join(d, "state")} + + +def sims_booted(state): + try: + with open(os.path.join(state, "guest", "state.json")) as f: + return [u for u, v in json.load(f)["devs"].items() if v["state"] == "Booted"] + except OSError: + return None + + +def simctl_calls(state): + out = [] + with open(os.path.join(state, "guest", "calls.log")) as f: + for line in f: + out.append(json.loads(line)) + return [c for c in out if c[:2] == ["xcrun", "simctl"]] + + +def sections(doc, name): + return [s for s in doc["sections"] if s["name"] == name] + + +CASES = { + "clean": (["scale", "--n", "1,2,3"] + FAST, ""), + "memfloor": (["scale", "--n", "1,2,3,4,5"] + FAST, "memfloor@3"), + "hostmem": (["scale", "--n", "1,2,3,4,5"] + FAST, "hostmem@3"), + "crash": (["scale", "--n", "1,2,3,4"] + FAST, "crash@2"), + "backwards": (["scale", "--n", "1,2,3"] + FAST, "backwards@2"), + "kernelgpu": (["scale", "--n", "1,2,3"] + FAST, "kernelgpu@2"), + "coherent": (["scale", "--n", "1,2,3"] + FAST, "coherent@2"), + "slow": (["scale", "--n", "1,2,3,4"] + FAST, "slow@3"), + "hang": (["scale", "--n", "1,2,3"] + FAST, "hang_simfps"), + "nomarker": (["scale", "--n", "1,2"] + FAST, "nomarker"), + "onebusy": (["scale", "--n", "1,2,3", "--mix", "one-busy"] + FAST, ""), + "agent": (["scale", "--n", "1,2", "--mix", "agent", "--agent-period", "2"] + FAST, ""), + "idlemix": (["scale", "--n", "1,2", "--mix", "idle"] + FAST, ""), + "idle": (["idle", "--secs", "3", "--settle-max", "2"], ""), + "lifecycle": (["lifecycle", "--cycles", "2", "--storm", "2", "--secs", "3"], ""), + "build": (["build"], ""), + "panel_skip": (["panel"], ""), +} + + +class Dry(unittest.TestCase): + @classmethod + def setUpClass(cls): + with concurrent.futures.ThreadPoolExecutor(8) as ex: + futs = {k: ex.submit(run_case, a, f) for k, (a, f) in CASES.items()} + futs["ssh"] = ex.submit(run_case, ["scale", "--n", "1,2,3"] + FAST, "", {"BENCH_STUB_SSH_TIMEOUT": "20"}) + futs["term"] = ex.submit(run_case, ["scale", "--n", "1,2,3,4,5,6"] + FAST, "", None, 35) + futs["deadline"] = ex.submit(run_case, ["scale", "--n", "1,2,3,4,5,6", "--deadline", "40"] + FAST, "") + cls.r = {k: f.result() for k, f in futs.items()} + + def test_clean_sweep(self): + r = self.r["clean"] + self.assertEqual(r["rc"], 0, r["out"][-800:]) + d = r["doc"] + self.assertEqual([s["n"] for s in d["data"]["steps"]], [1, 2, 3]) + self.assertEqual(d["data"]["summary"]["max_n"]["smooth60"]["max_n"], 2) + self.assertTrue(d["publishable"], d.get("problems")) + self.assertEqual(sims_booted(r["state"]), []) + self.assertEqual(sections(d, "cleanup")[0]["status"], "PASS") + for s in d["data"]["steps"]: + self.assertEqual(len(s["sims"]), s["n"]) + for sim in s["sims"]: + self.assertIn("input_p95_ms", sim) + self.assertIn("guest_rss_mb", sim) + + def test_every_simctl_call_names_the_bench_set(self): + for k in ("clean", "onebusy", "lifecycle", "agent"): + calls = simctl_calls(self.r[k]["state"]) + self.assertTrue(calls, k) + for c in calls: + if c[2:4] == ["list", "devicetypes"] or c[2:4] == ["list", "runtimes"] or c[2:5] == ["list", "devices", "booted"]: + continue # global, read only + self.assertEqual(c[2], "--set", (k, c)) + + def test_memory_floor_stops_the_sweep_and_keeps_the_data(self): + r = self.r["memfloor"] + d = r["doc"] + self.assertEqual([s["n"] for s in d["data"]["steps"]], [1, 2, 3]) # the guard looks before booting: it refuses the 4th + self.assertIn("guest free memory", d["data"]["summary"]["stopped"]) + self.assertEqual(sims_booted(r["state"]), []) + self.assertEqual(self.r["hostmem"]["doc"]["data"]["steps"][-1]["n"], 3) + self.assertIn("host free memory", self.r["hostmem"]["doc"]["data"]["summary"]["stopped"]) + + def test_crash_report_is_instability_and_ends_the_sweep(self): + d = self.r["crash"]["doc"] + self.assertEqual(d["data"]["steps"][-1]["n"], 2) + self.assertFalse(d["data"]["steps"][-1]["stable"]) + self.assertIn("instability", d["data"]["summary"]["stopped"]) + self.assertEqual(sections(d, "stability")[0]["status"], "FAIL") + self.assertNotEqual(self.r["crash"]["rc"], 0) + + def test_strip_out_of_order_fails_the_run_but_keeps_smaller_n(self): + r = self.r["backwards"] + d = r["doc"] + self.assertEqual(d["gates"]["strip"]["status"], "FAIL") + self.assertEqual(d["data"]["steps"][0]["gates"]["strip"]["status"], "PASS") # N=1 intact + self.assertEqual(d["data"]["steps"][-1]["n"], 2) + self.assertEqual(r["rc"], 1) + + def test_kernel_and_coherent_gates(self): + k = self.r["kernelgpu"]["doc"] + self.assertEqual(k["gates"]["kernel_gpu"]["status"], "FAIL") + self.assertFalse(k["data"]["steps"][-1]["stable"]) + c = self.r["coherent"]["doc"] + self.assertEqual(c["gates"]["coherent_census"]["status"], "FAIL") + self.assertEqual(c["data"]["steps"][-1]["n"], 2) + + def test_floor_breach_stops_early(self): + d = self.r["slow"]["doc"] + self.assertEqual(d["data"]["steps"][-1]["n"], 3) + self.assertIn("floor", d["data"]["summary"]["stopped"]) + self.assertEqual(d["data"]["summary"]["max_n"]["smooth60"]["max_n"], 2) + + def test_hung_observer_is_reported_not_swallowed(self): + d = self.r["hang"]["doc"] + self.assertTrue(d["data"]["steps"][0]["stability"]["observer_errors"]) + self.assertEqual(sims_booted(self.r["hang"]["state"]), []) + + def test_unanswered_input_probe_is_not_dropped(self): + d = self.r["nomarker"]["doc"] + cal = d["data"]["calibration"] + self.assertEqual(cal["chosen"], "none") + self.assertEqual(d["data"]["steps"][0]["targets"]["responsive"], None) # unknown, never a pass + + def test_one_busy_mix(self): + d = self.r["onebusy"]["doc"] + st = d["data"]["steps"][-1] + self.assertEqual([s["role"] for s in st["sims"]], ["busy", "static", "static"]) + self.assertIn("input_p95_ms", st["sims"][1]) + self.assertNotIn("fps", st["sims"][1]) + self.assertIsNone(st["sims"][1].get("longest_gap_s")) + + def test_agent_mix(self): + st = self.r["agent"]["doc"]["data"]["steps"][-1] + self.assertEqual([s["role"] for s in st["sims"]], ["agent", "agent"]) + self.assertTrue(any(s.get("actions") for s in st["sims"]), st["sims"]) + + def test_idle_mix_has_no_frames_or_probes(self): + st = self.r["idlemix"]["doc"]["data"]["steps"][-1] + self.assertEqual({s["role"] for s in st["sims"]}, {"idle"}) + self.assertNotIn("input_p95_ms", st["sims"][0]) + + def test_idle_lifecycle_build_panel(self): + self.assertEqual(self.r["idle"]["rc"], 0, self.r["idle"]["out"][-500:]) + self.assertIn("IDLE", self.r["idle"]["doc"]["sections"][0]["line"]) + lc = self.r["lifecycle"]["doc"] + self.assertTrue(sections(lc, "lifecycle")) + self.assertTrue(sections(lc, "storm")) + self.assertIn("leak_qemu_rss", lc["gates"]) + self.assertEqual(sims_booted(self.r["lifecycle"]["state"]), []) + self.assertEqual(sections(self.r["build"]["doc"], "xcodebuild")[0]["status"], "SKIP") + self.assertEqual(sections(self.r["panel_skip"]["doc"], "panel")[0]["status"], "SKIP") + + def test_ssh_timeout_mid_run_keeps_results_and_cleans_up(self): + r = self.r["ssh"] + d = r["doc"] + self.assertTrue(d["data"]["steps"], r["out"][-600:]) + self.assertIn("ssh timed out", d["data"]["summary"]["stopped"]) + self.assertEqual(sims_booted(r["state"]), []) + + def test_sigterm_runs_the_one_trap(self): + r = self.r["term"] + self.assertEqual(r["rc"], 143, r["out"][-500:]) + self.assertEqual(sims_booted(r["state"]), []) + self.assertIn("cleanup: bench simulators", r["out"]) + self.assertTrue(r["doc"] is not None and r["doc"]["data"]["steps"]) + + def test_deadline_refuses_a_window_that_does_not_fit(self): + r = self.r["deadline"] + self.assertNotEqual(r["rc"], 0, r["out"][-500:]) # a window that does not fit before the deadline is refused + self.assertEqual(sims_booted(r["state"]), []) + self.assertIn("no time for", r["doc"]["data"]["summary"]["stopped"]) + self.assertEqual(r["doc"]["data"]["stopped_kind"], "error") + + +class KnobsAndAB(unittest.TestCase): + def ab(self, extra, faults="", kill_after=None): + d = tempfile.mkdtemp(prefix="dry-ab-") + live = os.path.join(d, "live.conf") + return d, live + + def test_ab_adopt_close_and_knob_restored(self): + # DRY's live switch file is created by the harness; the knob must come back byte for byte + r = run_case(["ab", "--knob", "live:group_x=1", "--n", "2", "--arm-settle", "0.2", "--secs", "3", "--settle-max", "2", "--threshold", "0.04"], "") + d = r["doc"] + self.assertEqual(r["rc"], 0, r["out"][-800:]) + self.assertIn(d["data"]["decision"]["decision"], ("ADOPT", "CLOSE")) + self.assertEqual([a["arm"] for a in d["data"]["arms"]], ["A", "B", "B", "A"]) + self.assertIn("AB live:group_x=1:", d["data"]["line"]) + self.assertIn("bar 4.0%", d["data"]["line"]) + self.assertIn("rule", d["params"]) + self.assertIn("restored: live switches file", r["out"]) + live = os.path.join(r["state"], "live.conf") + with open(live) as f: + self.assertEqual(f.read(), "surface_coherent=1\n") + self.assertEqual(sims_booted(r["state"]), []) + + def test_ab_sigterm_mid_arm_restores_the_knob(self): + r = run_case(["ab", "--knob", "live:group_x=1", "--n", "2", "--arm-settle", "0.2", "--secs", "6", "--settle-max", "2"], "", None, kill_after=18) + self.assertEqual(r["rc"], 143, r["out"][-600:]) + live = os.path.join(r["state"], "live.conf") + with open(live) as f: + self.assertEqual(f.read(), "surface_coherent=1\n") + self.assertEqual(sims_booted(r["state"]), []) + + def test_ab_needs_a_knob(self): + r = run_case(["ab", "--n", "1", "--secs", "3"], "") + self.assertNotEqual(r["rc"], 0) + self.assertIn("a knob is needed", r["out"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/bench/tests/test_modules.py b/src/bench/tests/test_modules.py index 29a5cf6..df74c82 100644 --- a/src/bench/tests/test_modules.py +++ b/src/bench/tests/test_modules.py @@ -116,6 +116,11 @@ def put(rel, text): self.assertEqual(a["cg_current"], 4294967296) self.assertEqual(a["cg_swap"], 1048576) self.assertEqual(a["qemu_rss_kb"], 2048) + self.assertEqual((a["mem_avail_kb"], a["mem_total_kb"]), (500000, 1000000)) + put("/proc/7/maps", "7f00 r-xp 0 00:00 0 /opt/mesa/lib/libvulkan_intel.so\n") + self.assertTrue(hostprobe.mesa_is_custom(7, root)) + put("/proc/7/maps", "7f00 r-xp 0 00:00 0 /usr/lib/x86_64-linux-gnu/libvulkan_intel.so\n") + self.assertFalse(hostprobe.mesa_is_custom(7, root)) put("/proc/vmstat", "pswpin 15\npswpout 120\n") put("/sys/fs/cgroup/machine/vm.scope/memory.swap.current", "3145728\n") b = hostprobe.mem_snapshot(7, root) @@ -196,7 +201,7 @@ def test_fake_result_validates_and_is_private(self): def test_privacy_catches_paths_ips_hosts(self): doc = helpers.fake_scale_result() - doc["env"]["x"] = "/srv/data/something/live.conf" + doc["env"]["x"] = "/var/lib/something/live.conf" self.assertTrue(any("path" in p for p in schema.validate(doc))) doc["env"]["x"] = "connected to 192.168.1.20" self.assertTrue(any("ip" in p for p in schema.validate(doc))) @@ -221,7 +226,7 @@ def test_scrub_then_publishable(self): def test_write_keeps_data_even_when_not_publishable(self): doc = helpers.fake_scale_result() - doc["env"]["x"] = "/srv/private/path" + doc["env"]["x"] = "/var/private/path" p = os.path.join(tempfile.mkdtemp(), "r.json") problems = schema.write(doc, p) self.assertTrue(problems) diff --git a/src/bench/tests/test_pure.py b/src/bench/tests/test_pure.py index 7241b79..5625623 100644 --- a/src/bench/tests/test_pure.py +++ b/src/bench/tests/test_pure.py @@ -177,6 +177,9 @@ def test_every_sim_must_pass(self): def test_unknown_when_nothing_to_judge(self): t = targets.parse_target("t: input_p95<=150") self.assertIsNone(targets.evaluate(t, self.sims([60]))["pass"]) + t = targets.parse_target("t: fps>=55, input_p95<=150") # one condition has no value anywhere: unknown, not a pass + self.assertIsNone(targets.evaluate(t, self.sims([60]))["pass"]) + self.assertFalse(targets.evaluate(t, self.sims([40]))["pass"]) # but a failed condition still fails it def test_max_sustainable_stops_at_first_fail(self): steps = [{"n": n, "sims": self.sims([f] * n), "gates_ok": True} for n, f in ((1, 60), (2, 58), (3, 50), (4, 60))] diff --git a/src/bench/tests/test_runctx.py b/src/bench/tests/test_runctx.py new file mode 100644 index 0000000..ee39877 --- /dev/null +++ b/src/bench/tests/test_runctx.py @@ -0,0 +1,89 @@ +"""The run's safety, without the rest: the watchdog, the one trap, knobs.""" +import os +import signal +import subprocess +import sys +import tempfile +import time +import unittest + + +def rd(path): + with open(path) as f: + return f.read() + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..")) +from macbench.runctx import LiveKnob, Run # noqa: E402 + +SCRIPT = r''' +import sys, time +sys.path.insert(0, %r) +from macbench.runctx import Run +r = Run(6, print, margin=2) +r.teardown.append(("t", lambda: print("TEARDOWN RAN", flush=True))) +r.undo.append(("u", lambda: print("UNDO RAN", flush=True))) +r.on_deadline = lambda: print("PARTIAL WRITTEN", flush=True) +r.install() +mode = sys.argv[1] +if mode == "hang": + time.sleep(60) +elif mode == "child": + p = r.popen(["sleep", "60"]) + print("CHILD", p.pid, flush=True) + time.sleep(60) +''' + + +class RunSafety(unittest.TestCase): + def go(self, mode, sig=None, after=1.0): + p = subprocess.Popen([sys.executable, "-c", SCRIPT % os.path.join(HERE, ".."), mode], stdout=subprocess.PIPE, text=True) + if sig: + time.sleep(after) + p.send_signal(sig) + out, _ = p.communicate(timeout=30) + return p.returncode, out + + def test_deadline_runs_partial_write_then_cleanup_and_exits_124(self): + t0 = time.time() + rc, out = self.go("hang") + self.assertEqual(rc, 124) + self.assertLess(time.time() - t0, 10) + self.assertLess(out.index("PARTIAL WRITTEN"), out.index("TEARDOWN RAN")) + self.assertLess(out.index("TEARDOWN RAN"), out.index("UNDO RAN")) # simulators down first, then knobs restored + + def test_sigterm_runs_cleanup_once_and_kills_children(self): + rc, out = self.go("child", signal.SIGTERM) + self.assertEqual(rc, 143) + self.assertEqual(out.count("TEARDOWN RAN"), 1) + pid = int(out.split("CHILD")[1].split()[0]) + time.sleep(0.2) + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + + def test_need_refuses_what_does_not_fit(self): + from macbench.runctx import DeadlineError + r = Run(100, margin=10) + r.need(40, "x") + with self.assertRaises(DeadlineError): + r.need(80, "y") + + def test_live_knob_restores_bytes(self): + d = tempfile.mkdtemp() + p = os.path.join(d, "live.conf") + with open(p, "w") as f: + f.write("a=1\n# comment\nb=2\nk=old\n") + r = Run(100) + k = LiveKnob(p, "k", "new", r) + k.set() + self.assertEqual(rd(p).splitlines()[-1], "k=new") + self.assertNotIn("k=old", rd(p)) + k.unset() + self.assertEqual(rd(p), "a=1\n# comment\nb=2\nk=old\n") + k.set() + r.cleanup() + self.assertEqual(rd(p), "a=1\n# comment\nb=2\nk=old\n") + + +if __name__ == "__main__": + unittest.main() From ab0aaf5b390c7a796af5cac0c96df7c0f4035a25 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:35:04 +0200 Subject: [PATCH 07/23] bench: --slim for the pool's Slim configuration; ANV from QEMU's own Mesa is flagged; meminfo parse fixed --- src/bench/README.md | 4 ++-- src/bench/guest/benchguest.py | 19 +++++++++++++++++++ src/bench/macbench/cli.py | 2 ++ src/bench/macbench/core.py | 6 ++++++ 4 files changed, 29 insertions(+), 2 deletions(-) diff --git a/src/bench/README.md b/src/bench/README.md index 68ec100..213a03a 100644 --- a/src/bench/README.md +++ b/src/bench/README.md @@ -14,7 +14,7 @@ DRY=1 macbox bench scale --n 1,2,3 # the whole control flow against stubs: ``` Requirements: Python 3.8+ on the host (standard library only; no Pillow, no numpy), the VM running, `ssh` to it. -Unit tests, no VM needed: `cd src/bench && python3 -m unittest discover -s tests -t .` (the pure parts, 55 cases) +Unit tests, no VM needed: `cd src/bench && python3 -m unittest discover -s tests -t .` (the pure parts and the run's safety) and `python3 -m unittest tests.test_dry` (the dry runs with injected faults, a couple of minutes). ## What a run does, and what it never touches @@ -57,7 +57,7 @@ refuse the run (`--allow-foreign` to measure anyway, recorded). Common options: `--out DIR` (results, default `bench-results/`), `--secs` (window length, default 40), `--nominal-fps 60`, `--tag key=value` (recorded, for example `mesa=26.2.4` when Mesa is not a package), -`--probe-method auto|openurl|appearance|cmd|none`, `--quiesce` (disables `mediaanalysisd` in each simulator, as +`--probe-method auto|openurl|appearance|cmd|none`, `--slim` (the pool's Slim configuration through simslim on the bench's simulators: without it they are stock simulators, which cost more; it needs simslim in the guest, `macbox-guest tools`, and whether simslim finds devices of a custom device set is untested: the run stops with that message if it does not), `--quiesce` (disables `mediaanalysisd` in each simulator, as some old harnesses did; recorded, off by default), `--no-observer-check`, `--no-golden`, `--shots N`. Every run prints one PASS/FAIL/SKIP/INFO line per section and exits 0, 1 (a section failed) or 2 (the run could not go on). The JSON has the same lines under `sections`. diff --git a/src/bench/guest/benchguest.py b/src/bench/guest/benchguest.py index 746c8b5..0d72482 100644 --- a/src/bench/guest/benchguest.py +++ b/src/bench/guest/benchguest.py @@ -440,6 +440,24 @@ def _boot_one(udid, res, i): "err": None if ok and rc2 == 0 else err.strip()[-120:]} +def slim_count(udid): + """Services simslim keeps disabled for a simulator (what macbox-guest counts), 0 = stock.""" + f = "/private/var/tmp/com.apple.CoreSimulator.SimDevice.%s/disabled.plist" % udid + if not os.path.exists(f): + return 0 + return run(["plutil", "-p", f], 15)[1].count("=> true") + + +def cmd_slim(a): + """The pool's Slim configuration on the bench's simulators: runs --cmd ({udid}) for each, then counts the + services left disabled. HYPOTHESIS: simslim finds devices of a custom device set; the result says.""" + res = [] + for u in a.udids.split(","): + rc, out, err, secs = run(["/bin/sh", "-c", a.cmd.replace("{udid}", u)], 900) + res.append({"udid": u, "rc": rc, "s": round(secs, 1), "services_off": slim_count(u), "err": (err or "").strip()[-120:] if rc else None}) + out_json({"sims": res}) + + def cmd_boot(a): us = a.udids.split(",") res = [None] * len(us) @@ -750,6 +768,7 @@ def add(name, fn, *opts): add("ensure-type", cmd_ensure_type, O("--scale", default="1"), O("--allow-restart", action="store_true")) add("create", cmd_create, O("--n", type=int, required=True), O("--type", required=True), O("--runtime", required=True), O("--prefix", default="bench"), O("--start", type=int, default=0)) + add("slim", cmd_slim, O("--udids", required=True), O("--cmd", required=True)) add("boot", cmd_boot, O("--udids", required=True), O("--parallel", action="store_true")) add("prepare", cmd_prepare, O("--udids", required=True), O("--roles", required=True), O("--install", action="store_true"), O("--quiesce", action="store_true")) diff --git a/src/bench/macbench/cli.py b/src/bench/macbench/cli.py index f844f04..c03683a 100644 --- a/src/bench/macbench/cli.py +++ b/src/bench/macbench/cli.py @@ -38,6 +38,8 @@ def add_common(p): p.add_argument("--lead", type=float, default=None, help=argparse.SUPPRESS) p.add_argument("--settle-step", type=float, default=3.0, help=argparse.SUPPRESS) p.add_argument("--settle-bg", type=float, default=40, help="the guest counts as settled below this background CPU %% (of one core)") + p.add_argument("--slim", action="store_true", help="make the bench simulators Slim like the pool's (simslim; a minute or two each)") + p.add_argument("--slim-cmd", default="$HOME/go/bin/simslim on {udid} --preserve-boot-state", help=argparse.SUPPRESS) p.add_argument("--quiesce", action="store_true", help="disable mediaanalysisd in each simulator (recorded; production does not)") p.add_argument("--fail-on-swap", action="store_true", help="a guest swap-out in a window fails the memory gate (default: reported, host swap fails)") p.add_argument("--golden-file", default=os.path.join(BENCH_DIR, "baselines", "golden.json")) diff --git a/src/bench/macbench/core.py b/src/bench/macbench/core.py index 37d0dec..7c3de1f 100644 --- a/src/bench/macbench/core.py +++ b/src/bench/macbench/core.py @@ -104,6 +104,12 @@ def grow(ctx, n, roles, install=True): r = ctx.guest.call(["create", "--n", new, "--type", ctx.type_id, "--runtime", ctx.runtime, "--start", have], 240) fresh = r["sims"] us = ",".join(s["udid"] for s in fresh) + if ctx.a.slim: + sl = ctx.guest.call(["slim", "--udids", us, "--cmd", ctx.a.slim_cmd], 900 * len(fresh)) + bad = [x for x in sl["sims"] if x["rc"] != 0 or not x["services_off"]] + if bad: + raise Stop("--slim: simslim did not slim a bench simulator (rc %s, %s services off): %s" % (bad[0]["rc"], bad[0]["services_off"], bad[0]["err"])) + ctx.doc["env"]["guest"]["slim_services_off"] = sl["sims"][0]["services_off"] b = ctx.guest.call(["boot", "--udids", us] + (["--parallel"] if new > 1 else []), 900) bad = [s for s in b["sims"] if not s.get("ok")] if bad: From 87641068729e399d7a0095d1177868c623af6e10 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:49:56 +0200 Subject: [PATCH 08/23] bench: guest memory counts cache, observer out of the simulators' trees, clone timing, light appearance, foreign launchd_sim check --- src/bench/guest/benchguest.py | 91 +++++++++++++++++++++++------- src/bench/guest/benchguest_stub.py | 9 ++- src/bench/tests/test_modules.py | 22 ++++++++ 3 files changed, 100 insertions(+), 22 deletions(-) diff --git a/src/bench/guest/benchguest.py b/src/bench/guest/benchguest.py index 0d72482..d0434ee 100644 --- a/src/bench/guest/benchguest.py +++ b/src/bench/guest/benchguest.py @@ -131,7 +131,7 @@ def parse_vm_stat(text): """vm_stat -> {"page": bytes, "free": pages, ...}. Keys lower-cased, spaces to underscores.""" d = {} m = re.search(r"page size of (\d+) bytes", text) - d["page"] = int(m.group(1)) if m else 16384 + d["page"] = int(m.group(1)) if m else (4096 if os.uname()[4] == "x86_64" else 16384) for line in text.splitlines(): m = re.match(r"^\"?([A-Za-z ()-]+?)\"?:\s+(\d+)\.?\s*$", line) if m: @@ -145,6 +145,8 @@ def mem_summary(vm): g = lambda k: vm.get(k, 0) used = (g("pages_active") + g("pages_wired_down") + g("pages_occupied_by_compressor")) * p return {"free_mb": round((g("pages_free") + g("pages_speculative")) * p, 1), "used_mb": round(used, 1), + # what the guest can hand out without swapping: free, speculative, inactive (file cache) and purgeable pages + "avail_mb": round((g("pages_free") + g("pages_speculative") + g("pages_inactive") + g("pages_purgeable")) * p, 1), "compressed_mb": round(g("pages_occupied_by_compressor") * p, 1), "swapins": g("swapins"), "swapouts": g("swapouts"), "pageins": g("pageins"), "pageouts": g("pageouts")} @@ -176,40 +178,62 @@ def parse_ps(text): return out -def sim_trees(procs, udids): +def own_pids(procs, me=None): + """The helper itself, its ancestors (ssh, shells) and everything it started (simfps, simctl, ps): the observer.""" + me = os.getpid() if me is None else me + by_pid = {p["pid"]: p for p in procs} + by_ppid = {} + for p in procs: + by_ppid.setdefault(p["ppid"], []).append(p["pid"]) + mine, stack = {me}, [me] + while stack: + for c in by_ppid.get(stack.pop(), []): + if c not in mine: + mine.add(c) + stack.append(c) + q = me + while q in by_pid and by_pid[q]["ppid"] > 1: + q = by_pid[q]["ppid"] + mine.add(q) + return mine + + +def sim_trees(procs, udids, exclude=()): """{udid: set of pids}: processes whose command line names the simulator's UDID (launchd_sim, SimMetalHost, - ...) and every descendant of them. A process that names no simulator is nobody's.""" + ...) and every descendant of them, never the observer (`exclude`: the helper's own tree, which names every + UDID on its command line). A process that names no simulator is nobody's.""" by_ppid = {} for p in procs: by_ppid.setdefault(p["ppid"], []).append(p["pid"]) out = {} for u in udids: - roots = [p["pid"] for p in procs if u in p["cmd"]] + roots = [p["pid"] for p in procs if u in p["cmd"] and p["pid"] not in exclude] seen, stack = set(roots), list(roots) while stack: for c in by_ppid.get(stack.pop(), []): - if c not in seen: + if c not in seen and c not in exclude: seen.add(c) stack.append(c) out[u] = seen return out -def attribute(before, after, udids, secs): +def attribute(before, after, udids, secs, exclude=()): """Per simulator and in total: CPU % of one core over the window (from the processes' CPU time deltas) and RSS MB at the end. A process that started or ended inside the window counts from its first sample or not at all: short-lived processes are undercounted (declared in the README).""" b = {p["pid"]: p for p in before} - trees = sim_trees(after, udids) + trees = sim_trees(after, udids, exclude) per = {} for u in udids: cpu = sum(max(0.0, p["cpu_s"] - b[p["pid"]]["cpu_s"]) for p in after if p["pid"] in trees[u] and p["pid"] in b) rss = sum(p["rss_kb"] for p in after if p["pid"] in trees[u]) / 1024.0 per[u] = {"cpu_pct": round(100 * cpu / secs, 1), "rss_mb": round(rss, 1), "procs": len(trees[u])} - tot = sum(max(0.0, p["cpu_s"] - b[p["pid"]]["cpu_s"]) for p in after if p["pid"] in b) + obs = sum(max(0.0, p["cpu_s"] - b[p["pid"]]["cpu_s"]) for p in after if p["pid"] in b and p["pid"] in exclude) + tot = sum(max(0.0, p["cpu_s"] - b[p["pid"]]["cpu_s"]) for p in after if p["pid"] in b and p["pid"] not in exclude) mine = set().union(*trees.values()) if trees else set() sims_cpu = sum(max(0.0, p["cpu_s"] - b[p["pid"]]["cpu_s"]) for p in after if p["pid"] in mine and p["pid"] in b) - return per, {"total_cpu_pct": round(100 * tot / secs, 1), "sims_cpu_pct": round(100 * sims_cpu / secs, 1), + return per, {"observer_cpu_pct": round(100 * obs / secs, 1), "total_cpu_pct": round(100 * tot / secs, 1), "sims_cpu_pct": round(100 * sims_cpu / secs, 1), "other_cpu_pct": round(100 * (tot - sims_cpu) / secs, 1), "sims_rss_mb": round(sum(v["rss_mb"] for v in per.values()), 1)} @@ -407,10 +431,8 @@ def cmd_ensure_type(a): os.rename(tmp, dest) res.update({"type": tid, "created": True, "needs_restart": True}) if a.allow_restart: - booted = [d for d in devices() if d.get("state") == "Booted"] - all_booted = run(["xcrun", "simctl", "list", "devices", "booted"], 30)[1] - if booted or re.search(r"\(Booted\)", all_booted): - res["restart"] = "refused: a simulator is booted" + if count_launchd_sim() or [d for d in devices() if d.get("state") == "Booted"]: + res["restart"] = "refused: a simulator is running (in any device set)" else: run(["pkill", "-f", "CoreSimulatorService"], 10) time.sleep(3) @@ -485,6 +507,7 @@ def cmd_prepare(a): if a.quiesce: simctl("spawn", u, "launchctl", "disable", "system/com.apple.mediaanalysisd", timeout=30) simctl("spawn", u, "launchctl", "bootout", "system/com.apple.mediaanalysisd", timeout=30) + simctl("ui", u, "appearance", "light", timeout=30) simctl("terminate", u, BUNDLE_ID, timeout=30) args = ["-probe"] + (["-static"] if role == "static" else []) rc_l = 0 if role in ("agent", "idle") else simctl("launch", u, BUNDLE_ID, *args, timeout=60)[0] @@ -499,14 +522,15 @@ def parse_name_cpu(text): def bg_cpu_pct(secs=3.0): """Everything not part of the load: CPU % of one core used by processes other than the renderers, the app and the observer (the settle condition: the guest has stopped doing its own housekeeping).""" - skip = re.compile(r"(SimMetalHost|backboardd|GlassBench|SimRenderServer|kernel_task|simfps|WindowServer|python3|/ps$|/top$)") + skip = re.compile(r"(^|/)(SimMetalHost|backboardd|GlassBench|SimRenderServer|kernel_task|simfps|WindowServer|python3?|Python|ps|top|ssh|sshd)( |$)") a = parse_ps(ps_text()) + mine = own_pids(a) t0 = now() time.sleep(secs) b = parse_ps(ps_text()) dt = now() - t0 am = {p["pid"]: p for p in a} - tot = sum(max(0.0, p["cpu_s"] - am[p["pid"]]["cpu_s"]) for p in b if p["pid"] in am and not skip.search(p["cmd"].split(" ", 1)[0])) + tot = sum(max(0.0, p["cpu_s"] - am[p["pid"]]["cpu_s"]) for p in b if p["pid"] in am and p["pid"] not in mine and not skip.search(p["cmd"])) return round(100 * tot / dt, 1) @@ -537,7 +561,7 @@ def run_window(a, udids, roles, mix, secs, t_go, tmp, probe_method, probe_every, inj, acts, threads = {u: [] for u in udids}, {u: [] for u in udids}, [] lvls = [] if mix in ("all-busy", "one-busy") and probe_method != "none": - targets = [u for u, r in zip(udids, roles) if mix == "all-busy" or r == "static"] + targets = list(udids) # the busy simulator too: it is what a one-busy fleet's user waits on for k, u in enumerate(targets): t = threading.Thread(target=probe_loop, args=(u, probe_method, cmd_tpl, t_go + min(1.0, secs * 0.15) + k * probe_every / max(len(targets), 1), probe_every, t_stop - probe_timeout - 0.2, inj[u], stop_evt)) @@ -565,7 +589,9 @@ def sample_pressure(): h.kill() ps1, vm1 = parse_ps(ps_text()), parse_vm_stat(vmstat_text()) t_end = now() - per, tot = attribute(ps0, ps1, udids, max(t_end - t_start, 0.001)) + if mix != "idle" and not STUB: + time.sleep(6) # GlassBench writes pace.json every 5 s: wait for the record that closes the window + per, tot = attribute(ps0, ps1, udids, max(t_end - t_start, 0.001), own_pids(ps1)) st = {d["udid"]: d.get("state") for d in devices()} sims = [] for i, u in enumerate(udids): @@ -621,6 +647,7 @@ def cmd_calibrate(a): "error": err} out_json({"methods": res}) finally: + simctl("ui", a.udid, "appearance", "light", timeout=30) # the appearance method leaves it dark shutil.rmtree(tmp, ignore_errors=True) @@ -686,8 +713,20 @@ def cmd_cycle(a): frames = analysis.parse_raw(open(raw).read()) if os.path.exists(raw) else [] first = next((x[0] for x in frames if x[0] >= t_l and x[1] >= 0 and x[1] != 65535), None) t["%s_first_frame_s" % phase] = None if first is None else round(first - t_l, 2) - t0 = now() + # clone of the shut-down, installed device (what a pool does for a new lease), then its boot simctl("shutdown", udid, timeout=120) + t0 = now() + cl = simctl("clone", udid, "%s-clone" % name, timeout=300)[1].strip().splitlines()[-1] + t["clone_s"] = round(now() - t0, 2) + try: + t0 = now() + simctl("boot", cl, timeout=300) + simctl("bootstatus", cl, timeout=600) + t["clone_boot_s"] = round(now() - t0, 2) + finally: + simctl("shutdown", cl, timeout=120) + simctl("delete", cl, timeout=120) + t0 = now() simctl("delete", udid, timeout=120) t["delete_s"] = round(now() - t0, 2) udid = None @@ -708,11 +747,20 @@ def cmd_teardown(a): out_json({"left": len(left), "booted": len(booted), "calls": [c[1:] for c in CALLS][-3:]}) +def count_launchd_sim(): + """Simulators running on this Mac in any device set (one launchd_sim each): the pool's, a lease's, the bench's.""" + return len([p for p in parse_ps(ps_text()) if re.search(r"(^|/)launchd_sim( |$)", p["cmd"])]) + + +def cmd_ips(a): + out_json({"ips": sorted(list_ips())}) + + def cmd_memcheck(a): vm = mem_summary(parse_vm_stat(vmstat_text())) - default_set = run(["xcrun", "simctl", "list", "devices", "booted"], 30)[1] # read only: the default set - out_json({"mem": vm, "pressure": pressure_level(), "booted": len([d for d in devices() if d.get("state") == "Booted"]), - "foreign_booted": len(re.findall(r"\(Booted\)", default_set)), "ips": len(list_ips())}) + own = len([d for d in devices() if d.get("state") == "Booted"]) + out_json({"mem": vm, "pressure": pressure_level(), "booted": own, "foreign_booted": max(0, count_launchd_sim() - own), + "ips": len(list_ips())}) def cmd_build(a): @@ -783,6 +831,7 @@ def add(name, fn, *opts): add("teardown", cmd_teardown) add("build", cmd_build, O("--xcodebuild", default="")) add("memcheck", cmd_memcheck) + add("ips", cmd_ips) add("calls", cmd_calls) a = ap.parse_args(argv) SET = a.set diff --git a/src/bench/guest/benchguest_stub.py b/src/bench/guest/benchguest_stub.py index 3a7ca34..a2ddee1 100644 --- a/src/bench/guest/benchguest_stub.py +++ b/src/bench/guest/benchguest_stub.py @@ -93,6 +93,10 @@ def _stub_run(argv): u = str(uuid.uuid4()).upper() s["devs"][u] = {"name": cmd[1], "state": "Shutdown", "type": cmd[2]} out = u + "\n" + elif verb == "clone": + u = str(uuid.uuid4()).upper() + s["devs"][u] = {"name": cmd[2], "state": "Shutdown", "type": s["devs"][cmd[1]].get("type")} + out = u + "\n" elif verb == "boot": time.sleep(0.05) s["devs"][cmd[1]]["state"] = "Booted" @@ -239,5 +243,8 @@ def ips(): s = load() calls[0] += 1 cr = fault("crash") - return set(s["ips"]) | ({"GlassBench-stub.ips"} if cr is not None and len(booted(s)) >= cr and calls[0] >= 2 else set()) + if cr is not None and len(booted(s)) >= cr and calls[0] >= 2 and "GlassBench-stub.ips" not in s["ips"]: + s["ips"].append("GlassBench-stub.ips") # stays on the "disk": the run-wide gate sees it later + save(s) + return set(s["ips"]) mod.list_ips = ips diff --git a/src/bench/tests/test_modules.py b/src/bench/tests/test_modules.py index df74c82..dba90fa 100644 --- a/src/bench/tests/test_modules.py +++ b/src/bench/tests/test_modules.py @@ -48,6 +48,28 @@ def test_vm_stat(self): self.assertAlmostEqual(m["compressed_mb"], 52949 * 16384 / 1048576.0, places=1) self.assertEqual(bg.parse_vm_stat(VM_STAT)["page"], 16384) + def test_available_memory_counts_cache_and_defaults_to_4k_pages(self): + m = bg.mem_summary(bg.parse_vm_stat(VM_STAT)) + self.assertAlmostEqual(m["avail_mb"], (573102 + 10000 + 100000) * 16384 / 1048576.0, places=1) # + inactive + self.assertGreater(m["avail_mb"], m["free_mb"]) + nohead = bg.parse_vm_stat(VM_STAT.replace("(page size of 16384 bytes)", "")) + if os.uname()[4] == "x86_64": + # without the header (x86 guest): 573102 free pages are 2.2 GiB at 4 KiB, not 8.7 GiB + self.assertEqual(nohead["page"], 4096) + self.assertAlmostEqual(573102 * 4096 / 1073741824.0, 2.19, places=2) + + def test_observer_is_nobodys_simulator_tree(self): + procs = bg.parse_ps(PS + " 500 1 2000 0:03.00 xcrun simctl --set /x AAAA-1 BBBB-2\n 501 500 2000 0:09.00 simfps AAAA-1\n") + own = bg.own_pids(procs, me=501) + self.assertEqual(own, {500, 501}) + trees = bg.sim_trees(procs, ["AAAA-1", "BBBB-2"], own) + self.assertNotIn(500, trees["AAAA-1"] | trees["BBBB-2"]) + self.assertNotIn(501, trees["AAAA-1"]) + later = [dict(p, cpu_s=p["cpu_s"] + 1.0) for p in procs] + per, tot = bg.attribute(procs, later, ["AAAA-1", "BBBB-2"], 2.0, own) + self.assertAlmostEqual(tot["observer_cpu_pct"], 100.0, places=1) # two processes x 1 s over 2 s + self.assertEqual(per["AAAA-1"]["procs"], 3) + def test_cputime(self): self.assertAlmostEqual(bg.parse_cputime("0:01.50"), 1.5) self.assertAlmostEqual(bg.parse_cputime("2:03.25"), 123.25) From 26fa95159787db429e3811623afe8973d55d0d12 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:49:56 +0200 Subject: [PATCH 09/23] bench: run-wide gates for every scenario, a sweep without a step fails, settle recorded, census dir from QEMU, cgroup-only swap gate --- src/bench/macbench/cli.py | 8 +++-- src/bench/macbench/core.py | 54 +++++++++++++++++++++++++++------ src/bench/macbench/hostprobe.py | 9 ++++++ src/bench/macbench/scale.py | 21 ++++++++----- src/bench/tests/test_dry.py | 25 ++++++++++++++- 5 files changed, 97 insertions(+), 20 deletions(-) diff --git a/src/bench/macbench/cli.py b/src/bench/macbench/cli.py index c03683a..cbbd3b2 100644 --- a/src/bench/macbench/cli.py +++ b/src/bench/macbench/cli.py @@ -177,7 +177,8 @@ def build_ctx(a, kind): live = os.path.join(runp, "live.conf") if runp and os.path.exists(os.path.join(runp, "live.conf")) else None host = hostprobe.Host(pid, live_conf=live) from .gates import Coherent - coh = Coherent(os.path.join(runp, "coherent") if runp else None, os.environ.get("BENCH_QEMU_LOG"), + census = hostprobe.qemu_env_value(pid, "REIMS_VGPU_CENSUS") or (os.path.join(runp, "coherent") if runp else None) + coh = Coherent(census, os.environ.get("BENCH_QEMU_LOG"), os.path.join(os.environ.get("BENCH_HOME", ""), "home/.cache/m2v-gpu/hang") if os.environ.get("BENCH_HOME") else None) guest = SshGuest(run, os.environ.get("BENCH_SSH") or os.environ.get("MACBOX_SSH_HOST") or "macbox", set_name) os.makedirs(a.out, exist_ok=True) @@ -206,7 +207,10 @@ def execute(a, kind, body): log("DRY=1: stub guest, fake host; nothing real is touched") rc = 0 try: - body(ctx) + try: + body(ctx) + finally: + core.run_gates(ctx) # every scenario: Coherent, kernel GPU lines and crash reports since the run began except core.Stop as e: ctx.section("stopped", "FAIL", "STOPPED: %s" % e) rc = 2 diff --git a/src/bench/macbench/core.py b/src/bench/macbench/core.py index 7c3de1f..dff5fac 100644 --- a/src/bench/macbench/core.py +++ b/src/bench/macbench/core.py @@ -53,10 +53,11 @@ def guard(ctx, k_new, label): if df is not None and df < a.min_disk_gib: return "free disk %.0f GiB below the %.0f GiB floor %s" % (df, a.min_disk_gib, label) m = ctx.guest.call(["memcheck"], 60) - free_gib = (m["mem"]["free_mb"] - k_new * ctx.est_sim_mb) / 1024.0 + avail = m["mem"].get("avail_mb", m["mem"]["free_mb"]) # free + speculative + inactive + purgeable: what a boot can take + free_gib = (avail - k_new * ctx.est_sim_mb) / 1024.0 if free_gib < a.min_guest_free_gib: - return "guest free memory would be %.1f GiB (%.1f now, %d x ~%d MB to boot), below the %.1f GiB floor %s" % ( - free_gib, m["mem"]["free_mb"] / 1024.0, k_new, ctx.est_sim_mb, a.min_guest_free_gib, label) + return "guest available memory would be %.1f GiB (%.1f now, %d x ~%d MB to boot), below the %.1f GiB floor %s" % ( + free_gib, avail / 1024.0, k_new, ctx.est_sim_mb, a.min_guest_free_gib, label) if (m.get("pressure") or 1) >= 2: return "guest memory pressure level %s %s" % (m.get("pressure"), label) if m.get("foreign_booted", 0) > 0 and not a.allow_foreign: @@ -92,6 +93,7 @@ def setup_guest(ctx): ctx.doc["env"]["guest"]["runtime_used"] = t.get("runtime_version") ctx.doc["env"]["guest"]["device_type"] = "stock %s" % t["stock_name"] if a.scale_x == 3 else "macbox %dx of %s" % (a.scale_x, t["stock_name"]) ctx.offset = ctx.guest.clock_offset() + ctx.run_mark = {"coh": ctx.coh.mark(), "t0": time.time(), "ips": set(ctx.guest.call(["ips"], 60)["ips"])} return info @@ -166,7 +168,7 @@ def system_dict(ctx, g, s0, s1, frames_total): "gpu_busy_pct": None if gpu_ns is None else round(100.0 * gpu_ns / 1e9 / dt, 1), "gpu_ms_per_frame": None if gpu_ns is None or not frames_total else round(gpu_ns / 1e6 / frames_total, 2), "window_s": round(dt, 2), - "guest": {"mem_used_mb": gu["mem"]["used_mb"], "mem_free_mb": gu["mem"]["free_mb"], "compressed_mb": gu["mem"]["compressed_mb"], + "guest": {"mem_used_mb": gu["mem"]["used_mb"], "mem_free_mb": gu["mem"]["free_mb"], "mem_avail_mb": gu["mem"].get("avail_mb"), "observer_cpu_pct": gu.get("observer_cpu_pct"), "compressed_mb": gu["mem"]["compressed_mb"], "swapins_d": gu["swapins_d"], "swapouts_d": gu["swapouts_d"], "pageouts_d": gu["pageouts_d"], "pressure_max": gu["pressure_max"], "total_cpu_pct": gu["total_cpu_pct"], "sims_cpu_pct": gu["sims_cpu_pct"], "other_cpu_pct": gu["other_cpu_pct"], "sims_rss_mb": gu["sims_rss_mb"]}, @@ -227,8 +229,10 @@ def step_gates(ctx, step, shots=None): gs.update(shots) swapped = step["system"]["guest"]["swapouts_d"] or 0 hs = step["system"]["host"] - host_swap = (hs.get("cgroup_swap_delta_mb") or 0) > 50 or (hs.get("pswpout_pages") or 0) > 25600 - gs["memory"] = gates.gate("FAIL" if swapped > 0 and ctx.a.fail_on_swap or host_swap else "PASS", + cg = hs.get("cgroup_swap_delta_mb") # the VM's own cgroup: other projects' swapping on the host is not this run's + host_swap = cg is not None and cg > 50 + guest_swap = swapped > 0 and ctx.a.fail_on_swap + gs["memory"] = gates.gate("FAIL" if guest_swap or host_swap else ("PASS" if cg is not None else "SKIP"), "guest swapouts +%s, pressure max %s; host cgroup swap %s MB, pswpout %s pages" % ( swapped, step["system"]["guest"]["pressure_max"], hs.get("cgroup_swap_delta_mb"), hs.get("pswpout_pages"))) step["gates"] = gs @@ -237,11 +241,43 @@ def step_gates(ctx, step, shots=None): return gs -def take_shots(ctx, count=2): - """Screenshots of every simulator, through the guest, judged by the screenshot gates.""" +def run_gates(ctx): + """The gates that hold for a whole run, whatever the scenario: Coherent and QEMU's log since the run began, kernel + GPU error lines since then, and crash reports that appeared on the guest. A scenario that stopped before its setup + has nothing to judge. Never raises: a gate that cannot be read is SKIP.""" + mk = getattr(ctx, "run_mark", None) + if not mk: + return + gs = {} + try: + for k, v in ctx.coh.check(mk["coh"]).items(): + gs["run_" + k] = v + except Exception as e: + gs["run_coherent"] = gates.gate("SKIP", "unreadable: %s" % type(e).__name__) + try: + kern, note = ctx.host.kernel_gpu(mk["t0"]) + gs["run_kernel_gpu"] = gates.kernel_gate(kern, note) + except Exception as e: + gs["run_kernel_gpu"] = gates.gate("SKIP", "unreadable: %s" % type(e).__name__) + try: + now = set(ctx.guest.call(["ips"], 60)["ips"]) + new = sorted(now - mk["ips"]) + gs["run_crash_reports"] = gates.gate("FAIL" if new else "PASS", ("%d new crash report(s): %s" % (len(new), ", ".join(new[:3]))) if new else "no new crash report on the guest") + except Exception as e: + gs["run_crash_reports"] = gates.gate("SKIP", "guest unreachable: %s" % type(e).__name__) + ctx.doc.setdefault("gates", {}).update(gs) + for name, v in gs.items(): + ctx.section("gate:" + name, v["status"], "GATE %s %s: %s" % (v["status"], name, v["detail"])) + + +def take_shots(ctx, count=2, judge_static=False): + """Screenshots of every simulator, through the guest, judged by the screenshot gates. Static screens are judged + for pixels only when `judge_static`: after a window with input probes their marker state is not the golden's + (the pixel gate runs on a freshly launched static app, see scale.golden_phase).""" udids = ",".join(s["udid"] for s in ctx.sims) r = ctx.guest.call(["shots", "--udids", udids, "--count", count], 60 + 30 * len(ctx.sims)) by = [] for s, one in zip(ctx.sims, r["sims"]): - by.append({"idx": s["idx"], "role": s["role"], "shots": one["shots"]}) + role = s["role"] if (s["role"] != "static" or judge_static) else "static-unjudged" + by.append({"idx": s["idx"], "role": role, "shots": one["shots"]}) return gates.shots_gates(by, ctx.golden), by diff --git a/src/bench/macbench/hostprobe.py b/src/bench/macbench/hostprobe.py index 34ac23f..21920f0 100644 --- a/src/bench/macbench/hostprobe.py +++ b/src/bench/macbench/hostprobe.py @@ -209,6 +209,15 @@ def qemu_env_switches(pid, root=""): return out +def qemu_env_value(pid, key, root=""): + """The raw value of one environment variable of the QEMU process (None when unreadable or unset).""" + t = _read(root, "/proc/%d/environ" % pid) or "" + for kv in t.split("\0"): + if kv.startswith(key + "="): + return kv.split("=", 1)[1] + return None + + def mesa_is_custom(pid, root=""): """True when QEMU maps an ANV driver from outside the system library directories (its own Mesa build).""" t = _read(root, "/proc/%d/maps" % pid) or "" diff --git a/src/bench/macbench/scale.py b/src/bench/macbench/scale.py index ec6a6af..9e09497 100644 --- a/src/bench/macbench/scale.py +++ b/src/bench/macbench/scale.py @@ -69,12 +69,13 @@ def run_scale(ctx): if a.golden_phase: ctx.pixel_gate = golden_phase(ctx) core.grow(ctx, n, roles) - core.settle(ctx) + st = core.settle(ctx) g, s0, s1, t_go = core.window(ctx, a.secs, mix) step = core.build_step(ctx, n, mix, g, s0, s1, t_go, boot) + step["settle"] = {k: st.get(k) for k in ("settled", "bg_pct", "waited_s")} shots, by = core.take_shots(ctx, a.shots) if a.shots and mix != "idle" else ({}, []) - if first and ctx.pixel_gate and "pixels" not in shots: - shots["pixels"] = ctx.pixel_gate + if first and ctx.pixel_gate: + shots["pixels"] = ctx.pixel_gate # judged on the freshly launched static app, not after input probes core.step_gates(ctx, step, shots) step["window"] = {"start_offset_s": round(t_go - ctx.marks["t0"] - ctx.offset, 1)} data["steps"].append(step) @@ -145,6 +146,7 @@ def _finish(ctx, stopped, partial=False): steps = data["steps"] if not steps and not stopped: return None + data["no_steps"] = not steps data["summary"] = analysis.summarize_scale(steps, a.targets, stopped) if steps else {"max_n": {}, "stopped": stopped, "standing_cost": {}} doc["data"]["summary"]["stopped"] = stopped doc["gates"] = _gate_rollup(steps, ctx) @@ -168,7 +170,8 @@ def _gate_rollup(steps, ctx): elif all(g["status"] == "SKIP" for _, g in rows): roll[name] = gates.gate("SKIP", rows[-1][1]["detail"]) else: - roll[name] = gates.gate("PASS", "%d step(s); %s" % (len(rows), rows[-1][1]["detail"])) + judged = [g for _, g in rows if g["status"] != "SKIP"] + roll[name] = gates.gate("PASS", "%d of %d step(s) judged; %s" % (len(judged), len(rows), judged[-1]["detail"])) return roll @@ -179,10 +182,12 @@ def _sections(ctx, data, stopped): gate_fail = [k for k, v in ctx.doc["gates"].items() if v["status"] == "FAIL"] unstable = any(not s["stable"] for s in steps) mx = ", ".join("%s=%s" % (k, v["max_n"] if v["max_n"] is not None else "none") for k, v in sm["max_n"].items()) - status = "FAIL" if gate_fail or unstable or data.get("stopped_kind") == "error" else "PASS" - ctx.section("scale", status, "SCALE %s %s %dx %s: max sustainable N: %s | steps %s%s | min Jain %s" % ( - status, a.mix, a.scale_x, "(DRY)" if ctx.host.dry else "", mx, [s["n"] for s in steps], - (" | stopped: %s" % stopped) if stopped else " | all requested N done", sm.get("fairness_min_jain"))) + status = "FAIL" if gate_fail or unstable or not steps or data.get("stopped_kind") == "error" else "PASS" + unsettled = [s["n"] for s in steps if (s.get("settle") or {}).get("settled") is False] + ctx.section("scale", status, "SCALE %s %s %dx %s: max sustainable N: %s | steps %s%s%s | min Jain %s" % ( + status, a.mix, a.scale_x, "(DRY)" if ctx.host.dry else "", mx or "none (no step ran)", [s["n"] for s in steps], + (" | stopped: %s" % stopped) if stopped else " | all requested N done", + (" | NOT SETTLED at N=%s (guest background CPU stayed above %g%%)" % (unsettled, a.settle_bg)) if unsettled else "", sm.get("fairness_min_jain"))) for name, v in ctx.doc["gates"].items(): ctx.section("gate:" + name, v["status"] if v["status"] != "PASS" else "PASS", "GATE %s %s: %s" % (v["status"], name, v["detail"])) st = [s for s in steps] diff --git a/src/bench/tests/test_dry.py b/src/bench/tests/test_dry.py index 7fdecda..745f73c 100644 --- a/src/bench/tests/test_dry.py +++ b/src/bench/tests/test_dry.py @@ -60,6 +60,7 @@ def sections(doc, name): CASES = { "clean": (["scale", "--n", "1,2,3"] + FAST, ""), "memfloor": (["scale", "--n", "1,2,3,4,5"] + FAST, "memfloor@3"), + "refuse": (["scale", "--n", "1,2"] + FAST, "memfloor@0"), "hostmem": (["scale", "--n", "1,2,3,4,5"] + FAST, "hostmem@3"), "crash": (["scale", "--n", "1,2,3,4"] + FAST, "crash@2"), "backwards": (["scale", "--n", "1,2,3"] + FAST, "backwards@2"), @@ -116,11 +117,33 @@ def test_memory_floor_stops_the_sweep_and_keeps_the_data(self): r = self.r["memfloor"] d = r["doc"] self.assertEqual([s["n"] for s in d["data"]["steps"]], [1, 2, 3]) # the guard looks before booting: it refuses the 4th - self.assertIn("guest free memory", d["data"]["summary"]["stopped"]) + self.assertIn("guest available memory", d["data"]["summary"]["stopped"]) self.assertEqual(sims_booted(r["state"]), []) self.assertEqual(self.r["hostmem"]["doc"]["data"]["steps"][-1]["n"], 3) self.assertIn("host free memory", self.r["hostmem"]["doc"]["data"]["summary"]["stopped"]) + def test_a_sweep_that_ran_no_step_fails(self): + r = self.r["refuse"] + self.assertEqual(r["doc"]["data"]["steps"], []) + self.assertNotEqual(r["rc"], 0) + self.assertEqual(sections(r["doc"], "scale")[0]["status"], "FAIL") + self.assertIn("no step ran", sections(r["doc"], "scale")[0]["line"]) + self.assertEqual(sims_booted(r["state"]), []) + + def test_steps_record_whether_the_guest_settled(self): + for st in self.r["clean"]["doc"]["data"]["steps"]: + self.assertIn("settled", st["settle"]) + self.assertIn("waited_s", st["settle"]) + + def test_run_wide_gates_catch_what_a_step_gate_does_not_name(self): + d = self.r["crash"]["doc"] + self.assertEqual(d["gates"]["run_crash_reports"]["status"], "FAIL") + self.assertEqual(sections(d, "gate:run_crash_reports")[0]["status"], "FAIL") + for k in ("clean", "lifecycle", "idle"): + g = self.r[k]["doc"]["gates"] + self.assertEqual(g["run_crash_reports"]["status"], "PASS", k) + self.assertIn("run_kernel_gpu", g) + def test_crash_report_is_instability_and_ends_the_sweep(self): d = self.r["crash"]["doc"] self.assertEqual(d["data"]["steps"][-1]["n"], 2) From ce068da5509ebc2352801efef6be617ed3eef459 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:49:56 +0200 Subject: [PATCH 10/23] bench: README: guard, settle, observer, run gates --- src/bench/README.md | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/src/bench/README.md b/src/bench/README.md index 213a03a..769c2aa 100644 --- a/src/bench/README.md +++ b/src/bench/README.md @@ -32,11 +32,16 @@ switch file (restored byte for byte on every way out). Safety, in every run: the **hard deadline** (`--deadline`, default 90 minutes) stops the run 60 s early, writes the result so far and runs the cleanup; **one exit trap** (normal end, error, SIGTERM/SIGINT, the deadline) kills the helpers it started (explicit pids, never a bare `wait`), shuts down and deletes the bench's simulators and -restores every knob; a **guard** refuses to boot more simulators when the guest's free memory would fall under +restores every knob; a **guard** refuses to boot more simulators when the guest's available memory (free + +speculative + inactive + purgeable pages) would fall under `--min-guest-free-gib` (default 1.5, after the estimated cost of the new simulators), the guest reports memory pressure level 2 or more, the host's available memory is under `--min-host-free-gib` (6), or the disk under -`--min-disk-gib` (12): the sweep stops there, it does not swap the VM to death. Simulators booted outside the bench -refuse the run (`--allow-foreign` to measure anyway, recorded). +`--min-disk-gib` (12): the sweep stops there, it does not swap the VM to death. A sweep that ran no step at all +(a refused guard, foreign simulators) is a FAIL with a non-zero exit. Simulators running outside the bench (a +`launchd_sim` that is not the bench's own) refuse the run (`--allow-foreign` to measure anyway, recorded). Before each +window the guest must settle: `settle` waits until the processes outside the simulators and the helper use less than +`--settle-bg` % of a core, for at most `--settle-max` s; the step records `settle.settled`, `bg_pct` and `waited_s`, +and the SCALE line names the N whose guest did not settle. ## Commands @@ -110,11 +115,13 @@ window's start and end: the CPU time of every process in the simulator's tree ov (CPU time has a 10 ms resolution; a process that started or ended inside the window is counted only if it was in both snapshots, so very short-lived processes are undercounted), and the RSS of the tree at the end. The tree is every process whose command line names the simulator's UDID (`launchd_sim`, the per-device helpers) and -all their descendants. RSS counts shared pages in each process: it overstates what leaving a simulator out would +all their descendants, never the bench's own helper, its ssh and shell ancestors, or the `simfps`/`simctl`/`ps` it +started: their CPU is `system.guest.observer_cpu_pct`. RSS counts shared pages in each process: it overstates what leaving a simulator out would free. `system.guest`: `total_cpu_pct`, `sims_cpu_pct` (the trees), `other_cpu_pct` (everything else), `sims_rss_mb`. **Guest memory** (`system.guest`): `mem_used_mb` = active + wired + compressor pages; `mem_free_mb` = free + -speculative; `compressed_mb`; `swapins_d`, `swapouts_d`, `pageouts_d` = the counters' change over the window; +speculative; `mem_avail_mb` is what the guard uses: free + speculative + inactive + purgeable (`vm_stat` pages; 4 KiB +when the header does not say, 16 KiB on arm64); `compressed_mb`; `swapins_d`, `swapouts_d`, `pageouts_d` = the counters' change over the window; `pressure_max` = the highest `kern.memorystatus_vm_pressure_level` sampled every 2 s (1 normal, 2 warning, 4 critical). @@ -157,12 +164,13 @@ written first. A gate that cannot be read is SKIP, shown as such, never PASS. |---|---| | `strip` | no frame out of order, no counter jump, the counter advanced no more than the app's own tick log says, under 1% of samples unreadable. Lost ticks are the fps metric, not a gate failure. | | `tears` | no screenshot of the scrolling GlassBench shows a torn band (card spacing and gap heights, the old `tears.py` rule, standard library PNG decoding); SKIP when no screenshot shows cards | -| `pixels` | GlassBench `-static` screenshots agree with each other and match the stored golden (`baselines/golden.json`, key: macOS, runtime, device type, size); SKIP with the pictures agreeing when no golden is stored: `macbox bench golden` records it | +| `pixels` | GlassBench `-static` screenshots agree with each other and match the stored golden (`baselines/golden.json`, key: macOS, runtime, device type, size), judged once on a freshly launched static app before the first window (a window with input probes flips the marker, so later static screens are not judged); SKIP with the pictures agreeing when no golden is stored: `macbox bench golden` records it | | `coherent_census` / `coherent_process` | in the census reports written since the run began: `overwrite_risk` and `sync_errors` stayed 0, and `late_copies_in_busy`, `late_beside_unplanned`, `plan_left_planned` did not rise. Deltas, not absolute counts. Reports come once a minute, so a short step may have none: SKIP. | | `qemu_log` | no new `command buffer failed` line in QEMU's log (stamp waits are counted in the detail) | | `gpu_hang` | the hang flag did not appear | | `kernel_gpu` | no kernel GPU error line since the step began (SKIP when the journal is not readable) | -| `memory` | no host swapping (cgroup swap +50 MB, or 100 MB of system swap-outs); guest swap-outs fail it with `--fail-on-swap` | +| `memory` | the VM's own cgroup swapped no more than 50 MB (SKIP when the cgroup is unreadable: other projects' swapping on the host is not this run's); guest swap-outs fail it with `--fail-on-swap` | +| `run_*` | every scenario, after its body: `run_coherent_census`, `run_coherent_process`, `run_qemu_log`, `run_gpu_hang` and `run_kernel_gpu` since the run began, and `run_crash_reports` (a new `.ips` on the guest since setup). They fail the exit code of `idle`, `lifecycle`, `soak`, `ab` too. | | `leak_*` | `lifecycle`: QEMU RSS and guest memory grow less than `--leak-rss-mb` / `--leak-guest-mb` per create/boot/delete cycle (least-squares slope); `soak`: less than `--soak-rss-mb-min` / `--soak-guest-mb-min` per minute | | `observer` | reading the strip does not move the surface frame rate by more than 4% (a second window of the first N without the strip read); otherwise the distinct-frame numbers are lower bounds | @@ -210,7 +218,7 @@ IPv4 addresses to ``), and `publishable` says whether anything slipped throu - The observer is not free: `simfps` polls at 2 kHz and `--strip` reads each new surface with the CPU, which also touches Coherent's surfaces. `observer` measures that at the first N; the guest CPU of the simulators includes it - only for the helper's own process, which is not in a simulator's tree. + only for the helper's own tree, which is excluded from the simulators' trees and reported as `observer_cpu_pct`. - Probes run `simctl` in the guest every 2.5 s per simulator, a small steady load that agents also make. - Windows are 40 s: slow drifts need `soak`; a stall rarer than once per window is invisible. - GPU numbers need DRM fdinfo of QEMU's process to be readable by the user running the bench. From f852dc734392dec7a8982e8c4ce17de090bc8758 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Mon, 16 Nov 2026 21:58:49 +0200 Subject: [PATCH 11/23] bench: observer CPU counts reaped helpers, census dir checked on the host, appearance reset, README lifecycle timings, more tests --- src/bench/README.md | 19 ++++++++++++++----- src/bench/guest/benchguest.py | 16 +++++++++++++++- src/bench/macbench/cli.py | 4 +++- src/bench/macbench/others.py | 4 ++-- src/bench/tests/test_modules.py | 24 ++++++++++++++++++++++++ src/bench/tests/test_runctx.py | 26 ++++++++++++++++++++++++++ 6 files changed, 84 insertions(+), 9 deletions(-) diff --git a/src/bench/README.md b/src/bench/README.md index 769c2aa..da461aa 100644 --- a/src/bench/README.md +++ b/src/bench/README.md @@ -39,7 +39,7 @@ pressure level 2 or more, the host's available memory is under `--min-host-free- `--min-disk-gib` (12): the sweep stops there, it does not swap the VM to death. A sweep that ran no step at all (a refused guard, foreign simulators) is a FAIL with a non-zero exit. Simulators running outside the bench (a `launchd_sim` that is not the bench's own) refuse the run (`--allow-foreign` to measure anyway, recorded). Before each -window the guest must settle: `settle` waits until the processes outside the simulators and the helper use less than +window the guest must settle: `settle` waits until every process except the load and its observers (the renderers `SimMetalHost`/`backboardd`/`SimRenderServer`, GlassBench, `simfps`, WindowServer, python, ps, ssh, matched by name against the whole command, and the helper's own tree) uses less than `--settle-bg` % of a core, for at most `--settle-max` s; the step records `settle.settled`, `bg_pct` and `waited_s`, and the SCALE line names the N whose guest did not settle. @@ -49,7 +49,7 @@ and the SCALE line names the N whose guest did not settle. |---|---| | `scale` | N = 1, 2, 3... simulators (`--n 1,2,3,4,6,8`, `--scale-x 1\|3`, `--mix`), per simulator numbers, the maximum sustainable N per quality target. Stops early when a floor is breached (`--floor`, default `fps>=20`), a gate fails or the system turns unstable. | | `idle` | The VM with no simulator: QEMU, GPU and memory standing cost against `--bar-qemu-cores` (0.15). | -| `lifecycle` | `--cycles` times: create, boot, install, launch, first app frame, shut down, resume, first frame, delete (medians); a per-cycle leak gate; `--storm N`: N simulators booted at once. | +| `lifecycle` | `--cycles` times: create, boot, install, launch, first app frame, shut down, resume, first frame, clone, boot the clone, delete (medians of the timings below); a per-cycle leak gate; `--storm N`: N simulators booted at once. | | `soak` | `--soak-n` busy simulators for `--soak-minutes`: slope of QEMU's RSS and of the guest's memory, and the frame rate at the start against the end. | | `build` | Build timing: the bench tools (swiftc) cold and warm; `--xcodebuild ''` times your own project cold and warm (GlassBench is not an Xcode project, so the section is SKIP without it). | | `panel` | The Device panel's stream: pluggable (see [The panel section](#the-panel-section)); SKIP without `--panel-cmd`. | @@ -116,9 +116,17 @@ window's start and end: the CPU time of every process in the simulator's tree ov was in both snapshots, so very short-lived processes are undercounted), and the RSS of the tree at the end. The tree is every process whose command line names the simulator's UDID (`launchd_sim`, the per-device helpers) and all their descendants, never the bench's own helper, its ssh and shell ancestors, or the `simfps`/`simctl`/`ps` it -started: their CPU is `system.guest.observer_cpu_pct`. RSS counts shared pages in each process: it overstates what leaving a simulator out would +started: their CPU is `system.guest.observer_cpu_pct` (the larger of the helper tree's CPU between the two `ps` snapshots and the CPU time of the helper's reaped children, `simfps` and the probes' `simctl`, over the window). RSS counts shared pages in each process: it overstates what leaving a simulator out would free. `system.guest`: `total_cpu_pct`, `sims_cpu_pct` (the trees), `other_cpu_pct` (everything else), `sims_rss_mb`. +**Lifecycle timings** (`lifecycle`, seconds, medians over `--cycles`; each cycle uses a fresh device): `create_s` +(`simctl create`); `cold_boot_s` (`boot` + `bootstatus`); `install_s`; `cold_launch_call_s` (the `launch` call); +`cold_first_frame_s` (from the launch call to the first counter value `simfps` reads on the new surface); `shutdown_s` +(shutting the booted device down), `resume_boot_s` and `resume_first_frame_s` (the same two again: "resume" here is +shutdown then boot, not Simlock's parking); `clone_s` (`simctl clone` of the shut-down device that has the app +installed), `clone_boot_s` (boot of the clone until `bootstatus` returns); `delete_s` (`simctl delete` of the source +device only: the shutdown before the clone is not timed). The clone is deleted untimed. + **Guest memory** (`system.guest`): `mem_used_mb` = active + wired + compressor pages; `mem_free_mb` = free + speculative; `mem_avail_mb` is what the guard uses: free + speculative + inactive + purgeable (`vm_stat` pages; 4 KiB when the header does not say, 16 KiB on arm64); `compressed_mb`; `swapins_d`, `swapouts_d`, `pageouts_d` = the counters' change over the window; @@ -217,8 +225,9 @@ IPv4 addresses to ``), and `publishable` says whether anything slipped throu ## Limits of the measurements - The observer is not free: `simfps` polls at 2 kHz and `--strip` reads each new surface with the CPU, which also - touches Coherent's surfaces. `observer` measures that at the first N; the guest CPU of the simulators includes it - only for the helper's own tree, which is excluded from the simulators' trees and reported as `observer_cpu_pct`. + touches Coherent's surfaces. `observer` measures that at the first N; the simulators' guest CPU does not include it + (the helper's own tree is excluded from every simulator's tree and reported as `observer_cpu_pct`), but the + observer still takes CPU the simulators would otherwise have. - Probes run `simctl` in the guest every 2.5 s per simulator, a small steady load that agents also make. - Windows are 40 s: slow drifts need `soak`; a stall rarer than once per window is invisible. - GPU numbers need DRM fdinfo of QEMU's process to be readable by the user running the bench. diff --git a/src/bench/guest/benchguest.py b/src/bench/guest/benchguest.py index d0434ee..73bcd17 100644 --- a/src/bench/guest/benchguest.py +++ b/src/bench/guest/benchguest.py @@ -14,6 +14,7 @@ import os import plistlib import re +import resource import shutil import subprocess import sys @@ -519,10 +520,14 @@ def parse_name_cpu(text): return text +# the load itself and its observers, matched against the WHOLE command (a runtime path contains spaces) +SETTLE_SKIP = re.compile(r"(^|/)(SimMetalHost|backboardd|GlassBench|SimRenderServer|kernel_task|simfps|WindowServer|python3?|Python|ps|top|ssh|sshd)( |$)") + + def bg_cpu_pct(secs=3.0): """Everything not part of the load: CPU % of one core used by processes other than the renderers, the app and the observer (the settle condition: the guest has stopped doing its own housekeeping).""" - skip = re.compile(r"(^|/)(SimMetalHost|backboardd|GlassBench|SimRenderServer|kernel_task|simfps|WindowServer|python3?|Python|ps|top|ssh|sshd)( |$)") + skip = SETTLE_SKIP a = parse_ps(ps_text()) mine = own_pids(a) t0 = now() @@ -557,6 +562,7 @@ def run_window(a, udids, roles, mix, secs, t_go, tmp, probe_method, probe_every, time.sleep(d) ps0, vm0 = parse_ps(ps_text()), parse_vm_stat(vmstat_text()) t_start = now() + ru0 = resource.getrusage(resource.RUSAGE_CHILDREN) t_stop = t_go + secs inj, acts, threads = {u: [] for u in udids}, {u: [] for u in udids}, [] lvls = [] @@ -587,11 +593,19 @@ def sample_pressure(): [t.join(30) for t in threads] for h in handles.values(): h.kill() + if probe_method == "appearance" and mix in ("all-busy", "one-busy"): + for u in udids: # the appearance probe leaves a simulator dark: fps and tears are judged in light + simctl("ui", u, "appearance", "light", timeout=30) ps1, vm1 = parse_ps(ps_text()), parse_vm_stat(vmstat_text()) t_end = now() + ru1 = resource.getrusage(resource.RUSAGE_CHILDREN) if mix != "idle" and not STUB: time.sleep(6) # GlassBench writes pace.json every 5 s: wait for the record that closes the window per, tot = attribute(ps0, ps1, udids, max(t_end - t_start, 0.001), own_pids(ps1)) + # simfps and the probes' simctl have exited (and been reaped) by now, so the two ps snapshots cannot see them: the + # CPU time of the helper's reaped children over the window is the observer's real cost + reaped = (ru1.ru_utime + ru1.ru_stime) - (ru0.ru_utime + ru0.ru_stime) + tot["observer_cpu_pct"] = round(max(tot["observer_cpu_pct"], 100 * reaped / max(t_end - t_start, 0.001)), 1) st = {d["udid"]: d.get("state") for d in devices()} sims = [] for i, u in enumerate(udids): diff --git a/src/bench/macbench/cli.py b/src/bench/macbench/cli.py index cbbd3b2..3a4c0f6 100644 --- a/src/bench/macbench/cli.py +++ b/src/bench/macbench/cli.py @@ -177,7 +177,9 @@ def build_ctx(a, kind): live = os.path.join(runp, "live.conf") if runp and os.path.exists(os.path.join(runp, "live.conf")) else None host = hostprobe.Host(pid, live_conf=live) from .gates import Coherent - census = hostprobe.qemu_env_value(pid, "REIMS_VGPU_CENSUS") or (os.path.join(runp, "coherent") if runp else None) + census = hostprobe.qemu_env_value(pid, "REIMS_VGPU_CENSUS") # the path QEMU was given, if it exists here too + if not (census and os.path.isdir(census)): + census = os.path.join(runp, "coherent") if runp else None coh = Coherent(census, os.environ.get("BENCH_QEMU_LOG"), os.path.join(os.environ.get("BENCH_HOME", ""), "home/.cache/m2v-gpu/hang") if os.environ.get("BENCH_HOME") else None) guest = SshGuest(run, os.environ.get("BENCH_SSH") or os.environ.get("MACBOX_SSH_HOST") or "macbox", set_name) diff --git a/src/bench/macbench/others.py b/src/bench/macbench/others.py index 2efba23..61b09f7 100644 --- a/src/bench/macbench/others.py +++ b/src/bench/macbench/others.py @@ -65,9 +65,9 @@ def run_lifecycle(ctx): med = {k: _median([c[k] for c in cyc if c.get(k) is not None]) for k in keys} data["median_s"] = med ok = all(v is not None for v in med.values()) and bool(cyc) - ctx.section("lifecycle", "PASS" if ok else "FAIL", "LIFECYCLE %s (%d cycles, medians, s): create %s, cold boot %s, install %s, launch call %s, first app frame %s, shutdown %s, resume boot %s, first frame after resume %s, delete %s" % ( + ctx.section("lifecycle", "PASS" if ok else "FAIL", "LIFECYCLE %s (%d cycles, medians, s): create %s, cold boot %s, install %s, launch call %s, first app frame %s, shutdown %s, resume boot %s, first frame after resume %s, clone %s, clone boot %s, delete %s" % ( "PASS" if ok else "FAIL", len(cyc), med.get("create_s"), med.get("cold_boot_s"), med.get("install_s"), med.get("cold_launch_call_s"), - med.get("cold_first_frame_s"), med.get("shutdown_s"), med.get("resume_boot_s"), med.get("resume_first_frame_s"), med.get("delete_s"))) + med.get("cold_first_frame_s"), med.get("shutdown_s"), med.get("resume_boot_s"), med.get("resume_first_frame_s"), med.get("clone_s"), med.get("clone_boot_s"), med.get("delete_s"))) leaks = {"qemu_rss": gates.leak_gate("QEMU RSS", xs, rss, a.leak_rss_mb, "MB/cycle"), "guest_used": gates.leak_gate("guest used memory", xs, used, a.leak_guest_mb, "MB/cycle")} data["leak"] = {"qemu_rss_mb": rss, "guest_used_mb": used} diff --git a/src/bench/tests/test_modules.py b/src/bench/tests/test_modules.py index dba90fa..0c1325f 100644 --- a/src/bench/tests/test_modules.py +++ b/src/bench/tests/test_modules.py @@ -70,6 +70,30 @@ def test_observer_is_nobodys_simulator_tree(self): self.assertAlmostEqual(tot["observer_cpu_pct"], 100.0, places=1) # two processes x 1 s over 2 s self.assertEqual(per["AAAA-1"]["procs"], 3) + def test_settle_ignores_the_load_by_whole_command(self): + for cmd in ("/Library/Developer/CoreSimulator/Volumes/iOS 26.5.simruntime/Contents/bin/backboardd", + "/x/SimMetalHost -d AAAA-1", "simfps -set /x", "/usr/bin/ssh host", "/usr/bin/python3 benchguest.py settle"): + self.assertTrue(bg.SETTLE_SKIP.search(cmd), cmd) + for cmd in ("/usr/libexec/mds", "/System/Library/Frameworks/Foo.framework/assetsd", "/usr/sbin/cfprefsd daemon"): + self.assertFalse(bg.SETTLE_SKIP.search(cmd), cmd) + + def test_count_launchd_sim_sees_every_device_set(self): + text = PS + " 600 1 20000 0:01.00 /Library/x/launchd_sim /other/set/CCCC-3/data\n" + saved = bg.ps_text + bg.ps_text = lambda: text + try: + self.assertEqual(bg.count_launchd_sim(), 2) + finally: + bg.ps_text = saved + + def test_qemu_env_value_reads_the_process_environ(self): + with tempfile.TemporaryDirectory() as root: + os.makedirs(os.path.join(root, "proc", "77")) + wr(os.path.join(root, "proc", "77", "environ"), "w", "A=1\0REIMS_VGPU_CENSUS=/run/x/coherent\0REIMS_VGPU_COHERENT=1\0") + self.assertEqual(hostprobe.qemu_env_value(77, "REIMS_VGPU_CENSUS", root), "/run/x/coherent") + self.assertIsNone(hostprobe.qemu_env_value(77, "NOPE", root)) + self.assertIsNone(hostprobe.qemu_env_value(78, "A", root)) + def test_cputime(self): self.assertAlmostEqual(bg.parse_cputime("0:01.50"), 1.5) self.assertAlmostEqual(bg.parse_cputime("2:03.25"), 123.25) diff --git a/src/bench/tests/test_runctx.py b/src/bench/tests/test_runctx.py index ee39877..72d62c0 100644 --- a/src/bench/tests/test_runctx.py +++ b/src/bench/tests/test_runctx.py @@ -87,3 +87,29 @@ def test_live_knob_restores_bytes(self): if __name__ == "__main__": unittest.main() + + +class GuardForeign(unittest.TestCase): + def test_a_simulator_outside_the_bench_refuses_the_run(self): + from macbench import core, fakes + + class A: + min_host_free_gib, min_disk_gib, min_guest_free_gib, allow_foreign, disk_path = 1.0, 1.0, 1.0, False, "/" + + class G: + def __init__(self, foreign): + self.foreign = foreign + + def call(self, args, timeout): + return {"mem": {"free_mb": 8000, "avail_mb": 9000}, "pressure": 1, "booted": 0, "foreign_booted": self.foreign} + + class Ctx: + a, est_sim_mb, host = A(), 1000.0, fakes.FakeHost() + c = Ctx() + c.guest = G(1) + self.assertIn("outside the bench", core.guard(c, 1, "x")) + c.a.allow_foreign = True + self.assertIsNone(core.guard(c, 1, "x")) + c.guest = G(0) + c.a.allow_foreign = False + self.assertIsNone(core.guard(c, 1, "x")) From 7b8e407bcbf6247a5454d1462d71ca2bb86f17d3 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Tue, 17 Nov 2026 04:36:45 +0200 Subject: [PATCH 12/23] bench: one persistent device set, run lock and leftover refusal, a verified cleanup that exits non-zero, idle-guard keep per simulator The bench's set is now 'bench' (not per run) so a seed can be kept; a run takes the set with a lock and refuses to start while an earlier run left devices, a lock or overrides. Teardown spares seeds, removes each device's overrides folder and ~/.sim-keep file, looks again and reports what is left (exit 20). macbox bench cleanup. sim-idle-guard keep (max 240) for every simulator; absent tool recorded, refusing tool fails. capture_env no longer drops runtime_used and device_type. Stub: the files outside the device live under its state dir. --- src/bench/guest/benchguest.py | 173 +++++++++++++++++++++++++++-- src/bench/guest/benchguest_stub.py | 40 ++++++- src/bench/macbench/cli.py | 67 +++++++++-- src/bench/macbench/core.py | 44 +++++++- src/bench/macbench/others.py | 4 +- src/bench/tests/test_dry.py | 11 +- src/bench/tests/test_dry_state.py | 93 ++++++++++++++++ 7 files changed, 396 insertions(+), 36 deletions(-) create mode 100644 src/bench/tests/test_dry_state.py diff --git a/src/bench/guest/benchguest.py b/src/bench/guest/benchguest.py index 73bcd17..3cd2986 100644 --- a/src/bench/guest/benchguest.py +++ b/src/bench/guest/benchguest.py @@ -11,6 +11,7 @@ import argparse import base64 import json +import math import os import plistlib import re @@ -26,6 +27,14 @@ sys.path.insert(0, os.path.dirname(HERE)) BUNDLE_ID = "dev.macbox.glassbench" MARKER = ".macbox-bench-set" +MANIFEST = ".bench-manifest.json" # what the bench made in its set: seeds (complete or not) and the devices it must clean up +LOCK = ".bench-run.lock" # present while a run owns the set; a leftover lock = a run that did not clean up +SEED_PREFIX = "bench-seed-" +# Things outside the device set that a bench simulator leaves behind or depends on. Module variables so the DRY stub +# can point them under its own state directory (nothing here is touched on the host). +OVERRIDES = "/private/var/tmp/com.apple.CoreSimulator.SimDevice" # + "." + udid: the Slim disables of one device +KEEP_DIR = os.path.expanduser("~/.sim-keep") # sim-idle-guard's keep files, one per udid +IDLE_GUARD = os.path.expanduser("~/.local/bin/sim-idle-guard") # `keep ` (max 240) PROBE_POINT = "36,175" # the input marker's centre, in points (GlassBench -probe) SET = None STUB = os.environ.get("BENCHGUEST_STUB") @@ -127,6 +136,69 @@ def devices(): return out +def is_seed(d): + return d.get("name", "").startswith(SEED_PREFIX) + + +def overrides_dir(udid): + return "%s.%s" % (OVERRIDES, udid) + + +def load_manifest(): + try: + with open(os.path.join(SET, MANIFEST)) as f: + m = json.load(f) + except (OSError, ValueError): + m = {} + m.setdefault("seeds", {}) + m.setdefault("devices", {}) + return m + + +def save_manifest(m): + p = os.path.join(SET, MANIFEST) + with open(p + ".tmp", "w") as f: + json.dump(m, f, indent=1, sort_keys=True) + os.replace(p + ".tmp", p) + + +def idle_guard_keep(udid, minutes): + """Registers a simulator with the Mac's sim-idle-guard (it shuts down any simulator nobody used for 15 minutes, in + every device set). {"state": "kept"|"absent"|"failed", "minutes", "err"}. 'absent' is recorded by the caller and the + run goes on; 'failed' (the tool is there and refused) is an error: the simulator would be shut down under the run.""" + minutes = max(1, min(int(math.ceil(minutes)), 240)) + if not (os.path.isfile(IDLE_GUARD) and os.access(IDLE_GUARD, os.X_OK)): + return {"state": "absent", "minutes": 0, "err": None} + rc, out, err, _ = run([IDLE_GUARD, "keep", udid, str(minutes)], 30) + if rc != 0: + return {"state": "failed", "minutes": minutes, "err": (err or out).strip()[-120:]} + return {"state": "kept", "minutes": minutes, "err": None} + + +def keep_or_fail(udid, minutes): + k = idle_guard_keep(udid, minutes) + if k["state"] == "failed": + raise GuestError("sim-idle-guard keep %s failed: %s (the Mac would shut the simulator down under the run)" % (udid, k["err"])) + return k + + +def drop_keep(udid): + """Removes ~/.sim-keep/; returns None or what went wrong (a missing file is fine).""" + try: + os.remove(os.path.join(KEEP_DIR, udid)) + except FileNotFoundError: + return None + except OSError as e: + return "keep file of %s: %s" % (udid, e) + return None + + +def drop_overrides(udid): + d = overrides_dir(udid) + shutil.rmtree(d, ignore_errors=True) + return "overrides folder of %s still there" % udid if os.path.exists(d) else None + + # --- parsers (unit-tested) ------------------------------------------------------------------------- def parse_vm_stat(text): """vm_stat -> {"page": bytes, "free": pages, ...}. Keys lower-cased, spaces to underscores.""" @@ -443,13 +515,19 @@ def cmd_ensure_type(a): def cmd_create(a): + """A fresh STOCK device (about 1.8 GB of guest disk each): only behind --stock-devices on the host.""" made = [] + man = load_manifest() for i in range(a.start, a.start + a.n): name = "%s-%d" % (a.prefix, i) rc, out, err, secs = simctl("create", name, a.type, a.runtime) if rc != 0: raise GuestError("create %s: %s" % (name, err.strip()[:200])) - made.append({"idx": i, "udid": out.strip().splitlines()[-1], "create_s": round(secs, 2)}) + udid = out.strip().splitlines()[-1] + man["devices"][udid] = {"name": name, "kind": "stock"} + save_manifest(man) + k = keep_or_fail(udid, a.keep_min) + made.append({"idx": i, "udid": udid, "create_s": round(secs, 2), "idle_guard": k["state"]}) out_json({"sims": made}) @@ -752,13 +830,89 @@ def cmd_cycle(a): shutil.rmtree(tmp, ignore_errors=True) +def leftovers(): + """What an earlier run left in the bench's set: [text]. Empty = the set is clean (settled seeds, shut down, are not + leftovers: they are kept on purpose).""" + out, man, devs = [], load_manifest(), devices() + complete = {s["udid"] for s in man["seeds"].values() if s.get("complete")} + for d in devs: + if is_seed(d): + if d["udid"] not in complete: + out.append("a seed that was never completed (%s %s)" % (d["name"], d["udid"])) + elif d.get("state") != "Shutdown": + out.append("the seed %s is %s" % (d["name"], d.get("state"))) + else: + out.append("device %s %s (%s)" % (d["name"], d["udid"], d.get("state"))) + here = {d["udid"] for d in devs} + for u in man["devices"]: + if u not in here and (os.path.exists(overrides_dir(u)) or os.path.exists(os.path.join(KEEP_DIR, u))): + out.append("overrides or keep file of the deleted device %s" % u) + lock = os.path.join(SET, LOCK) + if os.path.exists(lock): + try: + with open(lock) as f: + out.append("the run lock of an earlier run (%s)" % f.read().strip()[:80]) + except OSError: + out.append("the run lock of an earlier run") + return out + + +def cmd_acquire(a): + """Takes the set for one run: refuses (ok false, with the list) while an earlier run left anything, then writes the + run lock (exclusive create: two runs cannot share the set).""" + left = leftovers() + if left: + return out_json({"ok": False, "leftovers": left}) + try: + fd = os.open(os.path.join(SET, LOCK), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + except FileExistsError: + return out_json({"ok": False, "leftovers": ["the run lock was taken just now by another run"]}) + with os.fdopen(fd, "w") as f: + f.write("run %s pid %d at %d\n" % (a.run, os.getpid(), int(now()))) + out_json({"ok": True, "leftovers": []}) + + def cmd_teardown(a): - """Shut down and delete EVERY device in the bench set (it is the bench's own, marker file checked).""" - simctl("shutdown", "all", timeout=300) - simctl("delete", "all", timeout=300) - left = [d["udid"] for d in devices()] - booted = [d["udid"] for d in devices() if d.get("state") == "Booted"] - out_json({"left": len(left), "booted": len(booted), "calls": [c[1:] for c in CALLS][-3:]}) + """Shuts down every device of the bench's set and deletes every one that is not a seed (all of them with --seeds), + then removes what lives outside the device: its overrides folder and its ~/.sim-keep file. It looks again at the end + and reports what is still there in `failed`/`left`/`booted`; the run lock is released only when nothing is.""" + failed = [] + man = load_manifest() + simctl("shutdown", "all", timeout=300) # a failure shows in the second look below + for d in devices(): + if is_seed(d) and not a.seeds: + continue + rc, _, err, _ = simctl("delete", d["udid"], timeout=120) + if rc != 0: + failed.append("delete %s %s: %s" % (d["name"], d["udid"], (err or "failed").strip()[:100])) + after = devices() + present = {d["udid"] for d in after} + seeds_now = {d["udid"] for d in after if is_seed(d)} + for u in sorted(set(man["devices"]) | {s["udid"] for s in man["seeds"].values()} | {d["udid"] for d in after}): + e = drop_keep(u) + if e: + failed.append(e) + if u not in present: + e = drop_overrides(u) + if e: + failed.append(e) + else: + man["devices"].pop(u, None) + man["seeds"] = {t: s for t, s in man["seeds"].items() if s["udid"] in seeds_now} + save_manifest(man) + left = [d for d in after if not is_seed(d) or a.seeds] + booted = [d for d in after if d.get("state") == "Booted"] + keep_left = [u for u in present | set(man["devices"]) if os.path.exists(os.path.join(KEEP_DIR, u))] + if keep_left: + failed.append("%d keep file(s) still in the idle guard's folder" % len(keep_left)) + if not (failed or left or booted): + try: + os.remove(os.path.join(SET, LOCK)) + except FileNotFoundError: + pass + except OSError as e: + failed.append("run lock: %s" % e) + out_json({"left": len(left), "booted": len(booted), "failed": failed, "seeds": len(seeds_now), "calls": [c[1:] for c in CALLS][-3:]}) def count_launchd_sim(): @@ -829,7 +983,7 @@ def add(name, fn, *opts): add("init-set", cmd_init_set) add("ensure-type", cmd_ensure_type, O("--scale", default="1"), O("--allow-restart", action="store_true")) add("create", cmd_create, O("--n", type=int, required=True), O("--type", required=True), O("--runtime", required=True), - O("--prefix", default="bench"), O("--start", type=int, default=0)) + O("--prefix", default="bench"), O("--start", type=int, default=0), O("--keep-min", type=float, default=60)) add("slim", cmd_slim, O("--udids", required=True), O("--cmd", required=True)) add("boot", cmd_boot, O("--udids", required=True), O("--parallel", action="store_true")) add("prepare", cmd_prepare, O("--udids", required=True), O("--roles", required=True), O("--install", action="store_true"), @@ -842,7 +996,8 @@ def add(name, fn, *opts): O("--no-strip", action="store_true"), O("--probe-timeout", type=float, default=2.0)) add("shots", cmd_shots, O("--udids", required=True), O("--count", type=int, default=2)) add("cycle", cmd_cycle, O("--type", required=True), O("--runtime", required=True), O("--prefix", default="bench")) - add("teardown", cmd_teardown) + add("acquire", cmd_acquire, O("--run", default="?")) + add("teardown", cmd_teardown, O("--seeds", action="store_true", help="delete the seeds too")) add("build", cmd_build, O("--xcodebuild", default="")) add("memcheck", cmd_memcheck) add("ips", cmd_ips) diff --git a/src/bench/guest/benchguest_stub.py b/src/bench/guest/benchguest_stub.py index a2ddee1..43491ac 100644 --- a/src/bench/guest/benchguest_stub.py +++ b/src/bench/guest/benchguest_stub.py @@ -2,7 +2,11 @@ directory ($BENCHGUEST_STATE) between calls, so `benchguest.py` runs its real control flow with no macOS. Faults ($BENCH_STUB_FAULT, comma separated): memfloor@N (guest free memory below any floor from N booted simulators), crash@N (a new crash report at N booted), backwards@N (a frame shown out of order at N), slow@N -(frame rate collapses from N), hang_simfps (simfps never answers), nomarker (the input marker never flips).""" +(frame rate collapses from N), hang_simfps (simfps never answers), nomarker (the input marker never flips), +cantdelete (simctl delete fails: a cleanup that cannot finish), noguard (no sim-idle-guard on the Mac), keepfail +(sim-idle-guard keep fails). +The files a simulator leaves outside its device (Slim overrides, ~/.sim-keep, the idle guard) live under +$BENCHGUEST_STATE too, so the bench's real file code runs against them.""" import json import os import random @@ -79,7 +83,17 @@ def stub_run(argv, timeout=120, env=None): def _stub_run(argv): _log(argv) s = load() - out, rc = "", 0 + out, err, rc = "", "", 0 + if argv and argv[0] == _MOD.IDLE_GUARD: + # the Mac's idle guard: `keep ` (max 240) writes ~/.sim-keep/ + if fault("keepfail") is not None: + return 1, "", "keep: refused (injected)", 0.01 + if argv[1] != "keep" or len(argv) != 4 or not argv[3].isdigit() or not 1 <= int(argv[3]) <= 240: + return 2, "", "usage: sim-idle-guard keep ", 0.01 + os.makedirs(_MOD.KEEP_DIR, exist_ok=True) + with open(os.path.join(_MOD.KEEP_DIR, argv[2]), "w") as f: + f.write("%s\n" % argv[3]) + return 0, "", "", 0.01 if argv[:3] == ["xcrun", "simctl", "--set"]: cmd = argv[4:] verb = cmd[0] @@ -105,8 +119,11 @@ def _stub_run(argv): if u in s["devs"]: s["devs"][u]["state"] = "Shutdown" elif verb == "delete": - for u in (list(s["devs"]) if cmd[1] == "all" else [cmd[1]]): - s["devs"].pop(u, None) + if fault("cantdelete") is not None: + err, rc = "Unable to delete (injected)", 1 + else: + for u in (list(s["devs"]) if cmd[1] == "all" else [cmd[1]]): + s["devs"].pop(u, None) elif verb == "launch": s["mode"][cmd[1]] = "static" if "-static" in cmd else "busy" elif verb == "terminate": @@ -140,7 +157,7 @@ def _stub_run(argv): else: out = "" save(s) - return rc, out, "", 0.01 + return rc, out, err, 0.01 def stub_ps(): @@ -229,7 +246,20 @@ def kill(self): pass +_MOD = None + + def install(mod): + global _MOD + _MOD = mod + mod.OVERRIDES = _p("overrides/SimDevice") + mod.KEEP_DIR = _p("sim-keep") + mod.IDLE_GUARD = _p("bin/sim-idle-guard") + if fault("noguard") is None: + os.makedirs(_p("bin"), exist_ok=True) + with open(mod.IDLE_GUARD, "w") as f: + f.write("#!/bin/sh\n") + os.chmod(mod.IDLE_GUARD, 0o755) mod.run = stub_run mod.ps_text = stub_ps mod.vmstat_text = stub_vmstat diff --git a/src/bench/macbench/cli.py b/src/bench/macbench/cli.py index 3a4c0f6..7a43552 100644 --- a/src/bench/macbench/cli.py +++ b/src/bench/macbench/cli.py @@ -24,6 +24,7 @@ def add_common(p): p.add_argument("--min-host-free-gib", type=float, default=6.0, help="refuse to boot more simulators below this host available memory") p.add_argument("--min-disk-gib", type=float, default=12.0, help="refuse below this free disk space (the VM's disk, host side)") p.add_argument("--disk-path", default=os.environ.get("BENCH_HOME"), help=argparse.SUPPRESS) + p.add_argument("--set-name", default=os.environ.get("BENCH_SET_NAME", "bench"), help="the bench's own device set in the guest (kept between runs: the seed lives in it)") p.add_argument("--allow-foreign", action="store_true", help="go on although simulators outside the bench are booted (recorded)") p.add_argument("--allow-restart", action="store_true", help="let a NEW device type restart CoreSimulatorService (only when nothing is booted)") p.add_argument("--scale-x", dest="scale_x", type=int, choices=(1, 2, 3), default=1, help="device pixel scale: 1 = the macbox 1x type, 3 = the stock iPhone (default 1)") @@ -106,6 +107,9 @@ def parser(): t = sp.add_parser("tools", help="copy the bench to the Mac and build simfps and GlassBench there") t.add_argument("--no-build", action="store_true") sp.add_parser("check", help="what this host and the VM offer the bench") + cl = sp.add_parser("cleanup", help="shut down and delete what an earlier run left in the bench's device set (seeds stay unless --seeds)") + cl.add_argument("--seeds", action="store_true", help="delete the seeds too (the next run makes a new one)") + cl.add_argument("--set-name", default=os.environ.get("BENCH_SET_NAME", "bench")) return ap @@ -152,7 +156,7 @@ def build_ctx(a, kind): alias = os.environ.get("MACBOX_SSH_HOST", "") clean = schema.scrubber([alias] if alias and alias != "macbox" else []) # not user names or the default alias: ordinary words stamp = schema.stamp() - set_name = "bench-%s" % stamp + set_name = a.set_name # ONE set for every run: the seed made in it is kept and reused faults = os.environ.get("BENCH_STUB_FAULT", "") if dry: tmp = os.environ.get("BENCH_DRY_KEEP") or tempfile.mkdtemp(prefix="macbench-dry-") @@ -221,21 +225,39 @@ def execute(a, kind, body): rc = 2 finally: run.cleanup() - info = ctx.cleanup_info - if info or any(l for l in run.teardown): - left_ok = (not info) or (info.get("booted", 0) == 0 and info.get("left", 0) == 0) - sec = "PASS" if left_ok and not run.problems else "FAIL" - ctx.section("cleanup", sec, "CLEANUP %s: bench simulators left booted %s, devices left %s, knobs restored %s%s" % ( - sec, info.get("booted", "n/a"), info.get("left", "n/a"), "yes" if not run.problems else "NO", - ("; " + "; ".join(run.problems)) if run.problems else "")) - problems = ctx.save() + rc = max(rc, finalize(ctx)) fails = [s for s in ctx.doc["sections"] if s["status"] == "FAIL"] - print("result: %s%s" % (os.path.basename(ctx.out_path), "" if not problems else " (NOT publishable as is: %s)" % "; ".join(problems[:3]))) if fails: return max(rc, 1) return rc +CLEANUP_FAILED_RC = 20 + + +def finalize(ctx): + """After the cleanup: its section (PASS only when verified), the idle guard's, the result file. Also what the abort + path of the run's monitor calls. Returns CLEANUP_FAILED_RC when the cleanup could not delete everything.""" + run, info = ctx.run, ctx.cleanup_info + rc = 0 + if ctx.taken or info or run.problems: + left_ok = bool(info) and not info.get("failed") and info.get("booted", 0) == 0 and info.get("left", 0) == 0 + sec = "PASS" if left_ok and not run.problems else "FAIL" + if sec == "FAIL": + rc = CLEANUP_FAILED_RC + ctx.section("cleanup", sec, "CLEANUP %s: bench simulators left booted %s, devices left %s, knobs restored %s%s" % ( + sec, info.get("booted", "n/a"), info.get("left", "n/a"), "yes" if not run.problems else "NO", + ("; " + "; ".join(run.problems)) if run.problems else "")) + if ctx.idle_guard: + absent = "absent" in ctx.idle_guard + ctx.doc["env"].setdefault("guest", {})["idle_guard"] = "absent" if absent else "kept" + ctx.section("idle_guard", "INFO", ("IDLE GUARD ABSENT: sim-idle-guard is not installed on the Mac, nothing keeps the simulators from an idle shutdown; " + "the run counts its booted simulators before every window") if absent else "IDLE GUARD kept every bench simulator for the run") + problems = ctx.save() + print("result: %s%s" % (os.path.basename(ctx.out_path), "" if not problems else " (NOT publishable as is: %s)" % "; ".join(problems[:3]))) + return rc + + def main(argv=None): a = parser().parse_args(argv) c = a.cmd @@ -245,6 +267,8 @@ def main(argv=None): return tools(a) if c == "check": return check(a) + if c == "cleanup": + return cleanup(a) if c == "scale": a.ns = [int(x) for x in a.ns.split(",")] a.targets = a.targets or targets.DEFAULT_TARGETS @@ -260,6 +284,29 @@ def main(argv=None): return execute(a, {"golden": "gates"}.get(c, c), body) +def cleanup(a): + """macbox bench cleanup: shut down and delete what an earlier run left in the bench's set; exit 0 only when verified.""" + run = Run(900, log) + faults = os.environ.get("BENCH_STUB_FAULT", "") + if os.environ.get("DRY") == "1": + keep = os.environ.get("BENCH_DRY_KEEP") + if not keep: + raise SystemExit("DRY cleanup needs BENCH_DRY_KEEP (the stub guest's state directory)") + guest = LocalGuest(run, a.set_name, os.path.join(keep, "guest"), BENCH_DIR, faults) + else: + guest = SshGuest(run, os.environ.get("BENCH_SSH") or os.environ.get("MACBOX_SSH_HOST") or "macbox", a.set_name) + try: + guest.call(["init-set"], 60) + info = guest.call(["teardown"] + (["--seeds"] if a.seeds else []), 300) + except (GuestError, DeadlineError) as e: + print("CLEANUP FAIL: the guest could not be reached or refused: %s" % e) + return CLEANUP_FAILED_RC + ok = not (info.get("failed") or info.get("left") or info.get("booted")) + print("CLEANUP %s: devices left %s, booted %s, seeds kept %s%s" % ("PASS" if ok else "FAIL", info.get("left"), info.get("booted"), info.get("seeds"), + ("; could not: " + "; ".join(info["failed"])) if info.get("failed") else "")) + return 0 if ok else CLEANUP_FAILED_RC + + def offline(a): if a.cmd == "compare": text, worse = compare.compare(schema.load(a.a), schema.load(a.b), a.noise) diff --git a/src/bench/macbench/core.py b/src/bench/macbench/core.py index dff5fac..b8ebb37 100644 --- a/src/bench/macbench/core.py +++ b/src/bench/macbench/core.py @@ -23,6 +23,8 @@ def __init__(self, args, run, guest, host, coherent, doc, out_path, clean, log=p self.marks = {} self.golden = None # fn(w, h) -> (hash or None, key) self.pixel_gate = None + self.taken = False # the bench's set is owned by this run (lock taken, cleanup registered) + self.idle_guard = set() # what sim-idle-guard keep answered for the bench's simulators: kept / absent def save(self): return schema.write(self.doc, self.out_path, self.clean) @@ -37,7 +39,8 @@ def capture_env(ctx, info): g = {k: info.get(k) for k in ("macos", "xcode", "arch", "cpus", "mem_mb")} rts = info.get("runtimes") or [] g["runtimes"] = ["%s %s" % (r.get("name"), r.get("build")) for r in rts] - ctx.doc["env"].update({"build_id": ctx.a.build_id or "unknown", "host": host, "guest": g, + ctx.doc["env"].setdefault("guest", {}).update(g) # keeps what setup_guest recorded (runtime, device type, seed) + ctx.doc["env"].update({"build_id": ctx.a.build_id or "unknown", "host": host, "switches": sw, "qemu_env": qenv, "tags": dict(kv.split("=", 1) for kv in (ctx.a.tag or []) if "=" in kv), "dry": bool(ctx.host.dry)}) @@ -77,14 +80,38 @@ def roles_for(mix, n): raise ValueError("unknown mix %r" % mix) -def setup_guest(ctx): - """Device set, device type, tools check. Returns the guest info.""" +def teardown(ctx): + """The cleanup of the bench's simulators. Raises, naming what is left, when anything could not be deleted: the + cleanup section must never say 'done' on a cleanup that was not verified.""" + info = ctx.guest.call(["teardown"], 300) + ctx.cleanup_info.update(info) + if info.get("failed") or info.get("left") or info.get("booted"): + raise RuntimeError("could not clean up the bench's device set: %s; %d device(s) left, %d booted: run `macbox bench cleanup` " + "(the next run refuses to start until it is clean)" % ("; ".join(info.get("failed") or ["nothing reported"])[:300], info.get("left", 0), info.get("booted", 0))) + + +def keep_minutes(ctx): + """How long the Mac's idle guard must leave the bench's simulators alone: the run's deadline plus 5 minutes, at most 240.""" + return max(5, min(240, int(ctx.a.deadline / 60.0) + 5)) + + +def setup_guest(ctx, sims=True): + """Device set, device type, tools check. Returns the guest info. With sims=False the run boots no simulator of its + own (idle, build): the set is not taken and nothing is registered for cleanup.""" a = ctx.a info = ctx.guest.call(["info"], 120) if not info.get("has_simfps") or not info.get("has_app"): raise Stop("the bench tools are not built in the guest: macbox bench tools") ctx.guest.call(["init-set"], 60) - ctx.run.teardown.append(("bench simulators", lambda: ctx.cleanup_info.update(ctx.guest.call(["teardown"], 240)))) + if not sims: + _run_mark(ctx) + return info + acq = ctx.guest.call(["acquire", "--run", ctx.doc["id"]], 60) + if not acq.get("ok"): + # refuse BEFORE the cleanup is registered: the leftovers are not this run's to delete (macbox bench cleanup does) + raise Stop("an earlier run left the bench's device set unclean (%s): run `macbox bench cleanup`, then start again" % "; ".join(acq.get("leftovers", [])[:4])) + ctx.run.teardown.append(("bench simulators", lambda: teardown(ctx))) + ctx.taken = True t = ctx.guest.call(["ensure-type", "--scale", str(a.scale_x)] + (["--allow-restart"] if a.allow_restart else []), 180) if t.get("needs_restart"): raise Stop("device type %s was just created and CoreSimulatorService must restart to load it, which would shut " @@ -92,9 +119,13 @@ def setup_guest(ctx): ctx.type_id, ctx.runtime = t["type"], t["runtime"] ctx.doc["env"]["guest"]["runtime_used"] = t.get("runtime_version") ctx.doc["env"]["guest"]["device_type"] = "stock %s" % t["stock_name"] if a.scale_x == 3 else "macbox %dx of %s" % (a.scale_x, t["stock_name"]) + _run_mark(ctx) + return info + + +def _run_mark(ctx): ctx.offset = ctx.guest.clock_offset() ctx.run_mark = {"coh": ctx.coh.mark(), "t0": time.time(), "ips": set(ctx.guest.call(["ips"], 60)["ips"])} - return info def grow(ctx, n, roles, install=True): @@ -103,8 +134,9 @@ def grow(ctx, n, roles, install=True): new = n - have boot = {} if new > 0: - r = ctx.guest.call(["create", "--n", new, "--type", ctx.type_id, "--runtime", ctx.runtime, "--start", have], 240) + r = ctx.guest.call(["create", "--n", new, "--type", ctx.type_id, "--runtime", ctx.runtime, "--start", have, "--keep-min", keep_minutes(ctx)], 240) fresh = r["sims"] + ctx.idle_guard.update(x.get("idle_guard") for x in fresh) us = ",".join(s["udid"] for s in fresh) if ctx.a.slim: sl = ctx.guest.call(["slim", "--udids", us, "--cmd", ctx.a.slim_cmd], 900 * len(fresh)) diff --git a/src/bench/macbench/others.py b/src/bench/macbench/others.py index 61b09f7..3110dff 100644 --- a/src/bench/macbench/others.py +++ b/src/bench/macbench/others.py @@ -15,7 +15,7 @@ def _fmt(x, unit=""): def run_idle(ctx): a = ctx.a - info = core.setup_guest(ctx) + info = core.setup_guest(ctx, sims=False) core.capture_env(ctx, info) ctx.marks = {"coh": ctx.coh.mark(), "t0": time.time()} m = ctx.guest.call(["memcheck"], 60) @@ -135,7 +135,7 @@ def run_soak(ctx): def run_build(ctx): a = ctx.a - info = core.setup_guest(ctx) + info = core.setup_guest(ctx, sims=False) core.capture_env(ctx, info) r = ctx.guest.call(["build"] + (["--xcodebuild", a.xcodebuild] if a.xcodebuild else []), 3900) ctx.doc["data"] = r diff --git a/src/bench/tests/test_dry.py b/src/bench/tests/test_dry.py index 745f73c..3f54588 100644 --- a/src/bench/tests/test_dry.py +++ b/src/bench/tests/test_dry.py @@ -15,13 +15,16 @@ FAST = ["--secs", "3", "--probe-every", "0.7", "--probe-timeout", "1", "--settle-max", "2", "--no-observer-check"] -def run_case(args, faults="", extra_env=None, kill_after=None, keep=True, timeout=240): +def run_case(args, faults="", extra_env=None, kill_after=None, keep=True, timeout=240, state=None): + """One real `run` process in DRY mode. `state` = a directory shared by several cases in a row (the stub guest, its + device set, the seed): the run after another one sees what the first left.""" d = tempfile.mkdtemp(prefix="dry-case-") - env = dict(os.environ, DRY="1", BENCH_STUB_FAULT=faults, BENCH_DRY_KEEP=os.path.join(d, "state") if keep else "", + env = dict(os.environ, DRY="1", BENCH_STUB_FAULT=faults, BENCH_DRY_KEEP=state or os.path.join(d, "state") if keep else "", BENCH_DEADLINE_MARGIN="3", **(extra_env or {})) if not keep: env.pop("BENCH_DRY_KEEP") - p = subprocess.Popen([sys.executable, RUN] + args + ["--out", os.path.join(d, "out")], env=env, stdout=subprocess.PIPE, + out_arg = [] if args[0] == "cleanup" else ["--out", os.path.join(d, "out")] + p = subprocess.Popen([sys.executable, RUN] + args + out_arg, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) if kill_after: time.sleep(kill_after) @@ -34,7 +37,7 @@ def run_case(args, faults="", extra_env=None, kill_after=None, keep=True, timeou if f.endswith(".json"): with open(os.path.join(od, f)) as fh: res = json.load(fh) - return {"rc": p.returncode, "out": out, "doc": res, "state": os.path.join(d, "state")} + return {"rc": p.returncode, "out": out, "doc": res, "state": state or os.path.join(d, "state")} def sims_booted(state): diff --git a/src/bench/tests/test_dry_state.py b/src/bench/tests/test_dry_state.py new file mode 100644 index 0000000..908c15d --- /dev/null +++ b/src/bench/tests/test_dry_state.py @@ -0,0 +1,93 @@ +"""DRY runs that follow each other in one stub guest: what a run leaves, what the next one refuses, the idle guard. +Each case is a real `run` process (stub guest, fake host); nothing real is touched.""" +import json +import os +import tempfile +import unittest + +from tests.test_dry import FAST, run_case, sections, simctl_calls + +ONE = ["scale", "--n", "1,2"] + FAST + + +def calls(state): + with open(os.path.join(state, "guest", "calls.log")) as f: + return [json.loads(l) for l in f] + + +def keep_calls(state): + return [c for c in calls(state) if c[0].endswith("sim-idle-guard")] + + +def files(state, sub): + try: + return sorted(os.listdir(os.path.join(state, "guest", sub))) + except OSError: + return [] + + +class CleanupAndLeftovers(unittest.TestCase): + def test_a_cleanup_that_cannot_delete_says_so_exits_nonzero_and_blocks_the_next_run(self): + state = tempfile.mkdtemp(prefix="dry-state-") + r1 = run_case(ONE, "cantdelete", state=state) + self.assertEqual(r1["rc"], 20, r1["out"][-800:]) + c = sections(r1["doc"], "cleanup")[0] + self.assertEqual(c["status"], "FAIL") + self.assertIn("could not clean up", c["line"]) + self.assertNotIn("CLEANUP PASS", r1["out"]) + self.assertTrue(os.path.exists(os.path.join(state, "guest", "set", ".bench-run.lock"))) + n_before = len([x for x in calls(state) if "clone" in x or "create" in x]) + r2 = run_case(ONE, "", state=state) # the next run refuses to start + self.assertEqual(r2["rc"], 2, r2["out"][-800:]) + self.assertIn("run `macbox bench cleanup`", r2["doc"]["sections"][0]["line"]) + self.assertEqual(n_before, len([x for x in calls(state) if "clone" in x or "create" in x])) # it made nothing + r3 = run_case(["cleanup"], "", state=state) # the explicit command cleans, verified + self.assertEqual(r3["rc"], 0, r3["out"]) + self.assertIn("CLEANUP PASS", r3["out"]) + r4 = run_case(ONE, "", state=state) + self.assertEqual(r4["rc"], 0, r4["out"][-800:]) + + def test_cleanup_command_fails_loudly_when_it_cannot_delete(self): + state = tempfile.mkdtemp(prefix="dry-state-") + run_case(ONE, "cantdelete", state=state) + r = run_case(["cleanup"], "cantdelete", state=state) + self.assertEqual(r["rc"], 20) + self.assertIn("CLEANUP FAIL", r["out"]) + + +class IdleGuard(unittest.TestCase): + def test_every_bench_simulator_is_kept_for_the_run_and_the_keep_files_are_removed(self): + state = tempfile.mkdtemp(prefix="dry-state-") + r = run_case(ONE, "", state=state) + self.assertEqual(r["rc"], 0, r["out"][-600:]) + made = {c[2] for c in keep_calls(state)} + self.assertEqual(len(made), 2) + for c in keep_calls(state): + self.assertEqual(c[1], "keep") + self.assertEqual(c[3], str(95)) # --deadline 5400 s = 90 min + 5 + self.assertEqual(files(state, "sim-keep"), []) # cleanup removed ~/.sim-keep/ + self.assertEqual(r["doc"]["env"]["guest"]["idle_guard"], "kept") + + def test_minutes_are_capped_at_240(self): + state = tempfile.mkdtemp(prefix="dry-state-") + r = run_case(ONE + ["--deadline", "30000"], "", state=state) + self.assertEqual(r["rc"], 0, r["out"][-600:]) + self.assertTrue(keep_calls(state)) + self.assertEqual({c[3] for c in keep_calls(state)}, {"240"}) + + def test_missing_tool_is_recorded_and_the_run_goes_on(self): + r = run_case(ONE, "noguard") + self.assertEqual(r["rc"], 0, r["out"][-600:]) + self.assertEqual(r["doc"]["env"]["guest"]["idle_guard"], "absent") + s = sections(r["doc"], "idle_guard")[0] + self.assertEqual(s["status"], "INFO") + self.assertIn("ABSENT", s["line"]) + + def test_a_refusing_tool_stops_the_run(self): + r = run_case(ONE, "keepfail") + self.assertNotEqual(r["rc"], 0) + self.assertIn("sim-idle-guard keep", r["out"]) + + +if __name__ == "__main__": + unittest.main() From 4df10aaa4a7f2c12b0c1f943eb27585801d18a42 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Tue, 17 Nov 2026 04:58:09 +0200 Subject: [PATCH 13/23] bench: bench simulators are production simulators: clones of a settled Slim seed made from Simlock's source device ONE seed per device type in the bench's set: simctl --set clone (the only command that names Simlock's set), idle-guard keep, boot, simslim --set on --profile (the sources are widget-trimmed, not Slim), >= 100 services off checked, the app, a real settle (two quiet samples, widget icons present), shutdown. Kept between runs, reused, --rebuild-seed; a half-made seed is deleted, a stale app refuses reuse. Every simulator = simctl clone of the seed + the seed's disabled.plist copied to its UDID folder (checked) before its first boot. Fresh stock devices only behind --stock-devices (one at a time, guard in front); --slim/simslim-per-device removed. No Simlock source: a clear message offering the stock path. Lifecycle cycles clone the seed. Stub models a source set, simslim, plutil, apps and chronod icons. --- src/bench/guest/benchguest.py | 359 +++++++++++++++++++++++++---- src/bench/guest/benchguest_stub.py | 95 +++++++- src/bench/macbench/cli.py | 12 +- src/bench/macbench/core.py | 99 ++++++-- src/bench/macbench/others.py | 21 +- src/bench/macbench/schema.py | 2 +- src/bench/tests/test_dry.py | 24 +- src/bench/tests/test_dry_seed.py | 264 +++++++++++++++++++++ src/bench/tests/test_dry_state.py | 2 +- 9 files changed, 788 insertions(+), 90 deletions(-) create mode 100644 src/bench/tests/test_dry_seed.py diff --git a/src/bench/guest/benchguest.py b/src/bench/guest/benchguest.py index 3cd2986..00af18d 100644 --- a/src/bench/guest/benchguest.py +++ b/src/bench/guest/benchguest.py @@ -10,6 +10,7 @@ """ import argparse import base64 +import hashlib import json import math import os @@ -35,6 +36,12 @@ OVERRIDES = "/private/var/tmp/com.apple.CoreSimulator.SimDevice" # + "." + udid: the Slim disables of one device KEEP_DIR = os.path.expanduser("~/.sim-keep") # sim-idle-guard's keep files, one per udid IDLE_GUARD = os.path.expanduser("~/.local/bin/sim-idle-guard") # `keep ` (max 240) +# Simlock's own source devices (read only: nothing is ever written to, booted, shut down or deleted in this set; the one +# command that names it is simlock_clone()) and the Slim tooling the fleet uses on every clone. +SOURCE_SET = os.environ.get("BENCH_SIMLOCK_SOURCES") or os.path.expanduser("~/.simlock04/devices/ios-sources") +SIMSLIM = os.environ.get("BENCH_SIMSLIM") or os.path.expanduser("~/go/bin/simslim") +SLIM_PROFILE = os.environ.get("BENCH_SLIM_PROFILE") or os.path.expanduser("~/.config/simslim/atlas.json") +MIN_SLIM_SERVICES = 100 # a Slim device has 168 services off today PROBE_POINT = "36,175" # the input marker's centre, in points (GlassBench -probe) SET = None STUB = os.environ.get("BENCHGUEST_STUB") @@ -531,6 +538,187 @@ def cmd_create(a): out_json({"sims": made}) +def simlock_clone(src_set, src_udid, name): + """THE one command that names Simlock's device set: `simctl --set clone `. + It reads the source and writes into the bench set; Simlock's set is never written, booted, shut down or deleted.""" + check_set() + argv = ["xcrun", "simctl", "--set", src_set, "clone", src_udid, name, SET] + CALLS.append(argv) + return run(argv, 900) + + +UUID = re.compile(r"^[0-9A-Fa-f]{8}(-[0-9A-Fa-f]{4}){3}-[0-9A-Fa-f]{12}$") + + +def find_source(type_id, src_set): + """Simlock's source device of this device type: (udid, device.plist dict), or (None, why). Reads the source folders' + device.plist only (never config.json); prefers a source that is shut down.""" + try: + names = sorted(n for n in os.listdir(src_set) if UUID.match(n)) + except OSError: + return None, "Simlock's source device set was not found at the configured path (Simlock is optional)" + found, types = [], set() + for n in names: + try: + with open(os.path.join(src_set, n, "device.plist"), "rb") as f: + pl = plistlib.load(f) + except (OSError, ValueError, plistlib.InvalidFileException): + continue + types.add(str(pl.get("deviceType")).rsplit(".", 1)[-1]) + if pl.get("deviceType") == type_id: + found.append((0 if pl.get("state") == 1 else 1, n, pl)) + if not found: + return None, "Simlock has no source device of type %s (it has: %s)" % (type_id.rsplit(".", 1)[-1], ", ".join(sorted(types)) or "none") + found.sort(key=lambda x: (x[0], x[1])) + return found[0][1], found[0][2] + + +def runtime_version(runtime_id): + try: + for r in json.loads(run(["xcrun", "simctl", "list", "runtimes", "-j"], 30)[1]).get("runtimes", []): + if r.get("identifier") == runtime_id: + return r.get("version") + except ValueError: + pass + return None + + +def top_text(top): + return ", ".join("%s %g%%" % (t["cmd"], t["cpu_pct"]) for t in top[:5]) or "nothing above 0%" + + +def delete_device(udid): + """A device of the bench's set: shut down, delete, remove what lives outside it (and from the manifest); returns problems.""" + probs = [] + simctl("shutdown", udid, timeout=120) + rc, _, err, _ = simctl("delete", udid, timeout=120) + if rc != 0: + probs.append("delete %s: %s" % (udid, err.strip()[:100])) + for e in (drop_keep(udid), drop_overrides(udid)): + if e: + probs.append(e) + man = load_manifest() + man["devices"].pop(udid, None) + if not probs: + man["seeds"] = {t: x for t, x in man["seeds"].items() if x["udid"] != udid} + save_manifest(man) + return probs + + +def cmd_seed(a): + """ONE settled production seed per device type in the bench's set: a clone of Simlock's own source device of that type + (widgets trimmed, as production), one boot, simslim with the fleet's profile (Simlock's sources are not Slim), the bench + app, a real settle, shutdown. Kept (shut down) between runs and reused; every bench simulator is a clone of it. A seed + that was never completed is deleted again, never left to be reused.""" + tid = a.type + name = SEED_PREFIX + tid.rsplit(".", 1)[-1] + man = load_manifest() + cur = man["seeds"].get(tid) + here = {d["udid"]: d for d in devices()} + stamp = app_stamp() + if cur and (a.rebuild or cur["udid"] not in here): + if cur["udid"] in here: + probs = delete_device(cur["udid"]) + if probs: + raise GuestError("the old seed could not be deleted: %s" % "; ".join(probs)) + man = load_manifest() + man["seeds"].pop(tid, None) + save_manifest(man) + cur = None + if cur: + if not cur.get("complete"): + raise GuestError("the seed in the bench's set was never completed: macbox bench cleanup --seeds") + u = cur["udid"] + if here[u].get("state") != "Shutdown": + simctl("shutdown", u, timeout=120) + n = slim_count(u) + why = ("only %d services off" % n) if n < MIN_SLIM_SERVICES else (None if os.path.isdir(icons_path(u)) else "its widgets never settled (no chronod icons)") + if not why and cur.get("app_stamp") != stamp: + why = "the bench app was rebuilt since (GlassBench differs)" + if why: + raise GuestError("the existing seed cannot be reused (%s): rebuild it with --rebuild-seed" % why) + return out_json(dict(cur, seed=u, reused=True, services_off=n, source_type=tid.rsplit(".", 1)[-1])) + src, pl = find_source(tid, os.path.expanduser(a.source_set or SOURCE_SET)) + if src is None: + return out_json({"seed": None, "no_source": pl, "udid": None}) + t0 = now() + rc, out, err, _ = simlock_clone(os.path.expanduser(a.source_set or SOURCE_SET), src, name) + if rc != 0: + raise GuestError("simctl clone out of Simlock's source device failed: %s" % (err or out).strip()[-200:]) + u = out.strip().splitlines()[-1] + runtime = pl.get("runtime") + man["seeds"][tid] = {"udid": u, "name": name, "complete": False, "type": tid, "runtime": runtime} + man["devices"][u] = {"name": name, "kind": "seed"} + save_manifest(man) # written before anything else is done to it: a crash leaves an incomplete seed, which is refused + try: + keep_or_fail(u, a.keep_min) + if simctl("boot", u, timeout=300)[0] != 0: + raise GuestError("the seed did not boot") + if simctl("bootstatus", u, timeout=600)[0] != 0: + raise GuestError("the seed never finished booting (bootstatus)") + sl = os.path.expanduser(a.simslim or SIMSLIM) + prof = os.path.expanduser(a.slim_profile or SLIM_PROFILE) + # Simlock's source devices are widget-trimmed but NOT Slim: simslim shuts the device down, rewrites its services and reboots it + rc, out, err, secs = run([sl, "--set", SET, "on", u, "--profile", prof], 900) + if rc != 0: + raise GuestError("simslim failed on the seed (rc %s): %s" % (rc, (err or out).strip()[-200:])) + if simctl("bootstatus", u, timeout=600)[0] != 0: + raise GuestError("the seed never finished booting after simslim (bootstatus)") + n = slim_count(u) + if n < MIN_SLIM_SERVICES: + raise GuestError("the seed is not Slim after simslim: %d services off, need >= %d" % (n, MIN_SLIM_SERVICES)) + app = os.path.join(HERE, "..", "GlassBench.app") + if simctl("install", u, app, timeout=180)[0] != 0: + raise GuestError("installing the bench app in the seed failed") + if simctl("get_app_container", u, BUNDLE_ID, timeout=30)[0] != 0: + raise GuestError("the bench app is not installed in the seed after install") + st = settle_wait(a.settle_max, a.bg_pct, a.step, a.consecutive, icons_of=u) + if not st["settled"]: + raise GuestError("the seed did not settle in %g s (background %.0f%% of a core, widgets' icons %s; top: %s)" % ( + a.settle_max, st["bg_pct"], "present" if st["icons"] else "missing", top_text(st["top"]))) + if simctl("shutdown", u, timeout=120)[0] != 0: + raise GuestError("the seed did not shut down") + except BaseException: + delete_device(u) + man = load_manifest() + man["seeds"].pop(tid, None) + save_manifest(man) + raise + drop_keep(u) + res = {"seed": u, "udid": u, "name": name, "type": tid, "runtime": runtime, "runtime_version": runtime_version(runtime), "complete": True, "app_stamp": stamp, + "source_type": tid.rsplit(".", 1)[-1], "reused": False, "made_s": round(now() - t0, 1), "services_off": n, "settle_s": st["waited_s"], + "settle_bg_pct": st["bg_pct"], "slimmed_s": round(secs, 1)} + man = load_manifest() + man["seeds"][tid] = {k: res[k] for k in ("udid", "name", "type", "runtime", "runtime_version", "complete", "app_stamp", "made_s", "services_off")} + save_manifest(man) + out_json(res) + + +def cmd_clone(a): + """Bench simulators = APFS clones of the seed (no disk cost) with the seed's Slim disables copied to each UDID's folder + before its first boot.""" + here = {d["udid"]: d for d in devices()} + if a.seed not in here or not is_seed(here[a.seed]): + raise GuestError("the seed is not in the bench's set") + if here[a.seed].get("state") != "Shutdown": + simctl("shutdown", a.seed, timeout=120) # a clone is taken of a shut-down device + man = load_manifest() + made = [] + for i in range(a.start, a.start + a.n): + name = "%s-%d" % (a.prefix, i) + rc, out, err, secs = simctl("clone", a.seed, name, timeout=300) + if rc != 0: + raise GuestError("clone of the seed %s: %s" % (name, (err or out).strip()[:200])) + u = out.strip().splitlines()[-1] + man["devices"][u] = {"name": name, "kind": "clone"} + save_manifest(man) + t = now() + off = copy_overrides(a.seed, u) + k = keep_or_fail(u, a.keep_min) + made.append({"idx": i, "udid": u, "clone_s": round(secs + now() - t, 2), "services_off": off, "idle_guard": k["state"]}) + out_json({"sims": made}) + + def _boot_one(udid, res, i): t0 = now() rc, _, err, _ = simctl("boot", udid, timeout=300) @@ -542,21 +730,49 @@ def _boot_one(udid, res, i): def slim_count(udid): - """Services simslim keeps disabled for a simulator (what macbox-guest counts), 0 = stock.""" - f = "/private/var/tmp/com.apple.CoreSimulator.SimDevice.%s/disabled.plist" % udid + """Services Slim keeps disabled for a simulator: `=> true` lines of its disabled.plist (what the fleet's check counts; + 168 today). 0 = stock or no file.""" + f = os.path.join(overrides_dir(udid), "disabled.plist") if not os.path.exists(f): return 0 return run(["plutil", "-p", f], 15)[1].count("=> true") -def cmd_slim(a): - """The pool's Slim configuration on the bench's simulators: runs --cmd ({udid}) for each, then counts the - services left disabled. HYPOTHESIS: simslim finds devices of a custom device set; the result says.""" - res = [] - for u in a.udids.split(","): - rc, out, err, secs = run(["/bin/sh", "-c", a.cmd.replace("{udid}", u)], 900) - res.append({"udid": u, "rc": rc, "s": round(secs, 1), "services_off": slim_count(u), "err": (err or "").strip()[-120:] if rc else None}) - out_json({"sims": res}) +def copy_overrides(seed_udid, udid, min_services=MIN_SLIM_SERVICES): + """The Slim disables live OUTSIDE the device, keyed by UDID: a clone boots stock unless the seed's file is in its own + folder (mode 0700) before its first boot. Fails when the seed has none or the copy did not take.""" + n = slim_count(seed_udid) + if n < min_services: + raise GuestError("the seed's disabled.plist has %d services off, a production simulator needs >= %d: rebuild the seed (macbox bench seed --rebuild-seed)" % (n, min_services)) + d = overrides_dir(udid) + os.makedirs(d, exist_ok=True) + os.chmod(d, 0o700) + shutil.copyfile(os.path.join(overrides_dir(seed_udid), "disabled.plist"), os.path.join(d, "disabled.plist")) + got = slim_count(udid) + if got != n: + raise GuestError("the overrides copy for %s holds %d services off, the seed's %d: it would boot stock" % (udid, got, n)) + return got + + +def icons_path(udid): + return os.path.join(SET, udid, "data", "Library", "chronod", "icons") # exists once the widgets have settled + + +def app_stamp(): + """A hash of the installed bench app (GlassBench.app of this checkout): a seed made with another build is stale.""" + root = os.path.join(HERE, "..", "GlassBench.app") + h = hashlib.sha256() + for dp, dn, fn in os.walk(root): + dn.sort() + for f in sorted(fn): + p = os.path.join(dp, f) + h.update(os.path.relpath(p, root).encode()) + try: + with open(p, "rb") as fh: + h.update(fh.read()) + except OSError: + pass + return h.hexdigest()[:16] def cmd_boot(a): @@ -581,6 +797,8 @@ def cmd_prepare(a): for u, role in zip(us, roles): t0 = now() rc_i = simctl("install", u, app, timeout=120)[0] if a.install else 0 + if a.check_app and simctl("get_app_container", u, BUNDLE_ID, timeout=30)[0] != 0: + raise GuestError("the bench app is not installed in %s: the seed is stale, rebuild it (--rebuild-seed)" % u) simctl("status_bar", u, "override", "--time", "9:41", "--batteryState", "charged", "--batteryLevel", "100", "--cellularBars", "4", "--wifiBars", "3", "--operatorName", "", timeout=30) if a.quiesce: @@ -602,29 +820,46 @@ def parse_name_cpu(text): SETTLE_SKIP = re.compile(r"(^|/)(SimMetalHost|backboardd|GlassBench|SimRenderServer|kernel_task|simfps|WindowServer|python3?|Python|ps|top|ssh|sshd)( |$)") -def bg_cpu_pct(secs=3.0): - """Everything not part of the load: CPU % of one core used by processes other than the renderers, the app and - the observer (the settle condition: the guest has stopped doing its own housekeeping).""" - skip = SETTLE_SKIP +def bg_sample(secs=3.0): + """(CPU % of one core used by everything that is not the load or its observers, the top processes by CPU in the same + interval as [{"cmd": basename, "cpu_pct", "load": True when it is part of the load}]): the settle condition (the + guest has stopped doing its own housekeeping) and what to show when it never does.""" a = parse_ps(ps_text()) mine = own_pids(a) t0 = now() time.sleep(secs) b = parse_ps(ps_text()) - dt = now() - t0 + dt = max(now() - t0, 0.001) am = {p["pid"]: p for p in a} - tot = sum(max(0.0, p["cpu_s"] - am[p["pid"]]["cpu_s"]) for p in b if p["pid"] in am and p["pid"] not in mine and not skip.search(p["cmd"])) - return round(100 * tot / dt, 1) + rows = [(max(0.0, p["cpu_s"] - am[p["pid"]]["cpu_s"]), p["cmd"]) for p in b if p["pid"] in am and p["pid"] not in mine] + tot = sum(d for d, c in rows if not SETTLE_SKIP.search(c)) + rows.sort(key=lambda r: -r[0]) + top = [{"cmd": os.path.basename(c.split(" /")[0].strip()) or c[:30], "cpu_pct": round(100 * d / dt, 1), "load": bool(SETTLE_SKIP.search(c))} + for d, c in rows[:6] if d > 0] + return round(100 * tot / dt, 1), top -def cmd_settle(a): - t0, series = now(), [] +def bg_cpu_pct(secs=3.0): + return bg_sample(secs)[0] + + +def settle_wait(max_s, bg_pct, step, consecutive=2, icons_of=None): + """Waits until the background CPU is under bg_pct for `consecutive` samples IN A ROW (and, for a seed, the widgets' + icons exist). {"settled", "bg_pct" (last sample), "waited_s", "series", "top" (at the last sample), "icons"}. Reaching + max_s is settled False: the caller fails the run, it never measures on top of this.""" + t0, series, top, icons = now(), [], [], True while True: - v = bg_cpu_pct(a.step) + v, top = bg_sample(step) series.append(v) - if v < a.bg_pct or now() - t0 > a.max_s: - break - out_json({"settled": series[-1] < a.bg_pct, "bg_pct": series[-1], "waited_s": round(now() - t0, 1), "series": series[-8:]}) + icons = icons_of is None or os.path.isdir(icons_path(icons_of)) + if len(series) >= consecutive and all(x < bg_pct for x in series[-consecutive:]) and icons: + return {"settled": True, "bg_pct": v, "waited_s": round(now() - t0, 1), "series": series[-8:], "top": top, "icons": icons} + if now() - t0 > max_s: + return {"settled": False, "bg_pct": v, "waited_s": round(now() - t0, 1), "series": series[-8:], "top": top, "icons": icons} + + +def cmd_settle(a): + out_json(settle_wait(a.max_s, a.bg_pct, a.step, a.consecutive)) def run_window(a, udids, roles, mix, secs, t_go, tmp, probe_method, probe_every, cmd_tpl, no_strip=False, probe_timeout=2.0): @@ -771,17 +1006,36 @@ def cmd_shots(a): def cmd_cycle(a): - """One lifecycle of a simulator: create, boot, install, launch, first app frame, shut down, resume - (boot again), first frame again, delete. Timings in seconds.""" + """One lifecycle of a simulator, timings in seconds. Production path (--seed): clone the seed (+ the Slim disables and the + idle-guard keep), boot (a clone's first boot), launch, first app frame, shut down, resume (boot again), first frame + again, delete. With --type/--runtime (the explicit stock path): create a fresh stock device instead, install the app, + and clone the installed device once more.""" from macbench import analysis tmp = tempfile.mkdtemp(prefix="benchcycle") t = {} udid = None + man = load_manifest() try: name = "%s-cycle" % a.prefix t0 = now() - udid = simctl("create", name, a.type, a.runtime)[1].strip().splitlines()[-1] - t["create_s"] = round(now() - t0, 2) + if a.seed: + if {d["udid"]: d for d in devices()}.get(a.seed, {}).get("state") != "Shutdown": + simctl("shutdown", a.seed, timeout=120) + rc, out, err, _ = simctl("clone", a.seed, name, timeout=300) + if rc != 0: + raise GuestError("clone of the seed: %s" % (err or out).strip()[:200]) + udid = out.strip().splitlines()[-1] + man["devices"][udid] = {"name": name, "kind": "clone"} + save_manifest(man) + copy_overrides(a.seed, udid) + keep_or_fail(udid, a.keep_min) + t["clone_s"] = round(now() - t0, 2) + else: + udid = simctl("create", name, a.type, a.runtime)[1].strip().splitlines()[-1] + man["devices"][udid] = {"name": name, "kind": "stock"} + save_manifest(man) + keep_or_fail(udid, a.keep_min) + t["create_s"] = round(now() - t0, 2) for phase in ("cold", "resume"): if phase == "resume": t0 = now() @@ -791,7 +1045,7 @@ def cmd_cycle(a): simctl("boot", udid, timeout=300) simctl("bootstatus", udid, timeout=600) t["%s_boot_s" % phase] = round(now() - t0, 2) - if phase == "cold": + if phase == "cold" and not a.seed: t0 = now() simctl("install", udid, os.path.join(HERE, "..", "GlassBench.app"), timeout=120) t["install_s"] = round(now() - t0, 2) @@ -805,28 +1059,35 @@ def cmd_cycle(a): frames = analysis.parse_raw(open(raw).read()) if os.path.exists(raw) else [] first = next((x[0] for x in frames if x[0] >= t_l and x[1] >= 0 and x[1] != 65535), None) t["%s_first_frame_s" % phase] = None if first is None else round(first - t_l, 2) - # clone of the shut-down, installed device (what a pool does for a new lease), then its boot - simctl("shutdown", udid, timeout=120) - t0 = now() - cl = simctl("clone", udid, "%s-clone" % name, timeout=300)[1].strip().splitlines()[-1] - t["clone_s"] = round(now() - t0, 2) - try: + if not a.seed: + # clone of the shut-down, installed device (what a pool does for a new lease), then its boot + simctl("shutdown", udid, timeout=120) t0 = now() - simctl("boot", cl, timeout=300) - simctl("bootstatus", cl, timeout=600) - t["clone_boot_s"] = round(now() - t0, 2) - finally: - simctl("shutdown", cl, timeout=120) - simctl("delete", cl, timeout=120) + cl = simctl("clone", udid, "%s-clone" % name, timeout=300)[1].strip().splitlines()[-1] + t["clone_s"] = round(now() - t0, 2) + man = load_manifest() + man["devices"][cl] = {"name": "%s-clone" % name, "kind": "stock"} + save_manifest(man) + try: + t0 = now() + simctl("boot", cl, timeout=300) + simctl("bootstatus", cl, timeout=600) + t["clone_boot_s"] = round(now() - t0, 2) + finally: + simctl("shutdown", cl, timeout=120) + simctl("delete", cl, timeout=120) + else: + simctl("shutdown", udid, timeout=120) t0 = now() - simctl("delete", udid, timeout=120) + problems = delete_device(udid) t["delete_s"] = round(now() - t0, 2) udid = None + if problems: + raise GuestError("; ".join(problems)) out_json({"timings": t}) finally: if udid: - simctl("shutdown", udid, timeout=60) - simctl("delete", udid, timeout=60) + delete_device(udid) shutil.rmtree(tmp, ignore_errors=True) @@ -984,18 +1245,22 @@ def add(name, fn, *opts): add("ensure-type", cmd_ensure_type, O("--scale", default="1"), O("--allow-restart", action="store_true")) add("create", cmd_create, O("--n", type=int, required=True), O("--type", required=True), O("--runtime", required=True), O("--prefix", default="bench"), O("--start", type=int, default=0), O("--keep-min", type=float, default=60)) - add("slim", cmd_slim, O("--udids", required=True), O("--cmd", required=True)) + add("seed", cmd_seed, O("--type", required=True), O("--source-set"), O("--simslim"), O("--slim-profile"), O("--keep-min", type=float, default=60), + O("--rebuild", action="store_true"), O("--settle-max", type=float, default=600), O("--bg-pct", type=float, default=40), + O("--step", type=float, default=10), O("--consecutive", type=int, default=2)) + add("clone", cmd_clone, O("--seed", required=True), O("--n", type=int, required=True), O("--prefix", default="bench"), O("--start", type=int, default=0), + O("--keep-min", type=float, default=60)) add("boot", cmd_boot, O("--udids", required=True), O("--parallel", action="store_true")) add("prepare", cmd_prepare, O("--udids", required=True), O("--roles", required=True), O("--install", action="store_true"), - O("--quiesce", action="store_true")) - add("settle", cmd_settle, O("--max-s", type=float, default=120), O("--bg-pct", type=float, default=40), O("--step", type=float, default=3)) + O("--check-app", action="store_true"), O("--quiesce", action="store_true")) + add("settle", cmd_settle, O("--max-s", type=float, default=120), O("--bg-pct", type=float, default=40), O("--step", type=float, default=3), O("--consecutive", type=int, default=2)) add("calibrate", cmd_calibrate, O("--udid", required=True), O("--methods", default="openurl,appearance"), O("--probe-cmd"), O("--time-scale", type=float, default=1.0)) add("measure", cmd_measure, O("--udids", required=True), O("--roles", required=True), O("--mix", default="all-busy"), O("--secs", type=float, default=40), O("--start-at", type=float, default=0), O("--probe-method", default="openurl"), O("--probe-every", type=float, default=2.5), O("--probe-cmd"), O("--agent-period", type=float, default=12), O("--no-strip", action="store_true"), O("--probe-timeout", type=float, default=2.0)) add("shots", cmd_shots, O("--udids", required=True), O("--count", type=int, default=2)) - add("cycle", cmd_cycle, O("--type", required=True), O("--runtime", required=True), O("--prefix", default="bench")) + add("cycle", cmd_cycle, O("--seed"), O("--type"), O("--runtime"), O("--prefix", default="bench"), O("--keep-min", type=float, default=60)) add("acquire", cmd_acquire, O("--run", default="?")) add("teardown", cmd_teardown, O("--seeds", action="store_true", help="delete the seeds too")) add("build", cmd_build, O("--xcodebuild", default="")) diff --git a/src/bench/guest/benchguest_stub.py b/src/bench/guest/benchguest_stub.py index 43491ac..e799e1b 100644 --- a/src/bench/guest/benchguest_stub.py +++ b/src/bench/guest/benchguest_stub.py @@ -4,7 +4,9 @@ simulators), crash@N (a new crash report at N booted), backwards@N (a frame shown out of order at N), slow@N (frame rate collapses from N), hang_simfps (simfps never answers), nomarker (the input marker never flips), cantdelete (simctl delete fails: a cleanup that cannot finish), noguard (no sim-idle-guard on the Mac), keepfail -(sim-idle-guard keep fails). +(sim-idle-guard keep fails), nosource (Simlock has no source device of the type), nosourceset (no Simlock set at all), +slimfail (simslim fails), slimfew (simslim leaves too few services off), noicons (the widgets never settle: no chronod icons), +noinstall (simctl install does nothing), unsettled@N (a stray process burns 1.7 cores from N booted simulators). The files a simulator leaves outside its device (Slim overrides, ~/.sim-keep, the idle guard) live under $BENCHGUEST_STATE too, so the bench's real file code runs against them.""" import json @@ -80,6 +82,35 @@ def stub_run(argv, timeout=120, env=None): return _stub_run(argv) +SRC_UDIDS = ["5A5A5A5A-0000-4000-8000-00000000000%d" % i for i in range(4)] +SRC_TYPES = ["com.apple.CoreSimulator.SimDeviceType.iPhone-17", "com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro", + "com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro", "dev.macbox.CoreSimulator.SimDeviceType.iPhone-17-Pro-1x"] + + +def make_source_set(path): + """Simlock's source set as the bench sees it: one folder per device with a device.plist (an iPhone 17 Pro shut down and one booted, its 1x type, another type).""" + import plistlib + os.makedirs(path, exist_ok=True) + for i, (typ, state) in enumerate(zip(SRC_TYPES, (1, 1, 3, 1))): + d = os.path.join(path, SRC_UDIDS[i]) + if os.path.exists(d): + continue # made once: a run must find it exactly as it was + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, "device.plist"), "wb") as f: + plistlib.dump({"UDID": SRC_UDIDS[i], "deviceType": typ, "runtime": "stub.runtime.iOS-26-5", "name": "src%d" % i, "state": state}, f) + + +def _stub_source(argv, s): + """`xcrun simctl --set ...`: ONLY clone into another set is allowed; anything else is recorded as a violation.""" + cmd = argv[4:] + if cmd[0] != "clone" or len(cmd) != 4 or cmd[1] not in SRC_UDIDS: + s.setdefault("source_violations", []).append(cmd) + return 1, "", "stub: Simlock's set may only be cloned from" + u = str(uuid.uuid4()).upper() + s["devs"][u] = {"name": cmd[2], "state": "Shutdown", "type": SRC_TYPES[SRC_UDIDS.index(cmd[1])], "apps": []} + return 0, u + "\n", "" + + def _stub_run(argv): _log(argv) s = load() @@ -94,7 +125,30 @@ def _stub_run(argv): with open(os.path.join(_MOD.KEEP_DIR, argv[2]), "w") as f: f.write("%s\n" % argv[3]) return 0, "", "", 0.01 - if argv[:3] == ["xcrun", "simctl", "--set"]: + if argv[:3] == ["xcrun", "simctl", "--set"] and argv[3] == _MOD.SOURCE_SET: + rc, out, err = _stub_source(argv, s) + elif argv and argv[0] == _MOD.SIMSLIM: + # simslim --set on --profile : shuts the device down, rewrites its services, reboots it + u = argv[4] + if fault("slimfail") is not None: + rc, err = 1, "simslim: injected failure" + elif s["devs"].get(u, {}).get("state") != "Booted": + rc, err = 1, "simslim: device is not booted" + else: + import plistlib + os.makedirs(_MOD.overrides_dir(u), exist_ok=True) + n = 20 if fault("slimfew") is not None else 168 + with open(os.path.join(_MOD.overrides_dir(u), "disabled.plist"), "wb") as f: + plistlib.dump({"com.apple.stub.service%d" % i: True for i in range(n)}, f) + elif argv and argv[0] == "plutil": + import plistlib + try: + with open(argv[2], "rb") as f: + d = plistlib.load(f) + out = "{\n" + "".join(' "%s" => %s\n' % (k, "true" if v is True else "false") for k, v in d.items()) + "}\n" + except (OSError, ValueError): + rc, err = 1, "plutil: no such file" + elif argv[:3] == ["xcrun", "simctl", "--set"]: cmd = argv[4:] verb = cmd[0] if verb == "list": @@ -105,15 +159,27 @@ def _stub_run(argv): out = json.dumps(RUNTIMES if cmd[1] == "runtimes" else TYPES) elif verb == "create": u = str(uuid.uuid4()).upper() - s["devs"][u] = {"name": cmd[1], "state": "Shutdown", "type": cmd[2]} + s["devs"][u] = {"name": cmd[1], "state": "Shutdown", "type": cmd[2], "apps": []} out = u + "\n" elif verb == "clone": - u = str(uuid.uuid4()).upper() - s["devs"][u] = {"name": cmd[2], "state": "Shutdown", "type": s["devs"][cmd[1]].get("type")} - out = u + "\n" + src = s["devs"][cmd[1]] + if src["state"] != "Shutdown": + rc, err = 1, "Unable to clone a device in state: %s" % src["state"] # HYPOTHESIS about real simctl + else: + u = str(uuid.uuid4()).upper() + s["devs"][u] = {"name": cmd[2], "state": "Shutdown", "type": src.get("type"), "apps": list(src.get("apps", []))} + if os.path.isdir(os.path.join(argv[3], cmd[1], "data")): + import shutil + shutil.copytree(os.path.join(argv[3], cmd[1], "data"), os.path.join(argv[3], u, "data")) # the clone has the seed's data + out = u + "\n" elif verb == "boot": time.sleep(0.05) s["devs"][cmd[1]]["state"] = "Booted" + if fault("noicons") is None: + os.makedirs(os.path.join(argv[3], cmd[1], "data", "Library", "chronod", "icons"), exist_ok=True) + elif verb == "install": + if fault("noinstall") is None: + s["devs"][cmd[1]].setdefault("apps", []).append("dev.macbox.glassbench") elif verb == "shutdown": for u in (list(s["devs"]) if cmd[1] == "all" else [cmd[1]]): if u in s["devs"]: @@ -122,8 +188,10 @@ def _stub_run(argv): if fault("cantdelete") is not None: err, rc = "Unable to delete (injected)", 1 else: + import shutil for u in (list(s["devs"]) if cmd[1] == "all" else [cmd[1]]): s["devs"].pop(u, None) + shutil.rmtree(os.path.join(argv[3], u), ignore_errors=True) elif verb == "launch": s["mode"][cmd[1]] = "static" if "-static" in cmd else "busy" elif verb == "terminate": @@ -138,6 +206,9 @@ def _stub_run(argv): with open(p, "wb") as f: f.write(png.write_png(rows, 402, 874, 3)) elif verb == "get_app_container": + if "dev.macbox.glassbench" not in s["devs"].get(cmd[1], {}).get("apps", []): + save(s) + return 1, "", "No such app", 0.01 d = _p("container") os.makedirs(os.path.join(d, "Documents"), exist_ok=True) u = cmd[1] @@ -260,6 +331,18 @@ def install(mod): with open(mod.IDLE_GUARD, "w") as f: f.write("#!/bin/sh\n") os.chmod(mod.IDLE_GUARD, 0o755) + mod.SOURCE_SET = _p("simlock-sources") + if fault("nosourceset") is None: + make_source_set(mod.SOURCE_SET) + if fault("nosource") is not None: + import shutil + for u in SRC_UDIDS[1:]: + shutil.rmtree(os.path.join(mod.SOURCE_SET, u)) + mod.SIMSLIM = _p("bin/simslim") + os.makedirs(_p("bin"), exist_ok=True) + with open(mod.SIMSLIM, "w") as f: + f.write("#!/bin/sh\n") + os.chmod(mod.SIMSLIM, 0o755) mod.run = stub_run mod.ps_text = stub_ps mod.vmstat_text = stub_vmstat diff --git a/src/bench/macbench/cli.py b/src/bench/macbench/cli.py index 7a43552..49711d1 100644 --- a/src/bench/macbench/cli.py +++ b/src/bench/macbench/cli.py @@ -39,8 +39,12 @@ def add_common(p): p.add_argument("--lead", type=float, default=None, help=argparse.SUPPRESS) p.add_argument("--settle-step", type=float, default=3.0, help=argparse.SUPPRESS) p.add_argument("--settle-bg", type=float, default=40, help="the guest counts as settled below this background CPU %% (of one core)") - p.add_argument("--slim", action="store_true", help="make the bench simulators Slim like the pool's (simslim; a minute or two each)") - p.add_argument("--slim-cmd", default="$HOME/go/bin/simslim on {udid} --preserve-boot-state", help=argparse.SUPPRESS) + p.add_argument("--stock-devices", action="store_true", help="create FRESH STOCK devices instead of cloning the production seed: about 1.8 GB of guest disk each, not what an agent gets; for a device type Simlock has no source for or a scenario that needs a stock device (recorded in the result)") + p.add_argument("--rebuild-seed", action="store_true", help="delete the bench's seed and make a new one (after `macbox bench tools` rebuilt the app, or when it cannot be reused)") + p.add_argument("--source-set", default=os.environ.get("BENCH_SIMLOCK_SOURCES"), help="Simlock's source device set in the guest, read only (default ~/.simlock04/devices/ios-sources)") + p.add_argument("--simslim", default=os.environ.get("BENCH_SIMSLIM"), help="simslim in the guest (default ~/go/bin/simslim)") + p.add_argument("--slim-profile", default=os.environ.get("BENCH_SLIM_PROFILE"), help="simslim profile in the guest (default ~/.config/simslim/atlas.json)") + p.add_argument("--seed-settle-max", type=float, default=600, help="the seed must settle within this many seconds or it is deleted and the run fails") p.add_argument("--quiesce", action="store_true", help="disable mediaanalysisd in each simulator (recorded; production does not)") p.add_argument("--fail-on-swap", action="store_true", help="a guest swap-out in a window fails the memory gate (default: reported, host swap fails)") p.add_argument("--golden-file", default=os.path.join(BENCH_DIR, "baselines", "golden.json")) @@ -58,7 +62,7 @@ def parser(): s.add_argument("--shots", type=int, default=2, help="screenshots per simulator per step for the tear gate (0: none)") s.add_argument("--no-observer-check", dest="observer_check", action="store_false", help="skip the strip-read cost window at the first N") s.add_argument("--no-golden", dest="golden_phase", action="store_false", help="skip the static reference screen at the first N") - for name, hlp in (("idle", "the VM with no simulators: the standing cost"), ("lifecycle", "create, boot, install, launch, shut down, resume, delete; leak per cycle; boot storm"), + for name, hlp in (("seed", "make the production seed (or report the one that is there)"), ("idle", "the VM with no simulators: the standing cost"), ("lifecycle", "create, boot, install, launch, shut down, resume, delete; leak per cycle; boot storm"), ("soak", "N busy simulators for minutes: memory growth, frame rate drift"), ("build", "build timing: the bench tools, optionally an xcodebuild"), ("panel", "the Device panel's stream (optional, needs Simlock/T3 Code)"), ("golden", "record the static screen's pixel hash for this setup"), ("ab", "A/B one knob, A B B A inside one window, ADOPT or CLOSE")): @@ -275,7 +279,7 @@ def main(argv=None): for t in a.targets + ["floor: " + a.floor]: targets.parse_target(t) return execute(a, "scale", scale.run_scale) - body = {"idle": others.run_idle, "lifecycle": others.run_lifecycle, "soak": others.run_soak, "build": others.run_build, + body = {"seed": others.run_seed, "idle": others.run_idle, "lifecycle": others.run_lifecycle, "soak": others.run_soak, "build": others.run_build, "panel": others.run_panel, "golden": others.run_golden, "ab": ab.run_ab}[c] for k, v in (("ns", [1]), ("mix", "all-busy"), ("targets", targets.DEFAULT_TARGETS), ("floor", "fps>=20"), ("shots", 2), ("observer_check", False), ("golden_phase", False)): diff --git a/src/bench/macbench/core.py b/src/bench/macbench/core.py index b8ebb37..9ddaae9 100644 --- a/src/bench/macbench/core.py +++ b/src/bench/macbench/core.py @@ -23,6 +23,9 @@ def __init__(self, args, run, guest, host, coherent, doc, out_path, clean, log=p self.marks = {} self.golden = None # fn(w, h) -> (hash or None, key) self.pixel_gate = None + self.stock = False # --stock-devices: fresh stock devices instead of clones of the seed + self.seed = None # the guest's answer to `seed` + self.prefix = "bench" self.taken = False # the bench's set is owned by this run (lock taken, cleanup registered) self.idle_guard = set() # what sim-idle-guard keep answered for the bench's simulators: kept / absent @@ -117,45 +120,99 @@ def setup_guest(ctx, sims=True): raise Stop("device type %s was just created and CoreSimulatorService must restart to load it, which would shut " "down every booted simulator: stop them, then run again with --allow-restart (or `macbox pool create`)" % t["type"]) ctx.type_id, ctx.runtime = t["type"], t["runtime"] - ctx.doc["env"]["guest"]["runtime_used"] = t.get("runtime_version") - ctx.doc["env"]["guest"]["device_type"] = "stock %s" % t["stock_name"] if a.scale_x == 3 else "macbox %dx of %s" % (a.scale_x, t["stock_name"]) + base = "stock %s" % t["stock_name"] if a.scale_x == 3 else "macbox %dx of %s" % (a.scale_x, t["stock_name"]) + g = ctx.doc["env"]["guest"] + g["runtime_used"] = t.get("runtime_version") + g["device_type"] = base + (" (fresh stock devices)" if a.stock_devices else " (production seed)") + g["device_source"] = "stock devices" if a.stock_devices else "clones of a seed made from a Simlock source device" + ctx.stock = bool(a.stock_devices) + ctx.prefix = "bench-%s" % ctx.doc["id"] _run_mark(ctx) + if not ctx.stock: + make_seed(ctx) return info +def make_seed(ctx): + """The production seed: a settled Slim clone of Simlock's own source device of this type, kept in the bench's set + (reused when it is there). Every bench simulator is a clone of it. Fails closed: no source, no Slim, no settle, no run.""" + a = ctx.a + reason = guard(ctx, 1, "before the seed") + if reason: + raise Stop(reason) + args = ["seed", "--type", ctx.type_id, "--keep-min", keep_minutes(ctx), "--settle-max", a.seed_settle_max, "--step", a.settle_step] + for flag, val in (("--source-set", a.source_set), ("--simslim", a.simslim), ("--slim-profile", a.slim_profile)): + if val: + args += [flag, val] + if a.rebuild_seed: + args.append("--rebuild") + r = ctx.guest.call(args, a.seed_settle_max + 2400) + if not r.get("seed"): + raise Stop("%s. A bench simulator must be a production simulator (a clone of a Simlock source device with the Slim disables), " + "so there is nothing to run on. For a device type Simlock has no source for, or a scenario that needs a stock device, " + "use --stock-devices: it creates fresh stock devices (about 1.8 GB of guest disk each, the host-disk guard checks " + "before each) and the result says so" % r.get("no_source", "no seed")) + ctx.seed = r + ctx.runtime = r.get("runtime") or ctx.runtime + g = ctx.doc["env"]["guest"] + g["runtime_used"] = r.get("runtime_version") or g.get("runtime_used") + g["seed"] = {k: r.get(k) for k in ("source_type", "reused", "made_s", "services_off", "settle_s", "settle_bg_pct")} + ctx.section("seed", "INFO", "SEED %s: %s services off (Slim), %s" % ( + "reused" if r.get("reused") else "made from a Simlock source device in %s s (simslim %s s, settled in %s s at %s%% background)" % ( + r.get("made_s"), r.get("slimmed_s"), r.get("settle_s"), r.get("settle_bg_pct")), + r.get("services_off"), "kept shut down between runs" if not r.get("reused") else "settled when it was made")) + + def _run_mark(ctx): ctx.offset = ctx.guest.clock_offset() ctx.run_mark = {"coh": ctx.coh.mark(), "t0": time.time(), "ips": set(ctx.guest.call(["ips"], 60)["ips"])} -def grow(ctx, n, roles, install=True): - """Make and boot simulators up to n (the ones already running stay), launch each with its role. Returns boot timings.""" +def grow(ctx, n, roles, install=True, parallel=False): + """Make and boot simulators up to n (the ones already running stay), launch each with its role. Returns boot timings. + Production path: clones of the seed (no disk cost, the seed's Slim disables copied to each). --stock-devices: fresh + stock devices, one at a time with the guard (host disk, guest memory) in front of each. parallel=True boots all the + new ones at once (the lifecycle's boot storm); otherwise they boot one after the other with the guard between.""" have = len(ctx.sims) new = n - have boot = {} if new > 0: - r = ctx.guest.call(["create", "--n", new, "--type", ctx.type_id, "--runtime", ctx.runtime, "--start", have, "--keep-min", keep_minutes(ctx)], 240) - fresh = r["sims"] - ctx.idle_guard.update(x.get("idle_guard") for x in fresh) - us = ",".join(s["udid"] for s in fresh) - if ctx.a.slim: - sl = ctx.guest.call(["slim", "--udids", us, "--cmd", ctx.a.slim_cmd], 900 * len(fresh)) - bad = [x for x in sl["sims"] if x["rc"] != 0 or not x["services_off"]] + keep = keep_minutes(ctx) + done, usable = [], [] + t0 = time.time() + if parallel and not ctx.stock: + r = ctx.guest.call(["clone", "--seed", ctx.seed["seed"], "--n", new, "--start", have, "--prefix", ctx.prefix, "--keep-min", keep], 240 + 60 * new) + fresh = r["sims"] + b = ctx.guest.call(["boot", "--udids", ",".join(x["udid"] for x in fresh)] + ["--parallel"], 900) + groups = [(fresh, b)] + else: + groups = [] + for k in range(new): + if k: + reason = guard(ctx, 1, "before booting simulator %d" % (have + k + 1)) + if reason: + raise Stop(reason) + if ctx.stock: + r = ctx.guest.call(["create", "--n", 1, "--type", ctx.type_id, "--runtime", ctx.runtime, "--start", have + k, "--keep-min", keep], 240) + else: + r = ctx.guest.call(["clone", "--seed", ctx.seed["seed"], "--n", 1, "--start", have + k, "--prefix", ctx.prefix, "--keep-min", keep], 240) + groups.append((r["sims"], ctx.guest.call(["boot", "--udids", r["sims"][0]["udid"]], 900))) + for fresh, b in groups: + ctx.idle_guard.update(x.get("idle_guard") for x in fresh) + bad = [s for s in b["sims"] if not s.get("ok")] if bad: - raise Stop("--slim: simslim did not slim a bench simulator (rc %s, %s services off): %s" % (bad[0]["rc"], bad[0]["services_off"], bad[0]["err"])) - ctx.doc["env"]["guest"]["slim_services_off"] = sl["sims"][0]["services_off"] - b = ctx.guest.call(["boot", "--udids", us] + (["--parallel"] if new > 1 else []), 900) - bad = [s for s in b["sims"] if not s.get("ok")] - if bad: - raise Stop("a simulator did not boot: %s" % bad[0].get("err")) - for s, bs in zip(fresh, b["sims"]): - ctx.sims.append({"idx": s["idx"], "udid": s["udid"]}) - boot = {"new": new, "all_usable_s": b["all_s"], "each_usable_s": [s["usable_s"] for s in b["sims"]]} + for s_, bs in zip(fresh, b["sims"]): + ctx.sims.append({"idx": s_["idx"], "udid": s_["udid"]}) # known to the cleanup and the count even when it failed + raise Stop("a simulator did not boot: %s" % bad[0].get("err")) + for s_, bs in zip(fresh, b["sims"]): + ctx.sims.append({"idx": s_["idx"], "udid": s_["udid"]}) + usable.append(bs["usable_s"]) + boot = {"new": new, "all_usable_s": round(time.time() - t0, 2) if not parallel else groups[0][1]["all_s"], "each_usable_s": usable} # roles may change with N (one-busy: only the first animates); (re)launch every simulator whose role changed or is new for i, s in enumerate(ctx.sims): want = roles[i] if s.get("role") != want: - ctx.guest.call(["prepare", "--udids", s["udid"], "--roles", want] + (["--install"] if not s.get("installed") else []) + + ctx.guest.call(["prepare", "--udids", s["udid"], "--roles", want] + (["--install"] if ctx.stock and not s.get("installed") else ["--check-app"]) + (["--quiesce"] if ctx.a.quiesce else []), 300) s["role"], s["installed"] = want, True return boot diff --git a/src/bench/macbench/others.py b/src/bench/macbench/others.py index 3110dff..a289483 100644 --- a/src/bench/macbench/others.py +++ b/src/bench/macbench/others.py @@ -13,6 +13,15 @@ def _fmt(x, unit=""): return "n/a" if x is None else "%g%s" % (x, unit) +def run_seed(ctx): + """macbox bench seed [--rebuild-seed]: make (or check and report) the production seed and nothing else.""" + info = core.setup_guest(ctx) + core.capture_env(ctx, info) + ctx.doc["data"] = {"seed": ctx.doc["env"]["guest"].get("seed")} + ctx.section("seed_ready", "PASS", "SEED READY: %s" % [s for s in ctx.doc["sections"] if s["name"] == "seed"][-1]["line"] if not ctx.stock else "SEED SKIP: --stock-devices makes no seed") + ctx.save() + + def run_idle(ctx): a = ctx.a info = core.setup_guest(ctx, sims=False) @@ -52,7 +61,7 @@ def run_lifecycle(ctx): mem0 = ctx.guest.call(["memcheck"], 60)["mem"] data = ctx.doc["data"] = {"cycles": cyc, "storm": None} for i in range(a.cycles): - r = ctx.guest.call(["cycle", "--type", ctx.type_id, "--runtime", ctx.runtime], 900) + r = ctx.guest.call(["cycle", "--keep-min", core.keep_minutes(ctx)] + (["--type", ctx.type_id, "--runtime", ctx.runtime] if ctx.stock else ["--seed", ctx.seed["seed"]]), 900) cyc.append(r["timings"]) m = ctx.guest.call(["memcheck"], 60) xs.append(i + 1) @@ -65,9 +74,11 @@ def run_lifecycle(ctx): med = {k: _median([c[k] for c in cyc if c.get(k) is not None]) for k in keys} data["median_s"] = med ok = all(v is not None for v in med.values()) and bool(cyc) - ctx.section("lifecycle", "PASS" if ok else "FAIL", "LIFECYCLE %s (%d cycles, medians, s): create %s, cold boot %s, install %s, launch call %s, first app frame %s, shutdown %s, resume boot %s, first frame after resume %s, clone %s, clone boot %s, delete %s" % ( - "PASS" if ok else "FAIL", len(cyc), med.get("create_s"), med.get("cold_boot_s"), med.get("install_s"), med.get("cold_launch_call_s"), - med.get("cold_first_frame_s"), med.get("shutdown_s"), med.get("resume_boot_s"), med.get("resume_first_frame_s"), med.get("clone_s"), med.get("clone_boot_s"), med.get("delete_s"))) + names = (("create_s", "create"), ("clone_s", "clone of the seed" if not ctx.stock else "clone"), ("cold_boot_s", "cold boot"), ("install_s", "install"), + ("cold_launch_call_s", "launch call"), ("cold_first_frame_s", "first app frame"), ("shutdown_s", "shutdown"), ("resume_boot_s", "resume boot"), + ("resume_first_frame_s", "first frame after resume"), ("clone_boot_s", "clone boot"), ("delete_s", "delete")) + ctx.section("lifecycle", "PASS" if ok else "FAIL", "LIFECYCLE %s (%d cycles, medians, s; %s): %s" % ( + "PASS" if ok else "FAIL", len(cyc), "fresh stock devices" if ctx.stock else "clones of the seed", ", ".join("%s %s" % (label, med[k]) for k, label in names if k in med))) leaks = {"qemu_rss": gates.leak_gate("QEMU RSS", xs, rss, a.leak_rss_mb, "MB/cycle"), "guest_used": gates.leak_gate("guest used memory", xs, used, a.leak_guest_mb, "MB/cycle")} data["leak"] = {"qemu_rss_mb": rss, "guest_used_mb": used} @@ -81,7 +92,7 @@ def run_lifecycle(ctx): ctx.section("storm", "SKIP", "BOOT STORM SKIP: " + reason) else: t0 = time.time() - boot = core.grow(ctx, n, ["busy"] * n) + boot = core.grow(ctx, n, ["busy"] * n, parallel=True) usable = time.time() - t0 data["storm"] = {"n": n, "boot_all_s": boot.get("all_usable_s"), "each_s": boot.get("each_usable_s"), "boot_and_launch_s": round(usable, 1)} ok = usable <= a.storm_bar_s diff --git a/src/bench/macbench/schema.py b/src/bench/macbench/schema.py index cc3b6e0..1432eda 100644 --- a/src/bench/macbench/schema.py +++ b/src/bench/macbench/schema.py @@ -6,7 +6,7 @@ import socket SCHEMA = 1 -KINDS = ("scale", "ab", "idle", "lifecycle", "build", "soak", "panel", "gates") +KINDS = ("scale", "ab", "idle", "lifecycle", "build", "soak", "panel", "gates", "seed") SECTION_STATUS = ("PASS", "FAIL", "SKIP", "INFO") _PATH = re.compile(r"(?:^|[\s\"'=:(,])(/(?:srv|home|Users|var|tmp|private|etc|opt|mnt|root|usr|run|Volumes|proc|sys)/[^\s\"',)]*)") diff --git a/src/bench/tests/test_dry.py b/src/bench/tests/test_dry.py index 3f54588..ba3b1cd 100644 --- a/src/bench/tests/test_dry.py +++ b/src/bench/tests/test_dry.py @@ -108,14 +108,27 @@ def test_clean_sweep(self): self.assertIn("guest_rss_mb", sim) def test_every_simctl_call_names_the_bench_set(self): + """Every simctl call names the bench's own set; the one exception is the clone OUT of Simlock's source set (a read of + the source, written into the bench's set), and nothing else may name Simlock's set.""" for k in ("clean", "onebusy", "lifecycle", "agent"): - calls = simctl_calls(self.r[k]["state"]) + state = self.r[k]["state"] + bench, src = os.path.join(state, "guest", "set"), os.path.join(state, "guest", "simlock-sources") + calls = simctl_calls(state) self.assertTrue(calls, k) + clones_from_source = 0 for c in calls: if c[2:4] == ["list", "devicetypes"] or c[2:4] == ["list", "runtimes"] or c[2:5] == ["list", "devices", "booted"]: continue # global, read only self.assertEqual(c[2], "--set", (k, c)) - + if c[3] == src: + self.assertEqual((c[4], len(c), c[-1]), ("clone", 8, bench), (k, c)) # clone , nothing else + clones_from_source += 1 + else: + self.assertEqual(c[3], bench, (k, c)) + self.assertEqual(clones_from_source, 1, k) # ONE seed for the device type, whatever N is + with open(os.path.join(state, "guest", "state.json")) as f: + self.assertEqual(json.load(f).get("source_violations", []), [], k) + def test_memory_floor_stops_the_sweep_and_keeps_the_data(self): r = self.r["memfloor"] d = r["doc"] @@ -126,11 +139,12 @@ def test_memory_floor_stops_the_sweep_and_keeps_the_data(self): self.assertIn("host free memory", self.r["hostmem"]["doc"]["data"]["summary"]["stopped"]) def test_a_sweep_that_ran_no_step_fails(self): - r = self.r["refuse"] + r = self.r["refuse"] # the guard refuses before the seed is even made: nothing ran self.assertEqual(r["doc"]["data"]["steps"], []) self.assertNotEqual(r["rc"], 0) - self.assertEqual(sections(r["doc"], "scale")[0]["status"], "FAIL") - self.assertIn("no step ran", sections(r["doc"], "scale")[0]["line"]) + st = sections(r["doc"], "stopped")[0] + self.assertEqual(st["status"], "FAIL") + self.assertIn("guest available memory", st["line"]) self.assertEqual(sims_booted(r["state"]), []) def test_steps_record_whether_the_guest_settled(self): diff --git a/src/bench/tests/test_dry_seed.py b/src/bench/tests/test_dry_seed.py new file mode 100644 index 0000000..db16cab --- /dev/null +++ b/src/bench/tests/test_dry_seed.py @@ -0,0 +1,264 @@ +"""The production seed, in DRY: made from a Simlock source device once, reused, never touching Simlock's set; every bench +simulator a clone of it with the seed's Slim disables; the explicit stock path; every way a seed can fail.""" +import json +import os +import shutil +import tempfile +import unittest + +from guest import benchguest_stub as stub +from tests.test_dry import FAST, run_case, sections +from tests.test_dry_state import calls, files + +ONE = ["scale", "--n", "1,2"] + FAST +_CACHE = {} + + +def parallel(jobs): + """Runs {name: fn} in threads once and caches the results: every DRY run is its own process, they do not share anything + but their own state directory.""" + import concurrent.futures + todo = {k: f for k, f in jobs.items() if k not in _CACHE} + with concurrent.futures.ThreadPoolExecutor(8) as ex: + futs = {k: ex.submit(f) for k, f in todo.items()} + for k, f in futs.items(): + _CACHE[k] = f.result() + return {k: _CACHE[k] for k in jobs} + + +def verbs(state, word): + return [c for c in calls(state) if word in c] + + +def devs(state): + with open(os.path.join(state, "guest", "state.json")) as f: + return json.load(f)["devs"] + + +def manifest(state): + with open(os.path.join(state, "guest", "set", ".bench-manifest.json")) as f: + return json.load(f) + + +def new_state(): + return tempfile.mkdtemp(prefix="dry-seed-") + + +def job_seed2(): + st = new_state() + r1 = run_case(ONE, "", state=st) + n1 = len([c for c in calls(st) if c[0].endswith("simslim")]) + return {"state": st, "r1": r1, "n_slim1": n1, "r2": run_case(ONE, "", state=st)} + + +def job_fail(fault, args=None): + def f(): + st = new_state() + return {"state": st, "r": run_case(args or ONE, fault, state=st), "next": None} + return f + + +def job_fail_then_ok(fault, args=None): + def f(): + st = new_state() + r = run_case(args or ONE, fault, state=st) + snap = {"devs": devs(st), "keep": files(st, "sim-keep")} + return {"state": st, "r": r, "snap": snap, "next": run_case(ONE, "", state=st)} + return f + + +def job_stale(): + st = new_state() + out = {"state": st, "first": run_case(ONE, "", state=st)} + p = os.path.join(st, "guest", "set", ".bench-manifest.json") + m = manifest(st) + for t in m["seeds"].values(): + t["app_stamp"] = "stale" + with open(p, "w") as f: + json.dump(m, f) + out["stale"] = run_case(ONE, "", state=st) + out["rebuilt"] = run_case(ONE + ["--rebuild-seed"], "", state=st) + return out + + +def job_seedcmd(): + st = new_state() + out = {"state": st, "seed": run_case(["seed", "--secs", "3"], "", state=st)} + out["ndevs"] = len(devs(st)) + out["cleanup"] = run_case(["cleanup", "--seeds"], "", state=st) + return out + + +JOBS = { + "seed2": job_seed2, + "nosource": job_fail_then_ok("nosource"), + "nosourceset": job_fail("nosourceset"), + "slimfail": job_fail_then_ok("slimfail"), + "slimfew": job_fail_then_ok("slimfew"), + "noicons": job_fail_then_ok("noicons", ONE + ["--seed-settle-max", "2"]), + "noinstall": job_fail_then_ok("noinstall"), + "stale": job_stale, + "seedcmd": job_seedcmd, + "stock": lambda: (lambda st: {"state": st, "r": run_case(ONE + ["--stock-devices"], "nosourceset", state=st)})(new_state()), + "lifecycle": lambda: (lambda st: {"state": st, "r": run_case(["lifecycle", "--cycles", "2", "--storm", "2", "--secs", "3"], "", state=st)})(new_state()), +} + + +class Seed(unittest.TestCase): + @classmethod + def setUpClass(cls): + d = parallel(JOBS)["seed2"] + cls.state, cls.r1, cls.r2, cls.n_slim1 = d["state"], d["r1"], d["r2"], d["n_slim1"] + + def test_the_seed_is_made_once_from_the_simlock_source_then_reused(self): + self.assertEqual((self.r1["rc"], self.r2["rc"]), (0, 0), self.r1["out"][-500:] + self.r2["out"][-500:]) + self.assertEqual(self.n_slim1, 1) + self.assertEqual(len([c for c in calls(self.state) if c[0].endswith("simslim")]), 1) # the second run did not make another + src = os.path.join(self.state, "guest", "simlock-sources") + from_source = [c for c in calls(self.state) if c[:4] == ["xcrun", "simctl", "--set", src]] + self.assertEqual(len(from_source), 1) + self.assertEqual(from_source[0][4:6], ["clone", "5A5A5A5A-0000-4000-8000-000000000003"]) # the 1x type's source + self.assertIn("made from a Simlock source device", sections(self.r1["doc"], "seed")[0]["line"]) + self.assertIn("reused", sections(self.r2["doc"], "seed")[0]["line"]) + self.assertFalse(self.r1["doc"]["env"]["guest"]["seed"]["reused"]) + self.assertTrue(self.r2["doc"]["env"]["guest"]["seed"]["reused"]) + + def test_the_seed_is_kept_shut_down_and_everything_else_is_gone(self): + d = devs(self.state) + self.assertEqual([v["name"] for v in d.values()], ["bench-seed-iPhone-17-Pro-1x"]) + self.assertEqual({v["state"] for v in d.values()}, {"Shutdown"}) + self.assertEqual(manifest(self.state)["seeds"]["dev.macbox.CoreSimulator.SimDeviceType.iPhone-17-Pro-1x"]["complete"], True) + self.assertEqual(files(self.state, "sim-keep"), []) + ov = files(self.state, "overrides") # only the seed's own Slim disables stay + self.assertEqual(len(ov), 1) + self.assertEqual(ov[0].split(".")[-1], list(d)[0]) + + def test_simlocks_set_is_read_only(self): + src = os.path.join(self.state, "guest", "simlock-sources") + fresh = tempfile.mkdtemp(prefix="dry-src-") + stub.make_source_set(os.path.join(fresh, "s")) + + def tree(root): + out = [] + for dp, _, fs in os.walk(root): + for f in fs: + with open(os.path.join(dp, f), "rb") as fh: + out.append((os.path.relpath(os.path.join(dp, f), root), fh.read())) + return sorted(out) + self.assertEqual(tree(src), tree(os.path.join(fresh, "s"))) + with open(os.path.join(self.state, "guest", "state.json")) as f: + self.assertEqual(json.load(f).get("source_violations", []), []) + for c in calls(self.state): + self.assertNotIn("config.json", " ".join(c)) + if c[:2] == ["xcrun", "simctl"] and c[3] == src: + self.assertEqual(c[4], "clone") # never boot, shutdown, delete, create... in it + + def test_every_clone_has_the_seeds_slim_disables_before_its_first_boot(self): + seed = list(devs(self.state))[0] + cs = calls(self.state) + boots = [c[-1] for c in cs if c[:2] == ["xcrun", "simctl"] and c[4:5] == ["boot"]] + clone_udids = [u for u in boots if u != seed] + self.assertTrue(clone_udids) + for u in clone_udids: + boot_at = next(i for i, c in enumerate(cs) if c[4:6] == ["boot", u]) + checked = [i for i, c in enumerate(cs) if c[0] == "plutil" and c[-1].endswith("SimDevice.%s/disabled.plist" % u)] + self.assertTrue(checked and checked[0] < boot_at, u) + for st in self.r1["doc"]["data"]["steps"]: + self.assertEqual(len(st["sims"]), st["n"]) + + def test_the_seed_is_slimmed_after_one_boot_and_before_the_app(self): + cs = calls(self.state) + kinds = [] + for c in cs: + if c[0].endswith("simslim"): + kinds.append("slim") + elif c[:2] == ["xcrun", "simctl"] and c[4:5] == ["boot"]: + kinds.append("boot") + elif c[:2] == ["xcrun", "simctl"] and c[4:5] == ["install"]: + kinds.append("install") + self.assertEqual(kinds[:3], ["boot", "slim", "install"]) + slim = next(c for c in cs if c[0].endswith("simslim")) + self.assertEqual(slim[1:4], ["--set", os.path.join(self.state, "guest", "set"), "on"]) + self.assertIn("--profile", slim) + + +class SeedFailures(unittest.TestCase): + def failing(self, key, needle): + d = parallel(JOBS)[key] + r = d["r"] + self.assertNotEqual(r["rc"], 0, r["out"][-500:]) + self.assertIn(needle, r["out"]) + self.assertEqual(d["snap"]["devs"], {}, "a failed seed must not stay in the set") # deleted again: never reusable + self.assertEqual(d["snap"]["keep"], []) + self.assertEqual(r["doc"]["data"]["steps"], []) + self.assertEqual(d["next"]["rc"], 0, d["next"]["out"][-500:]) # and the next run is not blocked + return r + + def test_no_source_device_of_the_type_says_so_and_offers_the_stock_path(self): + r = self.failing("nosource", "Simlock has no source device of type iPhone-17-Pro-1x (it has: iPhone-17)") + self.assertIn("--stock-devices", r["out"]) + self.assertEqual([c for c in calls(r["state"]) if "create" in c], []) # it did not fall back to stock devices by itself + + def test_no_simlock_at_all_says_so(self): + r = parallel(JOBS)["nosourceset"]["r"] + self.assertNotEqual(r["rc"], 0) + self.assertIn("Simlock is optional", r["out"]) + self.assertIn("--stock-devices", r["out"]) + + def test_simslim_failure(self): + self.failing("slimfail", "simslim failed on the seed") + + def test_a_seed_that_is_not_slim_enough_fails(self): + self.failing("slimfew", "not Slim after simslim: 20 services off") + + def test_a_seed_whose_widgets_never_settle_fails(self): + self.failing("noicons", "the seed did not settle") + + def test_the_app_must_be_in_the_seed(self): + self.failing("noinstall", "not installed in the seed") + + +class SeedLifecycle(unittest.TestCase): + def test_a_stale_app_stops_the_run_and_rebuild_seed_fixes_it(self): + d = parallel(JOBS)["stale"] + self.assertEqual(d["first"]["rc"], 0) + self.assertNotEqual(d["stale"]["rc"], 0) + self.assertIn("--rebuild-seed", d["stale"]["out"]) + self.assertEqual(d["rebuilt"]["rc"], 0, d["rebuilt"]["out"][-500:]) + self.assertEqual(len([c for c in calls(d["state"]) if c[0].endswith("simslim")]), 2) + self.assertFalse(d["rebuilt"]["doc"]["env"]["guest"]["seed"]["reused"]) + + def test_the_seed_command_and_cleanup_seeds(self): + d = parallel(JOBS)["seedcmd"] + self.assertEqual(d["seed"]["rc"], 0, d["seed"]["out"][-500:]) + self.assertEqual(d["ndevs"], 1) + self.assertEqual(d["cleanup"]["rc"], 0, d["cleanup"]["out"]) + self.assertEqual(devs(d["state"]), {}) + self.assertEqual(files(d["state"], "overrides"), []) + + def test_stock_devices_are_an_explicit_path_and_say_so(self): + d = parallel(JOBS)["stock"] + r, state = d["r"], d["state"] # needs no Simlock at all (nosourceset) + self.assertEqual(r["rc"], 0, r["out"][-600:]) + cs = calls(state) + self.assertEqual(len([c for c in cs if "create" in c]), 2) + self.assertEqual([c for c in cs if c[0].endswith("simslim")], []) + self.assertIn("fresh stock devices", r["doc"]["env"]["guest"]["device_type"]) + self.assertEqual(r["doc"]["env"]["guest"]["device_source"], "stock devices") + self.assertEqual(devs(state), {}) + self.assertEqual(files(state, "sim-keep"), []) + + def test_the_lifecycle_cycles_clone_the_seed(self): + d = parallel(JOBS)["lifecycle"] + r, state = d["r"], d["state"] + self.assertEqual(r["rc"], 0, r["out"][-600:]) + med = r["doc"]["data"]["median_s"] + self.assertIn("clone_s", med) + self.assertNotIn("create_s", med) + self.assertNotIn("install_s", med) + self.assertEqual([c for c in calls(state) if "create" in c], []) + self.assertEqual(len(devs(state)), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/bench/tests/test_dry_state.py b/src/bench/tests/test_dry_state.py index 908c15d..58b09aa 100644 --- a/src/bench/tests/test_dry_state.py +++ b/src/bench/tests/test_dry_state.py @@ -61,7 +61,7 @@ def test_every_bench_simulator_is_kept_for_the_run_and_the_keep_files_are_remove r = run_case(ONE, "", state=state) self.assertEqual(r["rc"], 0, r["out"][-600:]) made = {c[2] for c in keep_calls(state)} - self.assertEqual(len(made), 2) + self.assertEqual(len(made), 3) # the seed (made in this run) and the two clones for c in keep_calls(state): self.assertEqual(c[1], "keep") self.assertEqual(c[3], str(95)) # --deadline 5400 s = 90 min + 5 From fcc6b3ab6cc04d20746e6f768173460c83c8dff6 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Tue, 17 Nov 2026 05:07:33 +0200 Subject: [PATCH 14/23] bench: fail closed: a settle at its cap fails the run, the booted count before every window, a dead frame counter, foreign simulators core.settle raises with the top processes by CPU (two quiet samples in a row, --settle-max 300): the calibration settle too. core.window counts the bench set's own Booted devices first (the guest's count is what a step records as 'booted') and fails on a mismatch with the expected number; a simulator outside the bench booted before or during a window yields / makes the step unstable. A new 'counter' gate fails any window in which simfps answered nothing or an error for a simulator or an animating one read no counter (scale steps, soak, observer window). The soak gates are sections now (they set the exit code). Stub: quiet background model (the clean DRY run used to pass with an unsettled N=3), faults unsettled, lostsim, counterfail, foreign. No result file is found by pattern: a test pins it. --- src/bench/guest/benchguest.py | 21 ++++- src/bench/guest/benchguest_stub.py | 24 ++++- src/bench/macbench/ab.py | 2 +- src/bench/macbench/cli.py | 4 +- src/bench/macbench/core.py | 69 +++++++++++++-- src/bench/macbench/others.py | 11 ++- src/bench/macbench/scale.py | 13 +-- src/bench/tests/test_dry.py | 2 +- src/bench/tests/test_dry_closed.py | 136 +++++++++++++++++++++++++++++ 9 files changed, 259 insertions(+), 23 deletions(-) create mode 100644 src/bench/tests/test_dry_closed.py diff --git a/src/bench/guest/benchguest.py b/src/bench/guest/benchguest.py index 00af18d..bb4cc11 100644 --- a/src/bench/guest/benchguest.py +++ b/src/bench/guest/benchguest.py @@ -920,6 +920,9 @@ def sample_pressure(): reaped = (ru1.ru_utime + ru1.ru_stime) - (ru0.ru_utime + ru0.ru_stime) tot["observer_cpu_pct"] = round(max(tot["observer_cpu_pct"], 100 * reaped / max(t_end - t_start, 0.001)), 1) st = {d["udid"]: d.get("state") for d in devices()} + own_up = len([1 for v in st.values() if v == "Booted"]) + foreign = {"before": max(0, len([p for p in ps0 if re.search(r"(^|/)launchd_sim( |$)", p["cmd"])]) - len(udids)), + "after": max(0, len([p for p in ps1 if re.search(r"(^|/)launchd_sim( |$)", p["cmd"])]) - own_up)} sims = [] for i, u in enumerate(udids): sj, err = res.get(u, (None, None)) @@ -935,7 +938,7 @@ def sample_pressure(): "guest": dict(tot, mem=m1, mem_before=m0, swapins_d=m1["swapins"] - m0["swapins"], swapouts_d=m1["swapouts"] - m0["swapouts"], pageouts_d=m1["pageouts"] - m0["pageouts"], pressure_before=lvl0, pressure_max=max([x for x in lvls + [pressure_level()] if x] or [None]) if any(lvls) else pressure_level()), - "new_ips": sorted(list_ips() - ips0), "procs_total": len(ps1)} + "new_ips": sorted(list_ips() - ips0), "procs_total": len(ps1), "foreign_booted": foreign} def cmd_measure(a): @@ -1181,6 +1184,21 @@ def count_launchd_sim(): return len([p for p in parse_ps(ps_text()) if re.search(r"(^|/)launchd_sim( |$)", p["cmd"])]) +def cmd_precheck(a): + """Before a measurement window: how many simulators of the bench's own set are Booted (counted from the set's own + device list, the seed included), which of the expected ones are not, and how many simulators outside the bench run on + this Mac (a `launchd_sim` that is not the bench's). Renews the idle-guard keep of the expected simulators.""" + expect = [u for u in a.udids.split(",") if u] + booted = sorted(d["udid"] for d in devices() if d.get("state") == "Booted") + kept = {} + for u in expect: + if a.keep_min: + k = keep_or_fail(u, a.keep_min) + kept[k["state"]] = kept.get(k["state"], 0) + 1 + out_json({"booted": len(booted), "booted_udids": booted, "missing": [u for u in expect if u not in booted], + "extra": [u for u in booted if u not in expect], "foreign_booted": max(0, count_launchd_sim() - len(booted)), "keep": kept}) + + def cmd_ips(a): out_json({"ips": sorted(list_ips())}) @@ -1265,6 +1283,7 @@ def add(name, fn, *opts): add("teardown", cmd_teardown, O("--seeds", action="store_true", help="delete the seeds too")) add("build", cmd_build, O("--xcodebuild", default="")) add("memcheck", cmd_memcheck) + add("precheck", cmd_precheck, O("--udids", default=""), O("--keep-min", type=float, default=0)) add("ips", cmd_ips) add("calls", cmd_calls) a = ap.parse_args(argv) diff --git a/src/bench/guest/benchguest_stub.py b/src/bench/guest/benchguest_stub.py index e799e1b..c2f7115 100644 --- a/src/bench/guest/benchguest_stub.py +++ b/src/bench/guest/benchguest_stub.py @@ -6,7 +6,9 @@ cantdelete (simctl delete fails: a cleanup that cannot finish), noguard (no sim-idle-guard on the Mac), keepfail (sim-idle-guard keep fails), nosource (Simlock has no source device of the type), nosourceset (no Simlock set at all), slimfail (simslim fails), slimfew (simslim leaves too few services off), noicons (the widgets never settle: no chronod icons), -noinstall (simctl install does nothing), unsettled@N (a stray process burns 1.7 cores from N booted simulators). +noinstall (simctl install does nothing), unsettled@N (a stray process burns 1.7 cores from N booted simulators), +lostsim@N (the Mac's idle guard shuts the newest simulator down once N are booted), counterfail@N (the frame counter +of the last simulator returns nothing from N booted), foreign@N (a simulator outside the bench boots when N are booted). The files a simulator leaves outside its device (Slim overrides, ~/.sim-keep, the idle guard) live under $BENCHGUEST_STATE too, so the bench's real file code runs against them.""" import json @@ -153,6 +155,10 @@ def _stub_run(argv): verb = cmd[0] if verb == "list": if cmd[1] == "devices": + ls = fault("lostsim") + if ls is not None and len(booted(s)) >= ls and not s.get("lost") and "-j" in cmd: + s["devs"][booted(s)[-1]]["state"] = "Shutdown" # the idle guard took it + s["lost"] = 1 devs = [{"udid": u, "name": d["name"], "state": d["state"], "isAvailable": True} for u, d in s["devs"].items()] out = json.dumps({"devices": {"stub.runtime.iOS-26-5": devs}}) else: @@ -234,16 +240,23 @@ def _stub_run(argv): def stub_ps(): s = load() now = time.time() - lines = [" 1 0 4000 %s /sbin/launchd" % _cpu(now - T0, 0.05), " 100 1 60000 %s /usr/sbin/WindowServer" % _cpu(now - T0, 0.2)] + lines = [" 1 0 4000 %s /sbin/launchd" % _cpu(now - T0, 0.01), " 100 1 60000 %s /usr/sbin/WindowServer" % _cpu(now - T0, 0.2)] pid = 200 for i, u in enumerate(sorted(booted(s))): busy = s["mode"].get(u) == "busy" base = "/stub/set/%s/data" % u - lines.append("%5d 1 20000 %s launchd_sim %s" % (pid, _cpu(now - T0, 0.05), base)) - lines.append("%5d %5d 150000 %s SpringBoard" % (pid + 1, pid, _cpu(now - T0, 0.15 if busy else 0.03))) + lines.append("%5d 1 20000 %s launchd_sim %s" % (pid, _cpu(now - T0, 0.003), base)) + lines.append("%5d %5d 150000 %s SpringBoard" % (pid + 1, pid, _cpu(now - T0, 0.03 if busy else 0.01))) lines.append("%5d %5d 300000 %s GlassBench" % (pid + 2, pid, _cpu(now - T0, 0.5 if busy else 0.01))) lines.append("%5d 1 90000 %s SimMetalHost -d %s" % (pid + 3, _cpu(now - T0, 0.3 if busy else 0.02), u)) pid += 10 + n = len(booted(s)) + us = fault("unsettled") + if us is not None and n >= us: # the Maps widget's retry loop: a process that never quiets down + lines.append("%5d 1 40000 %s /usr/libexec/diagnosticd" % (900, _cpu(now - T0, 1.7))) + fo = fault("foreign") + if fo is not None and n >= fo: # someone else's simulator on the Mac, in another device set + lines.append("%5d 1 20000 %s launchd_sim /stub/other-set/FOREIGN/data" % (950, _cpu(now - T0, 0.01))) return "\n".join(lines) + "\n" @@ -272,6 +285,9 @@ def __init__(self, udid, secs, strip, start_at, raw_path): self.udid, self.secs, self.strip, self.start_at, self.raw = udid, secs, strip, start_at, raw_path def wait(self, timeout): + cf = fault("counterfail") + if cf is not None and len(booted(load())) >= cf and self.udid == sorted(booted(load()))[-1]: + return None, "simfps: no surface for the simulator (injected)" if fault("hang_simfps") is not None: return None, "simfps hung (killed after %ds)" % int(timeout) d = self.start_at + self.secs - time.time() diff --git a/src/bench/macbench/ab.py b/src/bench/macbench/ab.py index fe131c4..ea97549 100644 --- a/src/bench/macbench/ab.py +++ b/src/bench/macbench/ab.py @@ -49,7 +49,7 @@ def run_ab(ctx): ctx.host.n = n core.grow(ctx, n, core.roles_for(mix, n)) ctx.probe_method = scale.calibrate(ctx, ctx.sims[0])["chosen"] if mix in ("all-busy", "one-busy") and metric == "input_p95" else "none" - core.settle(ctx) + core.settle(ctx, "before the A/B") ctx.doc["data"] = {"knob": knob.describe(), "metric": metric, "arms": [], "n": n, "mix": mix} vals = {"A": [], "B": []} steps = {"A": [], "B": []} diff --git a/src/bench/macbench/cli.py b/src/bench/macbench/cli.py index 49711d1..7cd117a 100644 --- a/src/bench/macbench/cli.py +++ b/src/bench/macbench/cli.py @@ -35,9 +35,9 @@ def add_common(p): p.add_argument("--probe-every", type=float, default=2.5, help="seconds between input probes of one simulator") p.add_argument("--probe-timeout", type=float, default=2.0, help="an input probe not answered in this time is a miss (counted at this value)") p.add_argument("--agent-period", type=float, default=12.0) - p.add_argument("--settle-max", type=float, default=120) + p.add_argument("--settle-max", type=float, default=300, help="a settle (the guest quiet for two samples) that reaches this many seconds FAILS the run") p.add_argument("--lead", type=float, default=None, help=argparse.SUPPRESS) - p.add_argument("--settle-step", type=float, default=3.0, help=argparse.SUPPRESS) + p.add_argument("--settle-step", type=float, default=10.0, help=argparse.SUPPRESS) p.add_argument("--settle-bg", type=float, default=40, help="the guest counts as settled below this background CPU %% (of one core)") p.add_argument("--stock-devices", action="store_true", help="create FRESH STOCK devices instead of cloning the production seed: about 1.8 GB of guest disk each, not what an agent gets; for a device type Simlock has no source for or a scenario that needs a stock device (recorded in the result)") p.add_argument("--rebuild-seed", action="store_true", help="delete the bench's seed and make a new one (after `macbox bench tools` rebuilt the app, or when it cannot be reused)") diff --git a/src/bench/macbench/core.py b/src/bench/macbench/core.py index 9ddaae9..21f99d4 100644 --- a/src/bench/macbench/core.py +++ b/src/bench/macbench/core.py @@ -218,8 +218,40 @@ def grow(ctx, n, roles, install=True, parallel=False): return boot -def settle(ctx): - r = ctx.guest.call(["settle", "--max-s", ctx.a.settle_max, "--bg-pct", ctx.a.settle_bg, "--step", ctx.a.settle_step], ctx.a.settle_max + 120) +def settle(ctx, at="before the window", max_s=None, bg=None): + """Waits until the guest is quiet (background CPU under --settle-bg for two samples in a row, up to --settle-max). + A settle that reaches its cap FAILS the run: a window now would measure whatever keeps the guest busy, not the + simulators. The top processes by CPU at that moment are recorded in the failure.""" + a = ctx.a + max_s = a.settle_max if max_s is None else max_s + bg = a.settle_bg if bg is None else bg + r = ctx.guest.call(["settle", "--max-s", max_s, "--bg-pct", bg, "--step", a.settle_step, "--consecutive", 2], max_s + 120 + 4 * a.settle_step) + if not r.get("settled"): + top = ", ".join("%s %g%%" % (t["cmd"], t["cpu_pct"]) for t in r.get("top", [])[:6]) or "nothing" + line = "NOT SETTLED %s: the guest's background CPU stayed at %.0f%% of a core (bar %g%%) for %g s; top processes by CPU: %s" % ( + at, r.get("bg_pct", -1), bg, max_s, top) + ctx.section("settle", "FAIL", line, {"at": at, "series": r.get("series"), "top": r.get("top")}) + raise Stop(line) + return r + + +def precheck(ctx, what): + """Before a measurement window: count the bench's OWN booted simulators and refuse any mismatch with the expected + number (a simulator the Mac's idle guard or someone shut down would be measured as if it ran); yield when a simulator + that is not the bench's is booted. The measured count is what the result records.""" + a = ctx.a + udids = [s["udid"] for s in ctx.sims] + r = ctx.guest.call(["precheck", "--udids", ",".join(udids), "--keep-min", keep_minutes(ctx)], 90) + ctx.idle_guard.update(r.get("keep", {})) + if r.get("foreign_booted") and not a.allow_foreign: + line = "YIELDING %s: %d simulator(s) outside the bench are booted (someone needs the Mac); --allow-foreign measures anyway" % (what, r["foreign_booted"]) + ctx.section("foreign", "FAIL", line) + raise Stop(line) + if r["booted"] != len(udids) or r.get("missing") or r.get("extra"): + line = "ABORT %s: %d of the bench's simulators are booted, expected %d (%d expected one(s) not booted, %d unexpected): one was shut down under the run (the Mac's idle guard?)" % ( + what, r["booted"], len(udids), len(r.get("missing", [])), len(r.get("extra", []))) + ctx.section("booted_check", "FAIL", line, {"booted": r["booted"], "expected": len(udids)}) + raise Stop(line) return r @@ -227,6 +259,7 @@ def window(ctx, secs, mix, no_strip=False): """One measurement window over every simulator of ctx.sims: the guest measures, the host samples the same window. Returns (guest result, host system dict, window start/end in guest time).""" a = ctx.a + pre = precheck(ctx, "before the %g s window" % secs) udids = ",".join(s["udid"] for s in ctx.sims) roles = ",".join(s["role"] for s in ctx.sims) lead = a.lead if a.lead is not None else 8 + 2 * len(ctx.sims) @@ -243,6 +276,7 @@ def window(ctx, secs, mix, no_strip=False): time.sleep(max(0, secs - (time.time() - host_go))) s1 = ctx.host.snap() g = pend.wait() + g["precheck"] = pre return g, s0, s1, t_go @@ -305,14 +339,37 @@ def build_step(ctx, n, mix, g, s0, s1, t_go, boot=None): system = system_dict(ctx, g, s0, s1, frames_total) died = [s["idx"] for s in sims if g["sims"][s["idx"]].get("state_after") != "Booted"] kern, knote = ctx.host.kernel_gpu(s0["wall"]) + foreign = max((g.get("foreign_booted") or {}).values() or [0]) stab = {"new_ips": len(g["new_ips"]), "ips": g["new_ips"][:5], "sims_died": died, "kernel_gpu_lines": kern, "kernel_note": knote, - "qemu_same": ctx.host.same_qemu(), "observer_errors": [s["idx"] for s in sims if s.get("observer_error")]} - stab["ok"] = not (stab["new_ips"] or died or kern or not stab["qemu_same"]) - return {"n": n, "mix": mix, "sims": sims, "system": system, "stability": stab, "boot": boot or {}} + "qemu_same": ctx.host.same_qemu(), "observer_errors": [s["idx"] for s in sims if s.get("observer_error")], + "foreign_booted": foreign} + stab["ok"] = not (stab["new_ips"] or died or kern or not stab["qemu_same"] or (foreign and not a.allow_foreign)) + pre = g.get("precheck") or {} + # "booted" is what the guest counted in the bench's set right before the window, never the expected number + return {"n": n, "mix": mix, "booted": pre.get("booted"), "expected_booted": len(ctx.sims), "sims": sims, "system": system, + "stability": stab, "boot": boot or {}, "counter": counter_gate(g["sims"], mix)} + + +def counter_gate(gsims, mix): + """A frame counter that returns nothing or an error for a simulator fails the window: every simulator of a window with + a counter must have an answer from simfps, and an animating one must have read counters from its surfaces.""" + if mix == "idle": + return gates.gate("SKIP", "no counter in the idle mix") + bad = [] + for i, s in enumerate(gsims): + if s.get("simfps_error") or not s.get("simfps"): + bad.append("sim %d: %s" % (i, s.get("simfps_error") or "simfps returned nothing")) + elif s.get("role") in ("busy", "agent"): + rd = [r for r in analysis.parse_raw(s.get("raw") or "") if 0 <= r[1] < 65535] + if len(rd) < 2: + bad.append("sim %d: no frame counter read from its surfaces (%d reads)" % (i, len(rd))) + if bad: + return gates.gate("FAIL", "; ".join(bad[:4])) + return gates.gate("PASS", "simfps answered for %d simulator(s), counters read" % len(gsims)) def step_gates(ctx, step, shots=None): - gs = {"strip": gates.strip_gate_of(step["sims"]), "kernel_gpu": gates.kernel_gate(step["stability"]["kernel_gpu_lines"], step["stability"]["kernel_note"])} + gs = {"counter": step["counter"], "strip": gates.strip_gate_of(step["sims"]), "kernel_gpu": gates.kernel_gate(step["stability"]["kernel_gpu_lines"], step["stability"]["kernel_note"])} gs.update(ctx.coh.check(ctx.marks["coh"])) if shots: gs.update(shots) diff --git a/src/bench/macbench/others.py b/src/bench/macbench/others.py index a289483..34e35be 100644 --- a/src/bench/macbench/others.py +++ b/src/bench/macbench/others.py @@ -30,7 +30,7 @@ def run_idle(ctx): m = ctx.guest.call(["memcheck"], 60) if (m["booted"] or m.get("foreign_booted")) and not a.allow_foreign: raise Stop("simulators are booted (%d of the bench, %d outside it): an idle VM has none; --allow-foreign to measure anyway" % (m["booted"], m.get("foreign_booted", 0))) - core.settle(ctx) + core.settle(ctx, "before the idle window") g, s0, s1, t_go = core.window(ctx, a.secs, "idle") sysd = core.system_dict(ctx, g, s0, s1, 0) kern, note = ctx.host.kernel_gpu(s0["wall"]) @@ -113,7 +113,7 @@ def run_soak(ctx): raise Stop(reason) ctx.host.n = n core.grow(ctx, n, ["busy"] * n) - core.settle(ctx) + core.settle(ctx, "before the soak") res = {} for label in ("start", "end"): if label == "end": @@ -141,6 +141,11 @@ def run_soak(ctx): ctx.section("soak_fps", "PASS" if ok else "FAIL", "SOAK %s: slowest simulator %.1f fps at the start, %.1f after %g min (bar: not below 95%%)" % ("PASS" if ok else "FAIL", f0, f1, a.soak_minutes)) ctx.doc["gates"].update(ctx.coh.check(ctx.marks["coh"])) ctx.doc["gates"]["strip"] = gates.strip_gate_of(res["end"]["sims"]) + cg = [res[k]["counter"] for k in ("start", "end")] + ctx.doc["gates"]["counter"] = next((g for g in cg if g["status"] == "FAIL"), cg[-1]) + for name in ("counter", "strip"): # a failed gate is a FAIL section: it sets the exit code + g = ctx.doc["gates"][name] + ctx.section("gate:" + name, g["status"], "GATE %s %s: %s" % (g["status"], name, g["detail"])) ctx.save() @@ -205,7 +210,7 @@ def run_golden(ctx): info = core.setup_guest(ctx) core.capture_env(ctx, info) core.grow(ctx, 1, ["static"]) - core.settle(ctx) + core.settle(ctx, "before the golden screen") scale.golden_phase(ctx) seen = getattr(ctx, "golden_seen", None) if not seen: diff --git a/src/bench/macbench/scale.py b/src/bench/macbench/scale.py index 9e09497..1c488eb 100644 --- a/src/bench/macbench/scale.py +++ b/src/bench/macbench/scale.py @@ -7,7 +7,7 @@ from .runctx import DeadlineError from .transport import GuestError -STEP_GATES = ("strip", "kernel_gpu", "coherent_census", "coherent_process", "qemu_log", "gpu_hang", "tears", "pixels", "memory") +STEP_GATES = ("counter", "strip", "kernel_gpu", "coherent_census", "coherent_process", "qemu_log", "gpu_hang", "tears", "pixels", "memory") def calibrate(ctx, sim): @@ -65,11 +65,11 @@ def run_scale(ctx): if first: if mix in ("all-busy", "one-busy"): data["calibration"] = calibrate(ctx, ctx.sims[0]) - ctx.guest.call(["settle", "--max-s", 20, "--bg-pct", 90, "--step", ctx.a.settle_step], 60) + core.settle(ctx, "before the input calibration", max_s=60, bg=90) if a.golden_phase: ctx.pixel_gate = golden_phase(ctx) core.grow(ctx, n, roles) - st = core.settle(ctx) + st = core.settle(ctx, "at N=%d" % n) g, s0, s1, t_go = core.window(ctx, a.secs, mix) step = core.build_step(ctx, n, mix, g, s0, s1, t_go, boot) step["settle"] = {k: st.get(k) for k in ("settled", "bg_pct", "waited_s")} @@ -111,6 +111,8 @@ def _why_unstable(step): out.append("%d new crash report(s)" % s["new_ips"]) if s["sims_died"]: out.append("simulator(s) %s no longer booted" % s["sims_died"]) + if s.get("foreign_booted") and s["foreign_booted"]: + out.append("%d simulator(s) outside the bench were booted during the window" % s["foreign_booted"]) if s["kernel_gpu_lines"]: out.append("%d kernel GPU error line(s)" % s["kernel_gpu_lines"]) if not s["qemu_same"]: @@ -133,10 +135,11 @@ def observer_check(ctx, step, mix): g, s0, s1, t_go = core.window(ctx, ctx.a.secs, mix, no_strip=True) with_strip = [s["surface_fps"] for s in step["sims"] if s.get("role") in ("busy", "agent") and s.get("surface_fps")] without = [s["simfps"]["fps"] for s in g["sims"] if s["role"] in ("busy", "agent") and s.get("simfps")] + missing = len([s for s in g["sims"] if s["role"] in ("busy", "agent")]) - len(without) # a counter that answered nothing is not a measurement ws, wo = stats.mean(with_strip), stats.mean(without) - d = stats.rel_diff(wo, ws) + d = None if missing else stats.rel_diff(wo, ws) return {"n": step["n"], "surface_fps_with_strip": stats.r1(ws), "surface_fps_without_strip": stats.r1(wo), - "delta": None if d is None else round(d, 4), "ok": d is not None and abs(d) <= 0.04} + "delta": None if d is None else round(d, 4), "missing": missing, "ok": d is not None and abs(d) <= 0.04} def _finish(ctx, stopped, partial=False): diff --git a/src/bench/tests/test_dry.py b/src/bench/tests/test_dry.py index ba3b1cd..1158858 100644 --- a/src/bench/tests/test_dry.py +++ b/src/bench/tests/test_dry.py @@ -87,7 +87,7 @@ class Dry(unittest.TestCase): def setUpClass(cls): with concurrent.futures.ThreadPoolExecutor(8) as ex: futs = {k: ex.submit(run_case, a, f) for k, (a, f) in CASES.items()} - futs["ssh"] = ex.submit(run_case, ["scale", "--n", "1,2,3"] + FAST, "", {"BENCH_STUB_SSH_TIMEOUT": "20"}) + futs["ssh"] = ex.submit(run_case, ["scale", "--n", "1,2,3"] + FAST, "", {"BENCH_STUB_SSH_TIMEOUT": "30"}) futs["term"] = ex.submit(run_case, ["scale", "--n", "1,2,3,4,5,6"] + FAST, "", None, 35) futs["deadline"] = ex.submit(run_case, ["scale", "--n", "1,2,3,4,5,6", "--deadline", "40"] + FAST, "") cls.r = {k: f.result() for k, f in futs.items()} diff --git a/src/bench/tests/test_dry_closed.py b/src/bench/tests/test_dry_closed.py new file mode 100644 index 0000000..b46cb00 --- /dev/null +++ b/src/bench/tests/test_dry_closed.py @@ -0,0 +1,136 @@ +"""Fail closed: a settle that reaches its cap, a booted count that does not match, a frame counter that answers nothing, a +foreign simulator, and result files that are never found by pattern. Real `run` processes against the stub guest.""" +import concurrent.futures +import os +import re +import unittest + +from macbench import core +from tests.test_dry import FAST, run_case, sections, sims_booted + +HERE = os.path.dirname(os.path.abspath(__file__)) +SWEEP = ["scale", "--n", "1,2,3"] + FAST +_R = {} + + +def results(): + if not _R: + cases = {"clean": (SWEEP, ""), "unsettled": (SWEEP, "unsettled@2"), "lost": (SWEEP, "lostsim@2"), "counter": (SWEEP, "counterfail@2"), + "foreign": (SWEEP, "foreign@2"), + "ab_lost": (["ab", "--knob", "live:group_x=1", "--n", "2", "--arm-settle", "0.2", "--secs", "3", "--settle-max", "2"], "lostsim@2"), + "soak_counter": (["soak", "--soak-n", "2", "--soak-minutes", "0.2", "--soak-every", "5", "--secs", "3", "--settle-max", "2"], "counterfail@2")} + with concurrent.futures.ThreadPoolExecutor(8) as ex: + futs = {k: ex.submit(run_case, a, f) for k, (a, f) in cases.items()} + _R.update({k: f.result() for k, f in futs.items()}) + return _R + + +class SettleCap(unittest.TestCase): + def test_a_settle_that_reaches_its_cap_fails_the_run_and_names_the_busy_processes(self): + r = results()["unsettled"] + self.assertNotEqual(r["rc"], 0, r["out"][-600:]) + s = sections(r["doc"], "settle")[0] + self.assertEqual(s["status"], "FAIL") + self.assertIn("NOT SETTLED at N=2", s["line"]) + self.assertIn("diagnosticd", s["line"]) # the top processes by CPU at that moment + self.assertIn("diagnosticd", [t["cmd"] for t in s["detail"]["top"]]) + self.assertEqual([st["n"] for st in r["doc"]["data"]["steps"]], [1]) # nothing was measured on top of it + self.assertEqual(sections(r["doc"], "scale")[0]["status"], "FAIL") + self.assertEqual(sims_booted(r["state"]), []) + + def test_a_clean_sweep_settles_every_step_and_says_how_long(self): + r = results()["clean"] + self.assertEqual(r["rc"], 0, r["out"][-600:]) + for st in r["doc"]["data"]["steps"]: + self.assertTrue(st["settle"]["settled"]) + self.assertIsNotNone(st["settle"]["waited_s"]) + + +class BootedCount(unittest.TestCase): + def test_the_count_is_the_guests_not_the_expected_number(self): + for st in results()["clean"]["doc"]["data"]["steps"]: + self.assertEqual(st["booted"], st["n"]) + self.assertEqual(st["expected_booted"], st["n"]) + + def test_a_mismatch_before_the_window_fails_the_run(self): + r = results()["lost"] + self.assertNotEqual(r["rc"], 0, r["out"][-600:]) + s = sections(r["doc"], "booted_check")[0] + self.assertEqual(s["status"], "FAIL") + self.assertIn("1 of the bench's simulators are booted, expected 2", s["line"]) + self.assertEqual(s["detail"], {"booted": 1, "expected": 2}) # the measured number is recorded + self.assertEqual([st["n"] for st in r["doc"]["data"]["steps"]], [1]) + self.assertEqual(sims_booted(r["state"]), []) + + def test_ab_arms_and_every_other_window_are_counted_too(self): + r = results()["ab_lost"] + self.assertNotEqual(r["rc"], 0, r["out"][-600:]) + self.assertEqual(sections(r["doc"], "booted_check")[0]["status"], "FAIL") + self.assertEqual(r["doc"]["data"].get("arms", []), []) + live = os.path.join(r["state"], "live.conf") + with open(live) as f: + self.assertEqual(f.read(), "surface_coherent=1\n") + + def test_a_simulator_outside_the_bench_makes_the_run_yield(self): + r = results()["foreign"] + self.assertNotEqual(r["rc"], 0, r["out"][-600:]) + self.assertIn("YIELDING", sections(r["doc"], "foreign")[0]["line"]) + self.assertEqual([st["n"] for st in r["doc"]["data"]["steps"]], [1]) + self.assertEqual(sims_booted(r["state"]), []) + + +class FrameCounter(unittest.TestCase): + def test_a_counter_that_answers_nothing_fails_the_window(self): + r = results()["counter"] + self.assertNotEqual(r["rc"], 0, r["out"][-600:]) + st = r["doc"]["data"]["steps"][-1] + self.assertEqual(st["n"], 2) + self.assertEqual(st["gates"]["counter"]["status"], "FAIL") + self.assertRegex(st["gates"]["counter"]["detail"], r"sim [01]: simfps: no surface") + self.assertEqual(r["doc"]["gates"]["counter"]["status"], "FAIL") + self.assertIn("gate failed at N=2: counter", r["doc"]["data"]["summary"]["stopped"]) + self.assertEqual(sims_booted(r["state"]), []) + + def test_a_soak_window_is_judged_too(self): + r = results()["soak_counter"] + self.assertEqual(r["doc"]["gates"]["counter"]["status"], "FAIL") + self.assertEqual(sections(r["doc"], "gate:counter")[0]["status"], "FAIL") + self.assertNotEqual(r["rc"], 0, r["out"][-600:]) + + def test_gate_unit(self): + ok = {"role": "busy", "simfps": {"fps": 60}, "raw": "1.0 5 255\n1.1 6 255\n"} + self.assertEqual(core.counter_gate([ok, dict(ok, role="static", raw="1.0 -1 255\n")], "all-busy")["status"], "PASS") + for bad in (dict(ok, simfps=None), dict(ok, simfps_error="hung"), dict(ok, raw="1.0 -1 255\n1.1 -1 255\n"), dict(ok, raw="")): + self.assertEqual(core.counter_gate([ok, bad], "all-busy")["status"], "FAIL", bad) + self.assertEqual(core.counter_gate([], "idle")["status"], "SKIP") + + +class ResultFilesAreNeverFoundByPattern(unittest.TestCase): + """The base2 sweep read `f-busyK-*` and also got `f-busyK-unread-*`. Here every per-window file is opened by the exact + path its owner built (a per-call temp directory; the window's own dict), nothing looks files up by prefix or glob.""" + + def sources(self): + for d in ("../macbench", "../guest"): + for f in sorted(os.listdir(os.path.join(HERE, d))): + if f.endswith(".py"): + with open(os.path.join(HERE, d, f)) as fh: + yield f, fh.read() + + def test_no_module_selects_files_by_pattern(self): + for name, text in self.sources(): + self.assertNotRegex(text, r"\bimport glob\b|\bfrom glob\b|\bfnmatch\b|\.glob\(|\brglob\b", name) + # the only directory listings: sets' devices (a UUID pattern), crash reports (exact suffix), /proc, and the source set + allowed = {"benchguest.py": 2, "hostprobe.py": 2} + for name, text in self.sources(): + n = len(re.findall(r"os\.listdir\(|os\.scandir\(", text)) + self.assertLessEqual(n, allowed.get(name, 0), name) + + def test_two_windows_of_one_step_keep_their_own_files(self): + with open(os.path.join(HERE, "..", "guest", "benchguest.py")) as fh: + src = fh.read() + self.assertEqual(len(re.findall(r'tempfile\.mkdtemp\(prefix="benchwin"\)', src)), 1) # one fresh directory per window + self.assertIn('raws[u] = os.path.join(tmp, "raw-%d.txt" % i)', src) # exact name per simulator, kept in a dict + + +if __name__ == "__main__": + unittest.main() From 94ea330ff664aeb2cafef6d5d3392385367d6a06 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Tue, 17 Nov 2026 05:15:20 +0200 Subject: [PATCH 15/23] bench: guards: host disk floors 18/14/11, swap-outs and compressor, the VM unchanged and answering, a monitor that aborts while waiting --disk-start-gib 18 / --disk-boot-gib 14 / --disk-abort-gib 11 on --disk-path (the filesystem the VM's storage guard watches); a disk that cannot be read refuses. Guard before EVERY boot (simulators boot one at a time; the boot storm is the one exception), guest swap-outs since the run began and the compressor (--max-compressor-mb 2048) refuse. A monitor thread (Run.watch) checks the host disk and QEMU's pid AND start time always, and the Mac (ssh answers twice, someone else's simulator, swap, compressor) outside measurement windows; a breach aborts the run from that thread: partial result, the one cleanup (the second caller waits for the first), closing sections, exit 8/9/7/10. Stub: faults diskstart, disklow, diskabort, diskunreadable, qemugone, swapgrow, compressor. --- src/bench/guest/benchguest_stub.py | 9 +- src/bench/macbench/ab.py | 2 +- src/bench/macbench/cli.py | 16 ++- src/bench/macbench/core.py | 102 +++++++++++++-- src/bench/macbench/fakes.py | 19 ++- src/bench/macbench/hostprobe.py | 12 +- src/bench/macbench/runctx.py | 60 ++++++++- src/bench/macbench/scale.py | 2 +- src/bench/macbench/transport.py | 51 ++++++-- src/bench/tests/test_dry_closed.py | 3 +- src/bench/tests/test_dry_guards.py | 76 +++++++++++ src/bench/tests/test_runctx.py | 7 +- src/bench/tests/test_runguards.py | 201 +++++++++++++++++++++++++++++ 13 files changed, 520 insertions(+), 40 deletions(-) create mode 100644 src/bench/tests/test_dry_guards.py create mode 100644 src/bench/tests/test_runguards.py diff --git a/src/bench/guest/benchguest_stub.py b/src/bench/guest/benchguest_stub.py index c2f7115..45a829c 100644 --- a/src/bench/guest/benchguest_stub.py +++ b/src/bench/guest/benchguest_stub.py @@ -8,7 +8,8 @@ slimfail (simslim fails), slimfew (simslim leaves too few services off), noicons (the widgets never settle: no chronod icons), noinstall (simctl install does nothing), unsettled@N (a stray process burns 1.7 cores from N booted simulators), lostsim@N (the Mac's idle guard shuts the newest simulator down once N are booted), counterfail@N (the frame counter -of the last simulator returns nothing from N booted), foreign@N (a simulator outside the bench boots when N are booted). +of the last simulator returns nothing from N booted), foreign@N (a simulator outside the bench boots when N are booted), swapgrow@N (the guest's swap-outs +grow from N booted), compressor@N (the compressor holds 3 GB from N booted). The files a simulator leaves outside its device (Slim overrides, ~/.sim-keep, the idle guard) live under $BENCHGUEST_STATE too, so the bench's real file code runs against them.""" import json @@ -272,12 +273,16 @@ def stub_vmstat(): total_pages = 16 * 1024 * 1024 * 1024 // page used = (3000 + 1100 * n) * 1048576 // page free = max(0, total_pages - used) + sw = fault("swapgrow") + swapouts = 500 if sw is not None and n >= sw else 0 + cp = fault("compressor") + comp = 200000 if cp is not None and n >= cp else 5000 + 300 * n mf = fault("memfloor") if mf is not None and n >= mf: free = 20 * 1048576 // page return ("Mach Virtual Memory Statistics: (page size of %d bytes)\nPages free: %d.\nPages active: %d.\nPages inactive: 1000.\n" "Pages speculative: 500.\nPages wired down: %d.\nPages occupied by compressor: %d.\nPageins: 1000.\nPageouts: 0.\n" - "Swapins: 0.\nSwapouts: 0.\n" % (page, free, used // 2, used // 4, 5000 + 300 * n)) + "Swapins: 0.\nSwapouts: %d.\n" % (page, free, used // 2, used // 4, comp, swapouts)) class StubSimfps: diff --git a/src/bench/macbench/ab.py b/src/bench/macbench/ab.py index ea97549..b6af917 100644 --- a/src/bench/macbench/ab.py +++ b/src/bench/macbench/ab.py @@ -54,7 +54,7 @@ def run_ab(ctx): vals = {"A": [], "B": []} steps = {"A": [], "B": []} bad = None - ctx.run.on_deadline = lambda: (_save(ctx), None)[1] + ctx.run.on_deadline = lambda reason=None: (_save(ctx), None)[1] for arm in abrule.abba_order(): (knob.set if arm == "B" else knob.unset)() time.sleep(a.arm_settle) diff --git a/src/bench/macbench/cli.py b/src/bench/macbench/cli.py index 7cd117a..82ec00a 100644 --- a/src/bench/macbench/cli.py +++ b/src/bench/macbench/cli.py @@ -22,8 +22,14 @@ def add_common(p): p.add_argument("--build-id", default=os.environ.get("BENCH_BUILD_ID"), help="the VM's build id (macbox bench fills it in)") p.add_argument("--min-guest-free-gib", type=float, default=1.5, help="refuse to boot more simulators below this guest free memory") p.add_argument("--min-host-free-gib", type=float, default=6.0, help="refuse to boot more simulators below this host available memory") - p.add_argument("--min-disk-gib", type=float, default=12.0, help="refuse below this free disk space (the VM's disk, host side)") - p.add_argument("--disk-path", default=os.environ.get("BENCH_HOME"), help=argparse.SUPPRESS) + p.add_argument("--disk-path", default=os.environ.get("BENCH_DISK_PATH") or os.environ.get("BENCH_HOME"), + help="a path on the filesystem that holds the VM's storage and that your storage guard watches (default $BENCH_DISK_PATH, else the macbox home); a reading that fails refuses to run") + p.add_argument("--disk-start-gib", type=float, default=18.0, help="start a run only with this much free disk (GiB)") + p.add_argument("--disk-boot-gib", type=float, default=14.0, help="boot another simulator only with this much free disk (GiB)") + p.add_argument("--disk-abort-gib", type=float, default=11.0, help="below this the run is aborted and cleaned up while it waits (GiB)") + p.add_argument("--max-compressor-mb", type=float, default=2048.0, help="refuse to boot (abort while waiting) when the guest's compressor holds more than this") + p.add_argument("--watch-every", type=float, default=10.0, help=argparse.SUPPRESS) + p.add_argument("--watch-guest-every", type=float, default=30.0, help=argparse.SUPPRESS) p.add_argument("--set-name", default=os.environ.get("BENCH_SET_NAME", "bench"), help="the bench's own device set in the guest (kept between runs: the seed lives in it)") p.add_argument("--allow-foreign", action="store_true", help="go on although simulators outside the bench are booted (recorded)") p.add_argument("--allow-restart", action="store_true", help="let a NEW device type restart CoreSimulatorService (only when nothing is booted)") @@ -167,7 +173,7 @@ def build_ctx(a, kind): os.makedirs(tmp, exist_ok=True) if not os.environ.get("BENCH_DRY_KEEP"): run.teardown.insert(0, ("dry state directory", lambda: shutil.rmtree(tmp, ignore_errors=True))) - a.lead, a.settle_step = 1.0, 0.3 + a.lead, a.settle_step, a.watch_every, a.watch_guest_every = 1.0, 0.3, 0.5, 1.0 host = fakes.FakeHost(faults) coh = fakes.FakeCoherent(host, faults) guest = LocalGuest(run, set_name, os.path.join(tmp, "guest"), BENCH_DIR, faults) @@ -213,6 +219,9 @@ def execute(a, kind, body): ctx = build_ctx(a, kind) run = ctx.run run.install() + run.on_abort = lambda reason: ctx.section("abort", "FAIL", "ABORT: %s" % reason) + run.final = lambda: finalize(ctx) + core.monitor(ctx) if ctx.host.dry: log("DRY=1: stub guest, fake host; nothing real is touched") rc = 0 @@ -229,6 +238,7 @@ def execute(a, kind, body): rc = 2 finally: run.cleanup() + run.join_abort() # when a monitor aborted the run it writes the result and owns the exit code rc = max(rc, finalize(ctx)) fails = [s for s in ctx.doc["sections"] if s["status"] == "FAIL"] if fails: diff --git a/src/bench/macbench/core.py b/src/bench/macbench/core.py index 21f99d4..b937442 100644 --- a/src/bench/macbench/core.py +++ b/src/bench/macbench/core.py @@ -26,6 +26,8 @@ def __init__(self, args, run, guest, host, coherent, doc, out_path, clean, log=p self.stock = False # --stock-devices: fresh stock devices instead of clones of the seed self.seed = None # the guest's answer to `seed` self.prefix = "bench" + self.swap0 = None # the guest's swap-out counter when the run began + self.in_window = False # a measurement window is running: the monitors keep off the guest self.taken = False # the bench's set is owned by this run (lock taken, cleanup registered) self.idle_guard = set() # what sim-idle-guard keep answered for the bench's simulators: kept / absent @@ -48,16 +50,30 @@ def capture_env(ctx, info): "dry": bool(ctx.host.dry)}) +def disk_reason(ctx, floor_gib, what, label=""): + """None when the filesystem that holds the VM's storage has `floor_gib` free; else why not. A reading that fails refuses + (a guard that cannot see the disk is not a guard): that is what let the 2026-11-16 outage through.""" + df = ctx.host.disk_free_gb(ctx.a.disk_path or "/") + if df is None: + return "free disk space of the VM's storage (--disk-path) cannot be read: refusing to %s %s" % (what, label) + if df < floor_gib: + return "free disk %.1f GiB is under the %g GiB floor to %s %s" % (df, floor_gib, what, label) + return None + + def guard(ctx, k_new, label): - """Refuses to boot k_new more simulators when the guest or the host would run short: returns a reason or None. - The sweep stops there; it never swaps the VM to death.""" + """Refuses to boot k_new more simulators when the host or the guest would run short: returns a reason or None. Checked + before every boot (and by the monitor while waiting). The sweep stops there; it never swaps the VM to death. + Host: free disk (>= --disk-boot-gib to boot another), available memory. Guest: available memory after the new + simulators, memory pressure, swap-outs since the run began (must not grow), the compressor's size, simulators that + are not the bench's.""" a = ctx.a hm = ctx.host.mem_avail_gb() if hm is not None and hm - k_new * 0.5 < a.min_host_free_gib: return "host free memory %.1f GiB below the %.1f GiB floor %s" % (hm, a.min_host_free_gib, label) - df = ctx.host.disk_free_gb(a.disk_path or "/") - if df is not None and df < a.min_disk_gib: - return "free disk %.0f GiB below the %.0f GiB floor %s" % (df, a.min_disk_gib, label) + why = disk_reason(ctx, a.disk_boot_gib, "boot another simulator", label) + if why: + return why m = ctx.guest.call(["memcheck"], 60) avail = m["mem"].get("avail_mb", m["mem"]["free_mb"]) # free + speculative + inactive + purgeable: what a boot can take free_gib = (avail - k_new * ctx.est_sim_mb) / 1024.0 @@ -66,11 +82,65 @@ def guard(ctx, k_new, label): free_gib, avail / 1024.0, k_new, ctx.est_sim_mb, a.min_guest_free_gib, label) if (m.get("pressure") or 1) >= 2: return "guest memory pressure level %s %s" % (m.get("pressure"), label) + mem_why = guest_memory_reason(ctx, m["mem"]) + if mem_why: + return "%s %s" % (mem_why, label) if m.get("foreign_booted", 0) > 0 and not a.allow_foreign: return "%d simulator(s) outside the bench are booted (the pool or someone's lease): not a clean measurement; --allow-foreign to go on" % m["foreign_booted"] return None +def guest_memory_reason(ctx, mem): + """Swap-outs since the run began (the guest started paging memory out: the simulators no longer fit) and the compressor's size.""" + base = ctx.swap0 + if base is not None and mem.get("swapouts", 0) > base: + return "guest swap-outs grew from %d to %d since the run began" % (base, mem["swapouts"]) + if mem.get("compressed_mb", 0) > ctx.a.max_compressor_mb: + return "guest compressor holds %.0f MB (limit %g MB)" % (mem["compressed_mb"], ctx.a.max_compressor_mb) + return None + + +def start_reason(ctx): + """Before the first simulator of a run: the host disk has --disk-start-gib free.""" + return disk_reason(ctx, ctx.a.disk_start_gib, "start a run that boots simulators", "") + + +def monitor(ctx): + ctx.run.watch("guards", monitor_check(ctx), ctx.a.watch_every) + + +def monitor_check(ctx): + """While the run waits (a boot, a settle, a window): the host's disk, QEMU and, outside measurement windows (their + own calls would be observer load), the Mac. Any breach aborts the run NOW: the monitor thread writes the partial + result, runs the one cleanup and exits (8 disk, 9 VM, 7 someone else's simulator, 10 guest memory).""" + a, st = ctx.a, {"dead": 0, "last_guest": 0.0} + + def check(): + why = disk_reason(ctx, a.disk_abort_gib, "go on", "") + if why: + return "%s; deleting the bench's simulators" % why, 8 + if not ctx.host.same_qemu(): + return "the VM's QEMU process is gone or was restarted", 9 + now = time.monotonic() + if ctx.in_window or not ctx.taken or now - st["last_guest"] < a.watch_guest_every: + return None + st["last_guest"] = now + if not ctx.guest.alive(): + st["dead"] += 1 + if st["dead"] >= 2: + return "the VM stopped answering over ssh (two checks in a row)", 9 + return None + st["dead"] = 0 + m = ctx.guest.quick(["memcheck"], 60) + if m.get("foreign_booted", 0) > 0 and not a.allow_foreign: + return "a simulator that is not the bench's is booted (someone needs the Mac): yielding", 7 + why = guest_memory_reason(ctx, m["mem"]) + if why: + return why, 10 + return None + return check + + def roles_for(mix, n): if mix == "all-busy": return ["busy"] * n @@ -109,6 +179,9 @@ def setup_guest(ctx, sims=True): if not sims: _run_mark(ctx) return info + why = start_reason(ctx) + if why: + raise Stop(why) acq = ctx.guest.call(["acquire", "--run", ctx.doc["id"]], 60) if not acq.get("ok"): # refuse BEFORE the cleanup is registered: the leftovers are not this run's to delete (macbox bench cleanup does) @@ -164,6 +237,7 @@ def make_seed(ctx): def _run_mark(ctx): + ctx.swap0 = ctx.guest.call(["memcheck"], 60)["mem"].get("swapouts", 0) # swap-outs may not grow from here on ctx.offset = ctx.guest.clock_offset() ctx.run_mark = {"coh": ctx.coh.mark(), "t0": time.time(), "ips": set(ctx.guest.call(["ips"], 60)["ips"])} @@ -269,13 +343,17 @@ def window(ctx, secs, mix, no_strip=False): "--probe-method", ctx.probe_method or "openurl", "--probe-every", a.probe_every, "--agent-period", a.agent_period, "--probe-timeout", a.probe_timeout] if a.probe_cmd: args += ["--probe-cmd", a.probe_cmd] - pend = ctx.guest.start(args, secs + lead + 240) - host_go = t_go - ctx.offset - time.sleep(max(0, host_go - time.time())) - s0 = ctx.host.snap() - time.sleep(max(0, secs - (time.time() - host_go))) - s1 = ctx.host.snap() - g = pend.wait() + ctx.in_window = True + try: + pend = ctx.guest.start(args, secs + lead + 240) + host_go = t_go - ctx.offset + time.sleep(max(0, host_go - time.time())) + s0 = ctx.host.snap() + time.sleep(max(0, secs - (time.time() - host_go))) + s1 = ctx.host.snap() + g = pend.wait() + finally: + ctx.in_window = False g["precheck"] = pre return g, s0, s1, t_go diff --git a/src/bench/macbench/fakes.py b/src/bench/macbench/fakes.py index 692ccd1..1a29149 100644 --- a/src/bench/macbench/fakes.py +++ b/src/bench/macbench/fakes.py @@ -38,7 +38,23 @@ def mem_avail_gb(self): return 0.5 return v + def _fault_n(self, name): + for f in self.faults: + if f == name or f.startswith(name + "@"): + return int(f.split("@")[1]) if "@" in f else 0 + return None + def disk_free_gb(self, path): + """Faults: diskstart (16 GiB from the start: under the 18 to start), disklow@N (12.5 from N simulators: under the 14 to + boot another), diskabort@N (9: under the 11 to go on), diskunreadable.""" + if self._fault_n("diskunreadable") is not None: + return None + for name, val in (("diskabort", 9.0), ("disklow", 12.5)): + k = self._fault_n(name) + if k is not None and self.n >= k: + return val + if self._fault_n("diskstart") is not None: + return 16.0 return 500.0 def kernel_gpu(self, since): @@ -47,7 +63,8 @@ def kernel_gpu(self, since): return 0, "" def same_qemu(self): - return True + k = self._fault_n("qemugone") # the VM restarted: another QEMU process + return not (k is not None and self.n >= k) def env(self): return {"kernel": "fake", "mesa": "fake", "cpu_model": "fake", "cpus": 20}, {"surface_coherent": "1"}, {"REIMS_VGPU_COHERENT": "1"} diff --git a/src/bench/macbench/hostprobe.py b/src/bench/macbench/hostprobe.py index 21920f0..a5e4bb7 100644 --- a/src/bench/macbench/hostprobe.py +++ b/src/bench/macbench/hostprobe.py @@ -259,6 +259,7 @@ def __init__(self, pid, root="", live_conf=None, hz=None): self.pid, self.root, self.live_conf = pid, root, live_conf self.hz = hz or os.sysconf("SC_CLK_TCK") self.n = 0 + self.born = self._starttime() # the QEMU process as this run first saw it: a restart is a different process def qemu_pid(self): return self.pid @@ -282,8 +283,17 @@ def disk_free_gb(self, path): def kernel_gpu(self, since): return kernel_gpu_lines(since) + def _starttime(self): + t = _read(self.root, "/proc/%d/stat" % self.pid) + try: + return t[t.rindex(")") + 2:].split()[19] if t else None # field 22: start time in ticks since boot + except (ValueError, IndexError): + return None + def same_qemu(self): - return os.path.exists("%s/proc/%d" % (self.root, self.pid)) + """The QEMU process is still the one the run started with (same pid AND same start time: a restarted VM may reuse the pid).""" + now = self._starttime() + return now is not None and now == self.born def env(self): sw = read_live_conf(self.live_conf) if self.live_conf else {} diff --git a/src/bench/macbench/runctx.py b/src/bench/macbench/runctx.py index 23be89c..c2bcf5b 100644 --- a/src/bench/macbench/runctx.py +++ b/src/bench/macbench/runctx.py @@ -28,6 +28,10 @@ def __init__(self, deadline_s, log=print, margin=60.0): self._lock = threading.Lock() self.expired = threading.Event() self.problems = [] # things cleanup found it could not restore + self.aborted = None # (reason, exit code) once a monitor ended the run + self.on_abort = None # fn(reason): records the abort in the result + self.final = None # fn(): after the cleanup, writes the result's closing sections (the cleanup's verdict) + self._done = threading.Event() # --- time def remaining(self): @@ -89,11 +93,62 @@ def _watchdog(self): self.cleanup() os._exit(124) + # --- monitors: what must stop the run while a long call is waiting + def watch(self, name, fn, every): + """Calls fn() every `every` seconds on a thread until the run is over; fn returns None (all well) or + (reason, exit code): the run is then aborted: partial result, cleanup, exit.""" + def loop(): + while not self.expired.wait(every): + try: + r = fn() + except Exception as e: # a check that cannot run is not a verdict + self.log("monitor %s: %s: %s" % (name, type(e).__name__, e)) + continue + if r: + self.abort(*r) + return + t = threading.Thread(target=loop, daemon=True, name="watch-" + name) + t.start() + return t + + def abort(self, reason, code): + """End the run NOW from a monitor thread: record why, write the partial result, run the one cleanup, write the + closing sections, exit with `code`. The main thread, which may be blocked in a call, is killed with its children.""" + with self._lock: + if self.aborted or self.cleaned: + return + self.aborted = (reason, code) + self.log("ABORT: %s" % reason) + self.expired.set() + for hook, arg in ((self.on_abort, (reason,)), (self.on_deadline, (reason,))): + if hook: + try: + hook(*arg) + except Exception as e: + self.log("abort hook failed: %s" % e) + self.cleanup() + if self.final: + try: + self.final() + except Exception as e: + self.log("closing the result failed: %s" % e) + os._exit(code) + + def join_abort(self): + """Called by the main thread after its own cleanup: when a monitor aborted the run, it owns the exit.""" + if self.aborted: + time.sleep(600) + def cleanup(self): with self._lock: if self.cleaned: - return - self.cleaned = True + already = True + else: + already = False + self.cleaned = True + if already: + self._done.wait(900) # another thread is cleaning up: do not exit before it is done + return self.expired.set() self.in_cleanup = True self.kill_children() @@ -111,6 +166,7 @@ def cleanup(self): except Exception as e: self.problems.append("%s: %s" % (label, e)) self.log("restore FAILED: %s: %s" % (label, e)) + self._done.set() class LiveKnob: diff --git a/src/bench/macbench/scale.py b/src/bench/macbench/scale.py index 1c488eb..709e546 100644 --- a/src/bench/macbench/scale.py +++ b/src/bench/macbench/scale.py @@ -51,7 +51,7 @@ def run_scale(ctx): "nominal_fps": a.nominal_fps, "window_s": a.secs, "targets": tg_specs, "floor": a.floor, "steps": [], "summary": {}, "calibration": None, "observer": None} ctx.marks = {"coh": ctx.coh.mark(), "t0": time.time()} - ctx.run.on_deadline = lambda: (_finish(ctx, "deadline"), None)[1] + ctx.run.on_deadline = lambda reason="deadline": (_finish(ctx, reason), None)[1] stopped, first, kind = None, True, None try: for n in a.ns: diff --git a/src/bench/macbench/transport.py b/src/bench/macbench/transport.py index 3cc49e1..e5b682b 100644 --- a/src/bench/macbench/transport.py +++ b/src/bench/macbench/transport.py @@ -51,26 +51,27 @@ def parse(self, what, rc, out, err): raise GuestError("%s: %s" % (what, d["error"])) return d - def _check(self, what, timeout): + def _check(self, what, timeout, counted=True): if self.run.in_cleanup: return min(timeout, 240) if self.run.expired.is_set(): raise DeadlineError("stopping: %s not started" % what) if self.run.remaining() < 5: raise DeadlineError("deadline: %s not started" % what) - self.calls += 1 - if self.fault_ssh_timeout and self.calls == self.fault_ssh_timeout: - raise GuestError("%s: ssh timed out (injected)" % what) + if counted: + self.calls += 1 + if self.fault_ssh_timeout and self.calls == self.fault_ssh_timeout: + raise GuestError("%s: ssh timed out (injected)" % what) return min(timeout, max(5, int(self.run.remaining()))) - def start(self, args, timeout=300): + def start(self, args, timeout=300, counted=True): what = args[0] - timeout = self._check(what, timeout) + timeout = self._check(what, timeout, counted) p = self.run.popen(self.argv(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) return Pending(self, p, what, timeout) - def call(self, args, timeout=300): - return self.start(args, timeout).wait() + def call(self, args, timeout=300, counted=True): + return self.start(args, timeout, counted).wait() def clock_offset(self): """Guest clock minus host clock in seconds, from the quickest of 5 round trips.""" @@ -90,6 +91,25 @@ def clock_offset(self): def argv_plain(self, cmd): return ["ssh", "-o", "ConnectTimeout=10", "-o", "BatchMode=yes", self.ssh_host, cmd] + def alive(self): + """The Mac answers over ssh (a monitor's question: it does not count as one of the run's calls).""" + try: + return subprocess.run(self.argv_plain("true"), capture_output=True, timeout=20).returncode == 0 + except (OSError, subprocess.TimeoutExpired): + return False + + def _env(self): + return None + + def quick(self, args, timeout=60): + """A helper call for the monitors: not counted, never injected with the DRY faults of the run's own calls, and a + timeout kills only itself (a hung monitor must not take the run's own call down with it).""" + try: + p = subprocess.run(self.argv(args), capture_output=True, text=True, timeout=timeout, env=self._env(), cwd=getattr(self, "bench_dir", None)) + except subprocess.TimeoutExpired: + raise GuestError("%s: no answer in %ds" % (args[0], timeout)) + return self.parse(args[0], p.returncode, p.stdout, p.stderr) + def sh(self, cmd, timeout=120): """A plain command in the guest: (rc, stdout).""" if self.run.expired.is_set() and not self.run.in_cleanup: @@ -109,16 +129,21 @@ def __init__(self, run, set_path, state_dir, bench_dir, faults=""): def argv(self, args): return [sys.executable, os.path.join(self.bench_dir, "guest", "benchguest.py"), "--set", self.set_dir] + [str(a) for a in args] - def start(self, args, timeout=300): + def start(self, args, timeout=300, counted=True): what = args[0] - timeout = self._check(what, timeout) - env = dict(os.environ, BENCHGUEST_STUB="1", BENCHGUEST_STATE=self.state_dir, BENCH_STUB_FAULT=self.faults, - BENCHGUEST_CALLS=os.path.join(self.state_dir, "calls.log"), PYTHONPATH=self.bench_dir) - p = self.run.popen(self.argv(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env, cwd=self.bench_dir) + timeout = self._check(what, timeout, counted) + p = self.run.popen(self.argv(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=self._env(), cwd=self.bench_dir) return Pending(self, p, what, timeout) + def _env(self): + return dict(os.environ, BENCHGUEST_STUB="1", BENCHGUEST_STATE=self.state_dir, BENCH_STUB_FAULT=self.faults, + BENCHGUEST_CALLS=os.path.join(self.state_dir, "calls.log"), PYTHONPATH=self.bench_dir) + def clock_offset(self): return 0.0 + def alive(self): + return True + def sh(self, cmd, timeout=120): return 0, "" diff --git a/src/bench/tests/test_dry_closed.py b/src/bench/tests/test_dry_closed.py index b46cb00..81a143e 100644 --- a/src/bench/tests/test_dry_closed.py +++ b/src/bench/tests/test_dry_closed.py @@ -74,7 +74,8 @@ def test_ab_arms_and_every_other_window_are_counted_too(self): def test_a_simulator_outside_the_bench_makes_the_run_yield(self): r = results()["foreign"] self.assertNotEqual(r["rc"], 0, r["out"][-600:]) - self.assertIn("YIELDING", sections(r["doc"], "foreign")[0]["line"]) + said = " | ".join(x["line"] for x in r["doc"]["sections"] if x["status"] == "FAIL") # the precheck, the guard or the monitor: whoever sees it first + self.assertRegex(said, r"YIELDING|someone needs the Mac|outside the bench are booted") self.assertEqual([st["n"] for st in r["doc"]["data"]["steps"]], [1]) self.assertEqual(sims_booted(r["state"]), []) diff --git a/src/bench/tests/test_dry_guards.py b/src/bench/tests/test_dry_guards.py new file mode 100644 index 0000000..6d51ec5 --- /dev/null +++ b/src/bench/tests/test_dry_guards.py @@ -0,0 +1,76 @@ +"""The guards end to end in DRY: the host-disk floors (start 18, boot another 14, abort and clean up under 11), a disk that +cannot be read, a restarted VM, and the one cleanup after an abort from the monitor thread.""" +import concurrent.futures +import json +import os +import unittest + +from tests.test_dry import FAST, run_case, sections, sims_booted +from tests.test_dry_state import calls, files + +SWEEP = ["scale", "--n", "1,2,3"] + FAST +_R = {} + + +def results(): + if not _R: + cases = {"diskstart": (SWEEP, "diskstart"), "disklow": (SWEEP, "disklow@2"), "diskabort": (SWEEP, "diskabort@2"), + "diskunreadable": (SWEEP, "diskunreadable"), "qemugone": (SWEEP, "qemugone@2"), + "stockdisk": (["scale", "--n", "2", "--stock-devices"] + FAST[:], "disklow@2"), "clean": (SWEEP, "")} + with concurrent.futures.ThreadPoolExecutor(8) as ex: + futs = {k: ex.submit(run_case, a, f) for k, (a, f) in cases.items()} + _R.update({k: f.result() for k, f in futs.items()}) + return _R + + +class HostDisk(unittest.TestCase): + def test_a_run_does_not_start_under_18_gib(self): + r = results()["diskstart"] + self.assertNotEqual(r["rc"], 0) + self.assertIn("16.0 GiB is under the 18 GiB floor", sections(r["doc"], "stopped")[0]["line"]) + self.assertEqual([c for c in calls(r["state"]) if "clone" in c or "create" in c], []) # not even a seed + + def test_another_simulator_is_not_booted_under_14_gib(self): + r = results()["disklow"] + self.assertEqual([s["n"] for s in r["doc"]["data"]["steps"]], [1, 2]) # with 2 booted the disk reads 12.5: the third is refused + self.assertIn("12.5 GiB is under the 14 GiB floor to boot another simulator", r["doc"]["data"]["summary"]["stopped"]) + self.assertEqual(sims_booted(r["state"]), []) + + def test_with_stock_devices_the_guard_stands_before_each_device(self): + r = results()["stockdisk"] + creates = [c for c in calls(r["state"]) if "create" in c] + self.assertEqual(len(creates), 1) # the second device was refused before it was made + self.assertIn("under the 14 GiB floor", " ".join(s["line"] for s in r["doc"]["sections"] if s["name"] in ("stopped", "scale"))) + self.assertEqual(sims_booted(r["state"]), []) + self.assertEqual(files(r["state"], "sim-keep"), []) + + def test_a_disk_that_cannot_be_read_refuses(self): + r = results()["diskunreadable"] + self.assertNotEqual(r["rc"], 0) + self.assertIn("cannot be read", r["out"]) + + def test_under_11_gib_while_waiting_the_run_is_aborted_and_cleaned_up(self): + r = results()["diskabort"] + self.assertEqual(r["rc"], 8, r["out"][-800:]) # the monitor thread owns the exit code + ab = sections(r["doc"], "abort")[0] + self.assertEqual(ab["status"], "FAIL") + self.assertIn("9.0 GiB is under the 11 GiB floor", ab["line"]) + self.assertEqual(sections(r["doc"], "cleanup")[0]["status"], "PASS") # verified, after the abort + self.assertEqual(sims_booted(r["state"]), []) + self.assertEqual(files(r["state"], "sim-keep"), []) + self.assertEqual([s["n"] for s in r["doc"]["data"]["steps"]], [1]) # what was measured is kept + + def test_a_restarted_vm_aborts_the_run(self): + r = results()["qemugone"] + self.assertEqual(r["rc"], 9, r["out"][-800:]) + self.assertIn("QEMU", sections(r["doc"], "abort")[0]["line"]) + self.assertEqual(sims_booted(r["state"]), []) + + def test_a_clean_run_is_not_aborted(self): + r = results()["clean"] + self.assertEqual(r["rc"], 0, r["out"][-500:]) + self.assertEqual(sections(r["doc"], "abort"), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/bench/tests/test_runctx.py b/src/bench/tests/test_runctx.py index 72d62c0..d4713bc 100644 --- a/src/bench/tests/test_runctx.py +++ b/src/bench/tests/test_runctx.py @@ -94,17 +94,18 @@ def test_a_simulator_outside_the_bench_refuses_the_run(self): from macbench import core, fakes class A: - min_host_free_gib, min_disk_gib, min_guest_free_gib, allow_foreign, disk_path = 1.0, 1.0, 1.0, False, "/" + min_host_free_gib, disk_boot_gib, min_guest_free_gib, allow_foreign, disk_path = 1.0, 1.0, 1.0, False, "/" + max_compressor_mb = 2048.0 class G: def __init__(self, foreign): self.foreign = foreign def call(self, args, timeout): - return {"mem": {"free_mb": 8000, "avail_mb": 9000}, "pressure": 1, "booted": 0, "foreign_booted": self.foreign} + return {"mem": {"free_mb": 8000, "avail_mb": 9000, "swapouts": 0, "compressed_mb": 0}, "pressure": 1, "booted": 0, "foreign_booted": self.foreign} class Ctx: - a, est_sim_mb, host = A(), 1000.0, fakes.FakeHost() + a, est_sim_mb, host, swap0 = A(), 1000.0, fakes.FakeHost(), None c = Ctx() c.guest = G(1) self.assertIn("outside the bench", core.guard(c, 1, "x")) diff --git a/src/bench/tests/test_runguards.py b/src/bench/tests/test_runguards.py new file mode 100644 index 0000000..8215139 --- /dev/null +++ b/src/bench/tests/test_runguards.py @@ -0,0 +1,201 @@ +"""The guards (host disk floors, guest swap and compressor, the VM still the same and answering, someone else's simulator) and +the monitor that applies them while the run waits. Pure: fake host, fake guest, no VM.""" +import argparse +import os +import subprocess +import sys +import tempfile +import time +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..")) +from macbench import cli, core, hostprobe # noqa: E402 + + +class FakeHost: + dry = True + + def __init__(self, disk=500.0, same=True, mem=40.0): + self.disk, self.same, self.mem = disk, same, mem + + def mem_avail_gb(self): + return self.mem + + def disk_free_gb(self, path): + return self.disk + + def same_qemu(self): + return self.same + + +class FakeGuest: + def __init__(self, mem=None, foreign=0, alive=True, pressure=1): + self.mem = mem or {"avail_mb": 9000, "free_mb": 5000, "swapouts": 7, "compressed_mb": 200} + self.foreign, self.up, self.pressure, self.calls = foreign, alive, pressure, [] + + def call(self, args, timeout=60, counted=True): + self.calls.append(args[0]) + return {"mem": self.mem, "foreign_booted": self.foreign, "pressure": self.pressure, "booted": 0} + + quick = call + + def alive(self): + self.calls.append("alive") + return self.up + + +def ns(**kw): + a = cli.parser().parse_args(["scale"]) + for k, v in kw.items(): + setattr(a, k, v) + return a + + +class Ctx: + def __init__(self, host=None, guest=None, **kw): + self.a = ns(**kw) + self.host, self.guest = host or FakeHost(), guest or FakeGuest() + self.est_sim_mb, self.swap0, self.in_window, self.taken = 1100.0, 7, False, True + + +class DiskFloors(unittest.TestCase): + def test_the_defaults_are_the_outage_lessons(self): + a = ns() + self.assertEqual((a.disk_start_gib, a.disk_boot_gib, a.disk_abort_gib, a.max_compressor_mb), (18.0, 14.0, 11.0, 2048.0)) + + def test_start_boot_and_abort_floors(self): + for disk, start, boot, keep_going in ((20, True, True, True), (17.9, False, True, True), (13.9, False, False, True), (10.9, False, False, False)): + c = Ctx(host=FakeHost(disk=disk)) + self.assertEqual(core.start_reason(c) is None, start, disk) + self.assertEqual(core.guard(c, 1, "x") is None, boot, disk) + self.assertEqual(core.disk_reason(c, c.a.disk_abort_gib, "go on") is None, keep_going, disk) + + def test_an_unreadable_disk_refuses_everything(self): + c = Ctx(host=FakeHost(disk=None)) + self.assertIn("cannot be read", core.start_reason(c)) + self.assertIn("cannot be read", core.guard(c, 1, "x")) + self.assertIn("cannot be read", monitor_result(c)[0]) + self.assertEqual(monitor_result(c)[1], 8) + + +class GuestGuards(unittest.TestCase): + def test_swapouts_may_not_grow_and_the_compressor_is_capped(self): + self.assertIsNone(core.guard(Ctx(), 1, "x")) + c = Ctx(guest=FakeGuest(mem={"avail_mb": 9000, "free_mb": 5000, "swapouts": 8, "compressed_mb": 200})) + self.assertIn("swap-outs grew from 7 to 8", core.guard(c, 1, "x")) + c = Ctx(guest=FakeGuest(mem={"avail_mb": 9000, "free_mb": 5000, "swapouts": 7, "compressed_mb": 2100})) + self.assertIn("compressor holds 2100 MB", core.guard(c, 1, "x")) + + def test_someone_elses_simulator_refuses_a_boot_unless_allowed(self): + self.assertIn("outside the bench", core.guard(Ctx(guest=FakeGuest(foreign=1)), 1, "x")) + self.assertIsNone(core.guard(Ctx(guest=FakeGuest(foreign=1), allow_foreign=True), 1, "x")) + + +def monitor_result(ctx): + ctx.a.watch_guest_every = 0.0 + return core.monitor_check(ctx)() + + +class Monitor(unittest.TestCase): + def test_quiet_run(self): + self.assertIsNone(monitor_result(Ctx())) + + def test_disk_under_the_abort_floor_ends_the_run_with_8(self): + r = monitor_result(Ctx(host=FakeHost(disk=10.5))) + self.assertEqual(r[1], 8) + self.assertIn("under the 11 GiB floor", r[0]) + + def test_a_restarted_qemu_ends_the_run_with_9(self): + self.assertEqual(monitor_result(Ctx(host=FakeHost(same=False)))[1], 9) + + def test_a_vm_that_stops_answering_twice_in_a_row_ends_the_run_with_9(self): + c = Ctx(guest=FakeGuest(alive=False)) + check = core.monitor_check(c) + c.a.watch_guest_every = 0.0 + self.assertIsNone(check()) # one miss is not a verdict + self.assertEqual(check()[1], 9) + c = Ctx(guest=FakeGuest(alive=False)) + c.a.watch_guest_every = 0.0 + check = core.monitor_check(c) + check() + c.guest.up = True + self.assertIsNone(check()) # an answer resets the count + c.guest.up = False + self.assertIsNone(check()) + + def test_someone_elses_simulator_makes_the_run_yield_with_7(self): + self.assertEqual(monitor_result(Ctx(guest=FakeGuest(foreign=1)))[1], 7) + self.assertIsNone(monitor_result(Ctx(guest=FakeGuest(foreign=1), allow_foreign=True))) + + def test_swap_growth_and_the_compressor_end_the_run_with_10(self): + self.assertEqual(monitor_result(Ctx(guest=FakeGuest(mem={"swapouts": 9, "compressed_mb": 1})))[1], 10) + self.assertEqual(monitor_result(Ctx(guest=FakeGuest(mem={"swapouts": 7, "compressed_mb": 9999})))[1], 10) + + def test_a_measurement_window_is_left_alone(self): + c = Ctx(guest=FakeGuest(foreign=1)) + c.in_window = True + self.assertIsNone(monitor_result(c)) + self.assertEqual(c.guest.calls, []) # no observer load: the guest was not asked + c.host.disk = 5 + self.assertEqual(monitor_result(c)[1], 8) # but the host-side checks go on + + +class QemuIdentity(unittest.TestCase): + def test_reads_a_real_process(self): + h = hostprobe.Host(os.getpid()) + self.assertIsNotNone(h.born) + self.assertTrue(h.same_qemu()) + + def test_same_pid_with_another_start_time_is_another_vm(self): + root = tempfile.mkdtemp() + os.makedirs(os.path.join(root, "proc", "42")) + + def stat(start): + with open(os.path.join(root, "proc", "42", "stat"), "w") as f: + f.write("42 (qemu-system-x86) S " + " ".join(["1"] * 18) + " %s 5\n" % start) + stat(1000) + h = hostprobe.Host(42, root=root, hz=100) + self.assertTrue(h.same_qemu()) + stat(1000) + self.assertTrue(h.same_qemu()) + stat(7777) + self.assertFalse(h.same_qemu()) + os.remove(os.path.join(root, "proc", "42", "stat")) + self.assertFalse(h.same_qemu()) + + +SCRIPT = r''' +import sys, time, threading +sys.path.insert(0, %r) +from macbench.runctx import Run +r = Run(60, print, margin=2) +r.teardown.append(("t", lambda: (time.sleep(0.5), print("TEARDOWN RAN", flush=True)))) +r.on_abort = lambda reason: print("ABORT RECORDED", reason, flush=True) +r.on_deadline = lambda reason=None: print("PARTIAL WRITTEN", reason, flush=True) +r.final = lambda: print("FINAL", flush=True) +r.install() +t0 = time.time() +r.watch("x", lambda: ("disk gone", 8) if time.time() - t0 > 0.3 else None, 0.1) +try: + time.sleep(30) # the main thread is blocked in a "call" +finally: + r.cleanup() # and runs the one cleanup too when it is woken: it must wait for the monitor's + r.join_abort() +''' + + +class AbortFromAMonitor(unittest.TestCase): + def test_partial_result_cleanup_then_exit_code(self): + t0 = time.time() + p = subprocess.run([sys.executable, "-c", SCRIPT % os.path.join(HERE, "..")], capture_output=True, text=True, timeout=40) + self.assertEqual(p.returncode, 8, p.stdout + p.stderr) + out = p.stdout + self.assertLess(time.time() - t0, 20) + for word in ("disk gone", "PARTIAL WRITTEN", "TEARDOWN RAN", "FINAL"): + self.assertIn(word, out) + self.assertLess(out.index("TEARDOWN RAN"), out.index("FINAL")) # the closing sections come after the cleanup + + +if __name__ == "__main__": + unittest.main() From 759a6be6ffb49fd023716d8324d315b8220e656b Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Tue, 17 Nov 2026 05:19:46 +0200 Subject: [PATCH 16/23] bench: README for production simulators, the guards and the new commands; check lists Simlock, simslim and the idle guard; compare warns on mixed device sources The seed's simslim profile is the only .json in the simslim config folder unless --slim-profile / BENCH_SLIM_PROFILE says otherwise (no machine-specific name in the repo). --- src/bench/README.md | 124 ++++++++++++++++++++++------- src/bench/guest/benchguest.py | 45 ++++++++++- src/bench/guest/benchguest_stub.py | 4 + src/bench/macbench/cli.py | 7 +- src/bench/macbench/compare.py | 3 + src/bench/tests/test_dry_closed.py | 5 +- src/bench/tests/test_modules.py | 22 +++++ 7 files changed, 176 insertions(+), 34 deletions(-) diff --git a/src/bench/README.md b/src/bench/README.md index da461aa..efc67a0 100644 --- a/src/bench/README.md +++ b/src/bench/README.md @@ -7,7 +7,9 @@ single-purpose tools it builds on (`simfps`, GlassBench, GenieBench) are describ ```sh macbox bench tools # once: copy the bench to the Mac, build simfps and GlassBench there macbox bench check # what this host and the VM offer +macbox bench seed # once: the production seed every bench simulator is cloned from (kept between runs) macbox bench scale --n 1,2,3,4,5,6 # the scaling sweep (the centrepiece) +macbox bench cleanup # after a run that could not clean up (the next run refuses to start until then) macbox bench compare baseline.json new.json macbox bench chart new.json # an HTML/SVG chart next to it DRY=1 macbox bench scale --n 1,2,3 # the whole control flow against stubs: no VM, no simulator, no GPU @@ -19,34 +21,94 @@ and `python3 -m unittest tests.test_dry` (the dry runs with injected faults, a c ## What a run does, and what it never touches -The bench has a **device set of its own** (`simctl --set`, a directory with a marker file), so it never sees the -pool's simulators or anyone's lease; every `simctl` the guest helper starts goes through one function that adds -`--set` and refuses a set without the marker. It creates its own devices, and at the end shuts down and deletes -all of them. The 3x device is the stock iPhone Pro; the **1x type** is the macbox 1x device type `macbox pool` -makes (same identifier, reused when it exists; made by the bench when it does not, and then CoreSimulatorService -must restart to load it, which closes every booted simulator: the bench refuses that unless nothing is booted -and `--allow-restart` is given). It starts no tracer. It reads `/proc`, `/sys`, the kernel journal and the VM's +The bench has a **device set of its own** (`simctl --set`, a directory with a marker file, the same one for every run so +its seed is kept), so it never sees the pool's simulators or anyone's lease; every `simctl` the guest helper starts goes +through one function that adds `--set` and refuses a set without the marker. The one exception is the clone out of +Simlock's source set (below), which is read-only for Simlock. At the end of a run the bench shuts down and deletes every +device it made except the seed. The 3x device is the stock iPhone Pro; the **1x type** is the macbox 1x device type +`macbox pool` makes (same identifier, reused when it exists; made by the bench when it does not, and then +CoreSimulatorService must restart to load it, which closes every booted simulator: the bench refuses that unless nothing +is booted and `--allow-restart` is given). It starts no tracer. It reads `/proc`, `/sys`, the kernel journal and the VM's census and log; it writes only its result file, and, with `ab --knob live:KEY=VALUE`, the one line in the VM's live switch file (restored byte for byte on every way out). -Safety, in every run: the **hard deadline** (`--deadline`, default 90 minutes) stops the run 60 s early, writes the -result so far and runs the cleanup; **one exit trap** (normal end, error, SIGTERM/SIGINT, the deadline) kills the -helpers it started (explicit pids, never a bare `wait`), shuts down and deletes the bench's simulators and -restores every knob; a **guard** refuses to boot more simulators when the guest's available memory (free + -speculative + inactive + purgeable pages) would fall under -`--min-guest-free-gib` (default 1.5, after the estimated cost of the new simulators), the guest reports memory -pressure level 2 or more, the host's available memory is under `--min-host-free-gib` (6), or the disk under -`--min-disk-gib` (12): the sweep stops there, it does not swap the VM to death. A sweep that ran no step at all -(a refused guard, foreign simulators) is a FAIL with a non-zero exit. Simulators running outside the bench (a -`launchd_sim` that is not the bench's own) refuse the run (`--allow-foreign` to measure anyway, recorded). Before each -window the guest must settle: `settle` waits until every process except the load and its observers (the renderers `SimMetalHost`/`backboardd`/`SimRenderServer`, GlassBench, `simfps`, WindowServer, python, ps, ssh, matched by name against the whole command, and the helper's own tree) uses less than -`--settle-bg` % of a core, for at most `--settle-max` s; the step records `settle.settled`, `bg_pct` and `waited_s`, -and the SCALE line names the N whose guest did not settle. +### Bench simulators are production simulators + +A fresh stock device is not what an agent gets: it runs stock services (a first boot keeps the vCPUs busy for minutes), +costs about 1.8 GB of guest disk (five of them took the guest's disk, then the host's, to the storage guard's limit and +stopped the VM) and keeps the Home Screen widgets, one of which retries a service Slim turns off in a tight loop +(~3 cores per "idle" simulator, for ever). So the bench makes simulators the way Simlock does: + +1. **One seed per device type**, kept shut down in the bench's set between runs: `simctl --set + clone ` (the source is Simlock's own device of that type: widgets trimmed, not Slim; Simlock's + set is only ever cloned from, never written, booted, shut down or deleted, and its config is not read), registered with + `sim-idle-guard`, one boot, `simslim --set on --profile ` (it reboots the device), + a check that its `disabled.plist` has at least 100 services off (168 today), the bench app installed, a real settle + (the guest quiet for two samples in a row and the widgets' `chronod/icons` present, `--seed-settle-max` 600 s) and a + shutdown. A seed that fails any step is deleted again, never left to be reused. +2. **Every bench simulator is `simctl clone `** (an APFS clone: no disk cost) with the seed's `disabled.plist` copied + into its own `/private/var/tmp/com.apple.CoreSimulator.SimDevice.` folder (mode 0700) before its first boot, and + checked there; the app is already in it. +3. The seed is **reused** by later runs (its Slim services, its widgets and the app's build are checked: a seed made for + another build of the bench app is refused, not silently reinstalled). `macbox bench seed --rebuild-seed` (or + `--rebuild-seed` on any run) makes a new one, which you need after `macbox bench tools` rebuilt the app. + +Simlock is an **optional** integration. Without a source device of the wanted type (`macbox bench check` lists the types +it finds) the run stops and says so. Two explicit ways out: `--scale-x 3` (the stock iPhone Pro type, +which Simlock usually has a source for), or **`--stock-devices`**: fresh stock devices, created one at a time with the +host-disk guard in front of each, for a device type Simlock has no source for or a scenario that needs a stock device. +The result records `env.guest.device_source` (`stock devices` or clones of a seed) and `compare` warns when two results +were made differently. Defaults of the Mac's paths: `~/.simlock04/devices/ios-sources` (`--source-set`), +`~/go/bin/simslim` (`--simslim`), the only `.json` in `~/.config/simslim/` (`--slim-profile`; several or none: the run stops and asks for the flag), `~/.local/bin/sim-idle-guard`. + +### Safety, in every run that boots simulators + +The **hard deadline** (`--deadline`, default 90 minutes) stops the run 60 s early, writes the result so far and runs +the cleanup; **one exit trap** (normal end, error, SIGTERM/SIGINT, the deadline, a monitor's abort) kills the helpers it +started (explicit pids, never a bare `wait`), shuts down and deletes the bench's simulators and restores every knob. + +- **Idle guard.** The Mac's `sim-idle-guard` shuts down every simulator nobody used for 15 minutes, in any device set. + Every bench simulator (and the seed while it is booted) is registered with `sim-idle-guard keep ` for the + run's length (the deadline plus 5 minutes, at most 240; renewed before every window) and `~/.sim-keep/` is removed + by the cleanup. A guard that is not installed is recorded in the result (`idle_guard`, a section) and the run goes on; a + guard that refuses `keep` stops it. +- **Disk.** `--disk-path` (a path on the filesystem that holds the VM's storage and that your storage guard watches) must + have `--disk-start-gib` 18 free to start, `--disk-boot-gib` 14 to boot another simulator (checked before **every** boot: + simulators boot one after the other, the lifecycle's boot storm is the one exception), and under `--disk-abort-gib` 11 the + run is aborted and cleaned up even while it waits. A disk reading that fails refuses. +- **Guest memory.** Before every boot: the guest's available memory (free + speculative + inactive + purgeable pages) after + the estimated cost of the new simulators stays over `--min-guest-free-gib` (1.5), no memory pressure level 2 or more, + **swap-outs have not grown** since the run began, the **compressor** holds at most `--max-compressor-mb` (2048), the + host's available memory is over `--min-host-free-gib` (6). The sweep stops there; it does not swap the VM to death. +- **A monitor** runs on its own thread while the run waits in a long call (a boot, a settle, a window): the host disk, the + VM's QEMU process (the same pid **and** start time) always; the Mac (answers over ssh twice in a row, no simulator that is + not the bench's, swap-outs, compressor) outside measurement windows, whose own load it must not add to. A breach aborts + the run from that thread: the partial result is written, the cleanup runs, the cleanup's verdict is added and the process + exits with 8 (disk), 9 (VM), 7 (someone else's simulator: **yield**, `--allow-foreign` to measure anyway) or 10 (guest + memory). +- **Settle fails closed.** Before a window the guest must settle: every process except the load and its observers (the + renderers `SimMetalHost`/`backboardd`/`SimRenderServer`, GlassBench, `simfps`, WindowServer, python, ps, ssh, matched by + name against the whole command, and the helper's own tree) uses less than `--settle-bg` % of a core for two samples in a + row. A settle that reaches `--settle-max` (300 s) **fails the run** (a `settle` FAIL section with the top processes by CPU + at that moment); nothing is measured on top of an unsettled guest. +- **The count of booted simulators.** Before every window (scale steps, A/B arms, soak, the observer window, idle) the guest + counts the Booted devices of the bench's own set and the run fails on any mismatch with the expected number (a + simulator the idle guard or someone shut down would otherwise be measured as if it ran). A step's `booted` is that + measured count, `expected_booted` the expected one. +- **The frame counter fails the window.** A simulator for which `simfps` returned nothing or an error, or an animating one + from whose surfaces no counter could be read, fails the `counter` gate (and so the step, the run and the exit code). +- **Leftovers.** A run takes the set with a lock and **refuses to start** while an earlier run left a device other than the + seed, a booted or never-completed seed, a lock, or overrides/keep files of a deleted device; `macbox bench cleanup` removes + them (`--seeds` also deletes the seeds). The cleanup looks again after deleting and says what it could not remove + (`CLEANUP FAIL`, exit code **20**, the lock stays); "cleanup done" is only printed for a verified one. +- A sweep that ran no step at all (a refused guard, foreign simulators) is a FAIL with a non-zero exit. ## Commands | | | |---|---| +| `seed` | Make the production seed for the device type (or check and report the one that is there); `--rebuild-seed` makes a new one. | +| `cleanup` | Shut down and delete what an earlier run left in the bench's set (verified; exit 20 when something stays); `--seeds` deletes the seeds too. | | `scale` | N = 1, 2, 3... simulators (`--n 1,2,3,4,6,8`, `--scale-x 1\|3`, `--mix`), per simulator numbers, the maximum sustainable N per quality target. Stops early when a floor is breached (`--floor`, default `fps>=20`), a gate fails or the system turns unstable. | | `idle` | The VM with no simulator: QEMU, GPU and memory standing cost against `--bar-qemu-cores` (0.15). | | `lifecycle` | `--cycles` times: create, boot, install, launch, first app frame, shut down, resume, first frame, clone, boot the clone, delete (medians of the timings below); a per-cycle leak gate; `--storm N`: N simulators booted at once. | @@ -62,10 +124,10 @@ and the SCALE line names the N whose guest did not settle. Common options: `--out DIR` (results, default `bench-results/`), `--secs` (window length, default 40), `--nominal-fps 60`, `--tag key=value` (recorded, for example `mesa=26.2.4` when Mesa is not a package), -`--probe-method auto|openurl|appearance|cmd|none`, `--slim` (the pool's Slim configuration through simslim on the bench's simulators: without it they are stock simulators, which cost more; it needs simslim in the guest, `macbox-guest tools`, and whether simslim finds devices of a custom device set is untested: the run stops with that message if it does not), `--quiesce` (disables `mediaanalysisd` in each simulator, as +`--probe-method auto|openurl|appearance|cmd|none`, `--stock-devices`, `--rebuild-seed`, `--source-set`, `--simslim`, `--slim-profile`, `--seed-settle-max` (see above), `--disk-path`, `--disk-start-gib`, `--disk-boot-gib`, `--disk-abort-gib`, `--max-compressor-mb`, `--set-name` (the bench's device set, default `bench`), `--quiesce` (disables `mediaanalysisd` in each simulator, as some old harnesses did; recorded, off by default), `--no-observer-check`, `--no-golden`, `--shots N`. -Every run prints one PASS/FAIL/SKIP/INFO line per section and exits 0, 1 (a section failed) or 2 (the run could not -go on). The JSON has the same lines under `sections`. +Every run prints one PASS/FAIL/SKIP/INFO line per section and exits 0, 1 (a section failed), 2 (the run could not +go on), 20 (the cleanup could not delete everything) or, when a monitor aborted it, 7, 8, 9 or 10 (above). The JSON has the same lines under `sections`. ### Workload mixes (`scale --mix`) @@ -119,13 +181,15 @@ all their descendants, never the bench's own helper, its ssh and shell ancestors started: their CPU is `system.guest.observer_cpu_pct` (the larger of the helper tree's CPU between the two `ps` snapshots and the CPU time of the helper's reaped children, `simfps` and the probes' `simctl`, over the window). RSS counts shared pages in each process: it overstates what leaving a simulator out would free. `system.guest`: `total_cpu_pct`, `sims_cpu_pct` (the trees), `other_cpu_pct` (everything else), `sims_rss_mb`. -**Lifecycle timings** (`lifecycle`, seconds, medians over `--cycles`; each cycle uses a fresh device): `create_s` -(`simctl create`); `cold_boot_s` (`boot` + `bootstatus`); `install_s`; `cold_launch_call_s` (the `launch` call); +**Lifecycle timings** (`lifecycle`, seconds, medians over `--cycles`; each cycle makes its own device the way a lease does: +a clone of the seed; with `--stock-devices` a fresh stock device, and then `create_s`, `install_s` and one more clone of the +installed device are timed too): `clone_s` (`simctl clone` of the seed + the Slim disables copied + the idle-guard keep); +`cold_boot_s` (the clone's first `boot` + `bootstatus`); `cold_launch_call_s` (the `launch` call); `cold_first_frame_s` (from the launch call to the first counter value `simfps` reads on the new surface); `shutdown_s` (shutting the booted device down), `resume_boot_s` and `resume_first_frame_s` (the same two again: "resume" here is -shutdown then boot, not Simlock's parking); `clone_s` (`simctl clone` of the shut-down device that has the app -installed), `clone_boot_s` (boot of the clone until `bootstatus` returns); `delete_s` (`simctl delete` of the source -device only: the shutdown before the clone is not timed). The clone is deleted untimed. +shutdown then boot, not Simlock's parking); `delete_s` (`simctl delete` of the device and its overrides folder; stock +path only: `clone_boot_s`, the boot of a clone of the installed device, deleted untimed). The boot storm +(`--storm N`) clones N simulators and boots them at once. **Guest memory** (`system.guest`): `mem_used_mb` = active + wired + compressor pages; `mem_free_mb` = free + speculative; `mem_avail_mb` is what the guard uses: free + speculative + inactive + purgeable (`vm_stat` pages; 4 KiB @@ -177,6 +241,7 @@ written first. A gate that cannot be read is SKIP, shown as such, never PASS. | `qemu_log` | no new `command buffer failed` line in QEMU's log (stamp waits are counted in the detail) | | `gpu_hang` | the hang flag did not appear | | `kernel_gpu` | no kernel GPU error line since the step began (SKIP when the journal is not readable) | +| `counter` | `simfps` answered with a result for every simulator of the window and every animating simulator's surfaces gave a readable frame counter (a counter that answers nothing or an error fails the window, it is not a missing metric) | | `memory` | the VM's own cgroup swapped no more than 50 MB (SKIP when the cgroup is unreadable: other projects' swapping on the host is not this run's); guest swap-outs fail it with `--fail-on-swap` | | `run_*` | every scenario, after its body: `run_coherent_census`, `run_coherent_process`, `run_qemu_log`, `run_gpu_hang` and `run_kernel_gpu` since the run began, and `run_crash_reports` (a new `.ips` on the guest since setup). They fail the exit code of `idle`, `lifecycle`, `soak`, `ab` too. | | `leak_*` | `lifecycle`: QEMU RSS and guest memory grow less than `--leak-rss-mb` / `--leak-guest-mb` per create/boot/delete cycle (least-squares slope); `soak`: less than `--soak-rss-mb-min` / `--soak-guest-mb-min` per minute | @@ -249,7 +314,8 @@ ssh macbox 'cd ~/.macbox/bench && ./build.sh' | `GenieBench.app` | the desktop load: a window that minimizes and restores, for counting the Genie animation's frames | | `scroll-local.html` | a page for Safari that scrolls itself under a blurred floating bar, with no network | -A run on the iOS Simulator, from the host: +A by-hand run on the iOS Simulator (a **fresh stock device**: about 1.8 GB of guest disk and a minutes-long first boot, not what an +agent's simulator is; `macbox bench scale` makes production simulators and is what to use for numbers): ```sh # a simulator of its own for the bench, so the pool is left alone (1x: the macbox 1x device type) diff --git a/src/bench/guest/benchguest.py b/src/bench/guest/benchguest.py index bb4cc11..f336209 100644 --- a/src/bench/guest/benchguest.py +++ b/src/bench/guest/benchguest.py @@ -40,7 +40,8 @@ # command that names it is simlock_clone()) and the Slim tooling the fleet uses on every clone. SOURCE_SET = os.environ.get("BENCH_SIMLOCK_SOURCES") or os.path.expanduser("~/.simlock04/devices/ios-sources") SIMSLIM = os.environ.get("BENCH_SIMSLIM") or os.path.expanduser("~/go/bin/simslim") -SLIM_PROFILE = os.environ.get("BENCH_SLIM_PROFILE") or os.path.expanduser("~/.config/simslim/atlas.json") +SLIM_PROFILE = os.environ.get("BENCH_SLIM_PROFILE") # else the one profile in SLIM_DIR +SLIM_DIR = os.path.expanduser("~/.config/simslim") MIN_SLIM_SERVICES = 100 # a Slim device has 168 services off today PROBE_POINT = "36,175" # the input marker's centre, in points (GlassBench -probe) SET = None @@ -418,6 +419,13 @@ def out_json(obj): sys.stdout.flush() +def _has_profile(): + try: + return os.path.isfile(resolve_profile()) + except GuestError: + return False + + def cmd_info(a): sw = run(["sw_vers"], 10)[1] ver = re.search(r"ProductVersion:\s*(\S+)", sw) @@ -432,6 +440,8 @@ def cmd_info(a): vm = parse_vm_stat(vmstat_text()) out_json({"macos": ver.group(1) if ver else None, "xcode": xc, "runtimes": rts, "arch": run(["uname", "-m"], 5)[1].strip(), "cpus": sysctl("hw.ncpu"), "mem_mb": int(sysctl("hw.memsize") or 0) // 1048576, "mem": mem_summary(vm), + "simlock_types": source_types(SOURCE_SET), "has_simslim": os.path.isfile(SIMSLIM) and os.access(SIMSLIM, os.X_OK), + "has_slim_profile": _has_profile(), "has_idle_guard": os.path.isfile(IDLE_GUARD) and os.access(IDLE_GUARD, os.X_OK), "has_simfps": bool(STUB) or os.path.exists(os.path.join(HERE, "..", "simfps")), "has_app": bool(STUB) or os.path.isdir(os.path.join(HERE, "..", "GlassBench.app")), "stub": bool(STUB)}) @@ -573,6 +583,21 @@ def find_source(type_id, src_set): return found[0][1], found[0][2] +def source_types(src_set): + """The device types Simlock has a source device for (names only), [] when Simlock's set is not there.""" + out = set() + try: + for n in os.listdir(src_set): + try: + with open(os.path.join(src_set, n, "device.plist"), "rb") as f: + out.add(str(plistlib.load(f).get("deviceType")).rsplit(".", 1)[-1]) + except (OSError, ValueError, plistlib.InvalidFileException): + pass + except OSError: + pass + return sorted(out) + + def runtime_version(runtime_id): try: for r in json.loads(run(["xcrun", "simctl", "list", "runtimes", "-j"], 30)[1]).get("runtimes", []): @@ -657,7 +682,7 @@ def cmd_seed(a): if simctl("bootstatus", u, timeout=600)[0] != 0: raise GuestError("the seed never finished booting (bootstatus)") sl = os.path.expanduser(a.simslim or SIMSLIM) - prof = os.path.expanduser(a.slim_profile or SLIM_PROFILE) + prof = resolve_profile(a.slim_profile) # Simlock's source devices are widget-trimmed but NOT Slim: simslim shuts the device down, rewrites its services and reboots it rc, out, err, secs = run([sl, "--set", SET, "on", u, "--profile", prof], 900) if rc != 0: @@ -738,6 +763,22 @@ def slim_count(udid): return run(["plutil", "-p", f], 15)[1].count("=> true") +def resolve_profile(arg=None): + """simslim's profile: --slim-profile, else $BENCH_SLIM_PROFILE, else the only *.json in ~/.config/simslim (ambiguous or + none: an error that says to pass --slim-profile).""" + chosen = arg or SLIM_PROFILE + if chosen: + return os.path.expanduser(chosen) + try: + found = sorted(n for n in os.listdir(SLIM_DIR) if n.endswith(".json")) + except OSError: + found = [] + if len(found) != 1: + raise GuestError("no simslim profile: %s; pass --slim-profile (or set BENCH_SLIM_PROFILE)" % ( + "%d profiles in the simslim config folder" % len(found) if found else "none in the simslim config folder")) + return os.path.join(SLIM_DIR, found[0]) + + def copy_overrides(seed_udid, udid, min_services=MIN_SLIM_SERVICES): """The Slim disables live OUTSIDE the device, keyed by UDID: a clone boots stock unless the seed's file is in its own folder (mode 0700) before its first boot. Fails when the seed has none or the copy did not take.""" diff --git a/src/bench/guest/benchguest_stub.py b/src/bench/guest/benchguest_stub.py index 45a829c..4fe9343 100644 --- a/src/bench/guest/benchguest_stub.py +++ b/src/bench/guest/benchguest_stub.py @@ -359,6 +359,10 @@ def install(mod): import shutil for u in SRC_UDIDS[1:]: shutil.rmtree(os.path.join(mod.SOURCE_SET, u)) + mod.SLIM_DIR = _p("simslim-config") + os.makedirs(mod.SLIM_DIR, exist_ok=True) + with open(os.path.join(mod.SLIM_DIR, "profile.json"), "w") as f: + f.write("{}") mod.SIMSLIM = _p("bin/simslim") os.makedirs(_p("bin"), exist_ok=True) with open(mod.SIMSLIM, "w") as f: diff --git a/src/bench/macbench/cli.py b/src/bench/macbench/cli.py index 82ec00a..38c61b7 100644 --- a/src/bench/macbench/cli.py +++ b/src/bench/macbench/cli.py @@ -49,7 +49,7 @@ def add_common(p): p.add_argument("--rebuild-seed", action="store_true", help="delete the bench's seed and make a new one (after `macbox bench tools` rebuilt the app, or when it cannot be reused)") p.add_argument("--source-set", default=os.environ.get("BENCH_SIMLOCK_SOURCES"), help="Simlock's source device set in the guest, read only (default ~/.simlock04/devices/ios-sources)") p.add_argument("--simslim", default=os.environ.get("BENCH_SIMSLIM"), help="simslim in the guest (default ~/go/bin/simslim)") - p.add_argument("--slim-profile", default=os.environ.get("BENCH_SLIM_PROFILE"), help="simslim profile in the guest (default ~/.config/simslim/atlas.json)") + p.add_argument("--slim-profile", default=os.environ.get("BENCH_SLIM_PROFILE"), help="simslim profile in the guest (default: the only .json in ~/.config/simslim)") p.add_argument("--seed-settle-max", type=float, default=600, help="the seed must settle within this many seconds or it is deleted and the run fails") p.add_argument("--quiesce", action="store_true", help="disable mediaanalysisd in each simulator (recorded; production does not)") p.add_argument("--fail-on-swap", action="store_true", help="a guest swap-out in a window fails the memory gate (default: reported, host swap fails)") @@ -383,6 +383,11 @@ def line(good, text): line(True, "guest answers: macOS %s, %s" % (d.get("macos"), d.get("xcode"))) line(d.get("has_simfps"), "simfps built in the guest (macbox bench tools)") line(d.get("has_app"), "GlassBench.app built in the guest (macbox bench tools)") + types = d.get("simlock_types") or [] + line(bool(types), "Simlock's source devices found (production simulators: %s)%s" % (", ".join(types) or "none", + "" if types else "; without them only --stock-devices works")) + line(d.get("has_simslim") and d.get("has_slim_profile"), "simslim and its profile in the guest (the seed is made Slim with them)") + line(d.get("has_idle_guard"), "sim-idle-guard in the guest (keeps the bench's simulators from its 15-minute shutdown)") except (OSError, ValueError, IndexError, subprocess.TimeoutExpired): line(False, "the guest's bench helper (macbox bench tools)") return 0 if ok else 1 diff --git a/src/bench/macbench/compare.py b/src/bench/macbench/compare.py index 68f9d17..0234970 100644 --- a/src/bench/macbench/compare.py +++ b/src/bench/macbench/compare.py @@ -50,6 +50,9 @@ def compare_scale(a, b, noise=NOISE): a["id"], b["id"], da.get("mix"), db.get("mix"), da.get("scale_x"), db.get("scale_x"), noise * 100)) if (da.get("mix"), da.get("scale_x")) != (db.get("mix"), db.get("scale_x")): lines.append(" warning: different mix or scale: the numbers are not comparable") + src = [(r.get("env", {}).get("guest") or {}).get("device_source") for r in (a, b)] + if src[0] != src[1]: + lines.append(" warning: the simulators were not made the same way (A: %s, B: %s): the numbers are not comparable" % tuple(src)) for name in sorted(set(da["summary"]["max_n"]) | set(db["summary"]["max_n"])): ma = (da["summary"]["max_n"].get(name) or {}).get("max_n") mb = (db["summary"]["max_n"].get(name) or {}).get("max_n") diff --git a/src/bench/tests/test_dry_closed.py b/src/bench/tests/test_dry_closed.py index 81a143e..c947eb9 100644 --- a/src/bench/tests/test_dry_closed.py +++ b/src/bench/tests/test_dry_closed.py @@ -120,8 +120,9 @@ def sources(self): def test_no_module_selects_files_by_pattern(self): for name, text in self.sources(): self.assertNotRegex(text, r"\bimport glob\b|\bfrom glob\b|\bfnmatch\b|\.glob\(|\brglob\b", name) - # the only directory listings: sets' devices (a UUID pattern), crash reports (exact suffix), /proc, and the source set - allowed = {"benchguest.py": 2, "hostprobe.py": 2} + # the only directory listings: crash reports (exact suffix), Simlock's source set (twice: a UUID pattern, device.plist read by exact + # path), the simslim config folder (one profile), /proc and /sys in hostprobe. A new listing has to be argued for here. + allowed = {"benchguest.py": 4, "hostprobe.py": 2} for name, text in self.sources(): n = len(re.findall(r"os\.listdir\(|os\.scandir\(", text)) self.assertLessEqual(n, allowed.get(name, 0), name) diff --git a/src/bench/tests/test_modules.py b/src/bench/tests/test_modules.py index 0c1325f..27c05c8 100644 --- a/src/bench/tests/test_modules.py +++ b/src/bench/tests/test_modules.py @@ -40,6 +40,28 @@ def wr(path, mode, text): f.write(text) +class SlimProfile(unittest.TestCase): + def test_explicit_env_then_the_only_file_else_an_error_that_says_what_to_pass(self): + import tempfile + d = tempfile.mkdtemp() + old = (bg.SLIM_DIR, bg.SLIM_PROFILE) + try: + bg.SLIM_DIR, bg.SLIM_PROFILE = d, None + with self.assertRaises(bg.GuestError) as e: + bg.resolve_profile() + self.assertIn("--slim-profile", str(e.exception)) + wr(os.path.join(d, "a.json"), "w", "{}") + self.assertEqual(bg.resolve_profile(), os.path.join(d, "a.json")) + wr(os.path.join(d, "b.json"), "w", "{}") + with self.assertRaises(bg.GuestError): + bg.resolve_profile() # two: no guessing + self.assertEqual(bg.resolve_profile("/x/p.json"), "/x/p.json") + bg.SLIM_PROFILE = "/env/p.json" + self.assertEqual(bg.resolve_profile(), "/env/p.json") + finally: + bg.SLIM_DIR, bg.SLIM_PROFILE = old + + class GuestParsers(unittest.TestCase): def test_vm_stat(self): m = bg.mem_summary(bg.parse_vm_stat(VM_STAT)) From 64c423390affc52931155d4fc93e444d83b80cb7 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Tue, 17 Nov 2026 05:19:50 +0200 Subject: [PATCH 17/23] macbox: bench usage lists seed and cleanup --- macbox | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macbox b/macbox index 861f8e9..27eeb0d 100755 --- a/macbox +++ b/macbox @@ -716,7 +716,7 @@ Day to day: macbox coherent [status|reset] Coherent's crash guard (MACBOX_COHERENT=auto|on|off) macbox video-kexts reims-util's video kexts into OpenCore (for MACBOX_VIDEO=on) macbox backup - macbox bench scale|idle|lifecycle|soak|build|panel|ab|golden|compare|chart|report|tools|check + macbox bench scale|idle|lifecycle|soak|build|panel|ab|golden|seed|cleanup|compare|chart|report|tools|check the scaling, regression and A/B suite (src/bench/README.md) EOF } From 3ae296db327020c5390939d71fd26c331797b562 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Tue, 17 Nov 2026 05:40:05 +0200 Subject: [PATCH 18/23] bench: review fixes - incomplete seeds are cleaned, foreign count ignores the bench's own transitions, icons recorded not gated, abort race, deadline verdict, lifecycle guard --- src/bench/README.md | 4 +- src/bench/guest/benchguest.py | 84 +++++++++++++++++------------- src/bench/guest/benchguest_stub.py | 12 ++++- src/bench/macbench/cli.py | 12 +++-- src/bench/macbench/core.py | 10 ++-- src/bench/macbench/others.py | 4 ++ src/bench/macbench/runctx.py | 5 ++ src/bench/macbench/scale.py | 2 +- src/bench/tests/test_dry_guards.py | 23 +++++++- src/bench/tests/test_dry_seed.py | 8 +-- src/bench/tests/test_dry_state.py | 49 ++++++++++++++++- src/bench/tests/test_modules.py | 12 +++++ src/bench/tests/test_runguards.py | 44 +++++++++++++++- 13 files changed, 215 insertions(+), 54 deletions(-) diff --git a/src/bench/README.md b/src/bench/README.md index efc67a0..74b78c3 100644 --- a/src/bench/README.md +++ b/src/bench/README.md @@ -17,7 +17,7 @@ DRY=1 macbox bench scale --n 1,2,3 # the whole control flow against stubs: Requirements: Python 3.8+ on the host (standard library only; no Pillow, no numpy), the VM running, `ssh` to it. Unit tests, no VM needed: `cd src/bench && python3 -m unittest discover -s tests -t .` (the pure parts and the run's safety) -and `python3 -m unittest tests.test_dry` (the dry runs with injected faults, a couple of minutes). +and `python3 -m unittest tests.test_dry tests.test_dry_state tests.test_dry_seed tests.test_dry_closed tests.test_dry_guards` (the dry runs with injected faults: the sweeps and their faults; what a run leaves and the idle guard, including a run killed mid-seed; the seed; the fail-closed gates; the host-disk, VM and abort guards; several minutes, run one module at a time on a busy host). ## What a run does, and what it never touches @@ -44,7 +44,7 @@ stopped the VM) and keeps the Home Screen widgets, one of which retries a servic set is only ever cloned from, never written, booted, shut down or deleted, and its config is not read), registered with `sim-idle-guard`, one boot, `simslim --set on --profile ` (it reboots the device), a check that its `disabled.plist` has at least 100 services off (168 today), the bench app installed, a real settle - (the guest quiet for two samples in a row and the widgets' `chronod/icons` present, `--seed-settle-max` 600 s) and a + (the guest quiet for two samples in a row, `--seed-settle-max` 600 s; whether the widgets' `chronod/icons` folder exists is recorded as `icons`, not required) and a shutdown. A seed that fails any step is deleted again, never left to be reused. 2. **Every bench simulator is `simctl clone `** (an APFS clone: no disk cost) with the seed's `disabled.plist` copied into its own `/private/var/tmp/com.apple.CoreSimulator.SimDevice.` folder (mode 0700) before its first boot, and diff --git a/src/bench/guest/benchguest.py b/src/bench/guest/benchguest.py index f336209..cbe1faf 100644 --- a/src/bench/guest/benchguest.py +++ b/src/bench/guest/benchguest.py @@ -657,12 +657,12 @@ def cmd_seed(a): if here[u].get("state") != "Shutdown": simctl("shutdown", u, timeout=120) n = slim_count(u) - why = ("only %d services off" % n) if n < MIN_SLIM_SERVICES else (None if os.path.isdir(icons_path(u)) else "its widgets never settled (no chronod icons)") + why = ("only %d services off" % n) if n < MIN_SLIM_SERVICES else None if not why and cur.get("app_stamp") != stamp: why = "the bench app was rebuilt since (GlassBench differs)" if why: raise GuestError("the existing seed cannot be reused (%s): rebuild it with --rebuild-seed" % why) - return out_json(dict(cur, seed=u, reused=True, services_off=n, source_type=tid.rsplit(".", 1)[-1])) + return out_json(dict(cur, seed=u, reused=True, services_off=n, source_type=tid.rsplit(".", 1)[-1], icons=os.path.isdir(icons_path(u)))) src, pl = find_source(tid, os.path.expanduser(a.source_set or SOURCE_SET)) if src is None: return out_json({"seed": None, "no_source": pl, "udid": None}) @@ -697,10 +697,11 @@ def cmd_seed(a): raise GuestError("installing the bench app in the seed failed") if simctl("get_app_container", u, BUNDLE_ID, timeout=30)[0] != 0: raise GuestError("the bench app is not installed in the seed after install") - st = settle_wait(a.settle_max, a.bg_pct, a.step, a.consecutive, icons_of=u) + st = settle_wait(a.settle_max, a.bg_pct, a.step, a.consecutive) + icons = os.path.isdir(icons_path(u)) # recorded, never a gate: whether a widget-trimmed seed grows this folder is unverified if not st["settled"]: - raise GuestError("the seed did not settle in %g s (background %.0f%% of a core, widgets' icons %s; top: %s)" % ( - a.settle_max, st["bg_pct"], "present" if st["icons"] else "missing", top_text(st["top"]))) + raise GuestError("the seed did not settle in %g s (background %.0f%% of a core; top: %s)" % ( + a.settle_max, st["bg_pct"], top_text(st["top"]))) if simctl("shutdown", u, timeout=120)[0] != 0: raise GuestError("the seed did not shut down") except BaseException: @@ -711,7 +712,7 @@ def cmd_seed(a): raise drop_keep(u) res = {"seed": u, "udid": u, "name": name, "type": tid, "runtime": runtime, "runtime_version": runtime_version(runtime), "complete": True, "app_stamp": stamp, - "source_type": tid.rsplit(".", 1)[-1], "reused": False, "made_s": round(now() - t0, 1), "services_off": n, "settle_s": st["waited_s"], + "source_type": tid.rsplit(".", 1)[-1], "reused": False, "made_s": round(now() - t0, 1), "services_off": n, "settle_s": st["waited_s"], "icons": icons, "settle_bg_pct": st["bg_pct"], "slimmed_s": round(secs, 1)} man = load_manifest() man["seeds"][tid] = {k: res[k] for k in ("udid", "name", "type", "runtime", "runtime_version", "complete", "app_stamp", "made_s", "services_off")} @@ -884,19 +885,18 @@ def bg_cpu_pct(secs=3.0): return bg_sample(secs)[0] -def settle_wait(max_s, bg_pct, step, consecutive=2, icons_of=None): - """Waits until the background CPU is under bg_pct for `consecutive` samples IN A ROW (and, for a seed, the widgets' - icons exist). {"settled", "bg_pct" (last sample), "waited_s", "series", "top" (at the last sample), "icons"}. Reaching - max_s is settled False: the caller fails the run, it never measures on top of this.""" - t0, series, top, icons = now(), [], [], True +def settle_wait(max_s, bg_pct, step, consecutive=2): + """Waits until the background CPU is under bg_pct for `consecutive` samples IN A ROW. {"settled", "bg_pct" (last + sample), "waited_s", "series", "top" (at the last sample)}. Reaching max_s is settled False: the caller fails the run, + it never measures on top of this.""" + t0, series, top = now(), [], [] while True: v, top = bg_sample(step) series.append(v) - icons = icons_of is None or os.path.isdir(icons_path(icons_of)) - if len(series) >= consecutive and all(x < bg_pct for x in series[-consecutive:]) and icons: - return {"settled": True, "bg_pct": v, "waited_s": round(now() - t0, 1), "series": series[-8:], "top": top, "icons": icons} + if len(series) >= consecutive and all(x < bg_pct for x in series[-consecutive:]): + return {"settled": True, "bg_pct": v, "waited_s": round(now() - t0, 1), "series": series[-8:], "top": top} if now() - t0 > max_s: - return {"settled": False, "bg_pct": v, "waited_s": round(now() - t0, 1), "series": series[-8:], "top": top, "icons": icons} + return {"settled": False, "bg_pct": v, "waited_s": round(now() - t0, 1), "series": series[-8:], "top": top} def cmd_settle(a): @@ -960,10 +960,10 @@ def sample_pressure(): # CPU time of the helper's reaped children over the window is the observer's real cost reaped = (ru1.ru_utime + ru1.ru_stime) - (ru0.ru_utime + ru0.ru_stime) tot["observer_cpu_pct"] = round(max(tot["observer_cpu_pct"], 100 * reaped / max(t_end - t_start, 0.001)), 1) - st = {d["udid"]: d.get("state") for d in devices()} - own_up = len([1 for v in st.values() if v == "Booted"]) - foreign = {"before": max(0, len([p for p in ps0 if re.search(r"(^|/)launchd_sim( |$)", p["cmd"])]) - len(udids)), - "after": max(0, len([p for p in ps1 if re.search(r"(^|/)launchd_sim( |$)", p["cmd"])]) - own_up)} + dev_list = devices() + st = {d["udid"]: d.get("state") for d in dev_list} + foreign = {"before": foreign_count([dict(d, state="Booted") if d["udid"] in udids else d for d in dev_list], ps0), + "after": foreign_count(dev_list, ps1)} sims = [] for i, u in enumerate(udids): sj, err = res.get(u, (None, None)) @@ -1135,9 +1135,9 @@ def cmd_cycle(a): shutil.rmtree(tmp, ignore_errors=True) -def leftovers(): +def leftovers(ignore_lock=False): """What an earlier run left in the bench's set: [text]. Empty = the set is clean (settled seeds, shut down, are not - leftovers: they are kept on purpose).""" + leftovers: they are kept on purpose). The one definition: `acquire` refuses on it and `teardown` verifies against it.""" out, man, devs = [], load_manifest(), devices() complete = {s["udid"] for s in man["seeds"].values() if s.get("complete")} for d in devs: @@ -1153,7 +1153,7 @@ def leftovers(): if u not in here and (os.path.exists(overrides_dir(u)) or os.path.exists(os.path.join(KEEP_DIR, u))): out.append("overrides or keep file of the deleted device %s" % u) lock = os.path.join(SET, LOCK) - if os.path.exists(lock): + if os.path.exists(lock) and not ignore_lock: try: with open(lock) as f: out.append("the run lock of an earlier run (%s)" % f.read().strip()[:80]) @@ -1178,22 +1178,23 @@ def cmd_acquire(a): def cmd_teardown(a): - """Shuts down every device of the bench's set and deletes every one that is not a seed (all of them with --seeds), - then removes what lives outside the device: its overrides folder and its ~/.sim-keep file. It looks again at the end - and reports what is still there in `failed`/`left`/`booted`; the run lock is released only when nothing is.""" + """Shuts down every device of the bench's set and deletes every one that is not a COMPLETE seed (every seed with --seeds; + a seed that was never completed is a leftover like any other device), then removes what lives outside the device: its + overrides folder and its ~/.sim-keep file. It looks again at the end with the same definition of 'leftover' that + `acquire` uses and reports what is still there in `failed`/`left`/`booted`; the run lock goes only when nothing is.""" failed = [] man = load_manifest() + complete = {s["udid"] for s in man["seeds"].values() if s.get("complete")} simctl("shutdown", "all", timeout=300) # a failure shows in the second look below for d in devices(): - if is_seed(d) and not a.seeds: + if d["udid"] in complete and not a.seeds: continue rc, _, err, _ = simctl("delete", d["udid"], timeout=120) if rc != 0: failed.append("delete %s %s: %s" % (d["name"], d["udid"], (err or "failed").strip()[:100])) after = devices() present = {d["udid"] for d in after} - seeds_now = {d["udid"] for d in after if is_seed(d)} - for u in sorted(set(man["devices"]) | {s["udid"] for s in man["seeds"].values()} | {d["udid"] for d in after}): + for u in sorted(set(man["devices"]) | {s["udid"] for s in man["seeds"].values()} | present): e = drop_keep(u) if e: failed.append(e) @@ -1203,21 +1204,22 @@ def cmd_teardown(a): failed.append(e) else: man["devices"].pop(u, None) - man["seeds"] = {t: s for t, s in man["seeds"].items() if s["udid"] in seeds_now} + man["seeds"] = {t: s for t, s in man["seeds"].items() if s["udid"] in present and s["udid"] in complete} save_manifest(man) - left = [d for d in after if not is_seed(d) or a.seeds] - booted = [d for d in after if d.get("state") == "Booted"] keep_left = [u for u in present | set(man["devices"]) if os.path.exists(os.path.join(KEEP_DIR, u))] if keep_left: failed.append("%d keep file(s) still in the idle guard's folder" % len(keep_left)) - if not (failed or left or booted): + remaining = leftovers(ignore_lock=True) + failed += ["still there: %s" % x for x in remaining] + booted = [d for d in after if d.get("state") == "Booted"] + if not (failed or remaining or booted): try: os.remove(os.path.join(SET, LOCK)) except FileNotFoundError: pass except OSError as e: failed.append("run lock: %s" % e) - out_json({"left": len(left), "booted": len(booted), "failed": failed, "seeds": len(seeds_now), "calls": [c[1:] for c in CALLS][-3:]}) + out_json({"left": len(remaining), "booted": len(booted), "failed": failed, "seeds": len([d for d in after if is_seed(d)]), "calls": [c[1:] for c in CALLS][-3:]}) def count_launchd_sim(): @@ -1225,6 +1227,18 @@ def count_launchd_sim(): return len([p for p in parse_ps(ps_text()) if re.search(r"(^|/)launchd_sim( |$)", p["cmd"])]) +def foreign_count(devs=None, procs=None): + """Simulators that are not the bench's: the `launchd_sim` processes minus the bench's own. A device of the bench's set + in ANY state counts as its own when the process's command line names its UDID (a device that is booting, shutting + down or being rebooted by simslim has a launchd_sim and is not Booted yet or any more); where the command line names + no device (HYPOTHESIS: it does carry the device's path), the number of Booted devices is the fallback.""" + devs = devices() if devs is None else devs + udids = [d["udid"] for d in devs] + procs = [p for p in (parse_ps(ps_text()) if procs is None else procs) if re.search(r"(^|/)launchd_sim( |$)", p["cmd"])] + named = len([p for p in procs if any(u in p["cmd"] for u in udids)]) + return max(0, len(procs) - max(named, len([d for d in devs if d.get("state") == "Booted"]))) + + def cmd_precheck(a): """Before a measurement window: how many simulators of the bench's own set are Booted (counted from the set's own device list, the seed included), which of the expected ones are not, and how many simulators outside the bench run on @@ -1237,7 +1251,7 @@ def cmd_precheck(a): k = keep_or_fail(u, a.keep_min) kept[k["state"]] = kept.get(k["state"], 0) + 1 out_json({"booted": len(booted), "booted_udids": booted, "missing": [u for u in expect if u not in booted], - "extra": [u for u in booted if u not in expect], "foreign_booted": max(0, count_launchd_sim() - len(booted)), "keep": kept}) + "extra": [u for u in booted if u not in expect], "foreign_booted": foreign_count(), "keep": kept}) def cmd_ips(a): @@ -1247,7 +1261,7 @@ def cmd_ips(a): def cmd_memcheck(a): vm = mem_summary(parse_vm_stat(vmstat_text())) own = len([d for d in devices() if d.get("state") == "Booted"]) - out_json({"mem": vm, "pressure": pressure_level(), "booted": own, "foreign_booted": max(0, count_launchd_sim() - own), + out_json({"mem": vm, "pressure": pressure_level(), "booted": own, "foreign_booted": foreign_count(), "ips": len(list_ips())}) diff --git a/src/bench/guest/benchguest_stub.py b/src/bench/guest/benchguest_stub.py index 4fe9343..11142f7 100644 --- a/src/bench/guest/benchguest_stub.py +++ b/src/bench/guest/benchguest_stub.py @@ -5,10 +5,10 @@ (frame rate collapses from N), hang_simfps (simfps never answers), nomarker (the input marker never flips), cantdelete (simctl delete fails: a cleanup that cannot finish), noguard (no sim-idle-guard on the Mac), keepfail (sim-idle-guard keep fails), nosource (Simlock has no source device of the type), nosourceset (no Simlock set at all), -slimfail (simslim fails), slimfew (simslim leaves too few services off), noicons (the widgets never settle: no chronod icons), +slimfail (simslim fails), slowslim (simslim takes 2 minutes: a run killed while the seed is half made), slimfew (simslim leaves too few services off), noicons (no chronod icons folder: recorded as icons=false, never a failure), noinstall (simctl install does nothing), unsettled@N (a stray process burns 1.7 cores from N booted simulators), lostsim@N (the Mac's idle guard shuts the newest simulator down once N are booted), counterfail@N (the frame counter -of the last simulator returns nothing from N booted), foreign@N (a simulator outside the bench boots when N are booted), swapgrow@N (the guest's swap-outs +of the last simulator returns nothing from N booted), foreign@N (a simulator outside the bench boots when N are booted), transition@N (a bench device that is not Booted still has its launchd_sim: it must not read as foreign), swapgrow@N (the guest's swap-outs grow from N booted), compressor@N (the compressor holds 3 GB from N booted). The files a simulator leaves outside its device (Slim overrides, ~/.sim-keep, the idle guard) live under $BENCHGUEST_STATE too, so the bench's real file code runs against them.""" @@ -133,6 +133,8 @@ def _stub_run(argv): elif argv and argv[0] == _MOD.SIMSLIM: # simslim --set on --profile : shuts the device down, rewrites its services, reboots it u = argv[4] + if fault("slowslim") is not None: + time.sleep(120) # a run that is killed while its seed is half made if fault("slimfail") is not None: rc, err = 1, "simslim: injected failure" elif s["devs"].get(u, {}).get("state") != "Booted": @@ -255,6 +257,12 @@ def stub_ps(): us = fault("unsettled") if us is not None and n >= us: # the Maps widget's retry loop: a process that never quiets down lines.append("%5d 1 40000 %s /usr/libexec/diagnosticd" % (900, _cpu(now - T0, 1.7))) + tr = fault("transition") + if tr is not None and n >= tr: # a bench device (the seed) that is shut down, its launchd_sim still alive: not foreign + for u, d in sorted(s["devs"].items()): + if d["state"] != "Booted": + lines.append("%5d 1 20000 %s launchd_sim /stub/set/%s/data" % (960, _cpu(now - T0, 0.003), u)) + break fo = fault("foreign") if fo is not None and n >= fo: # someone else's simulator on the Mac, in another device set lines.append("%5d 1 20000 %s launchd_sim /stub/other-set/FOREIGN/data" % (950, _cpu(now - T0, 0.01))) diff --git a/src/bench/macbench/cli.py b/src/bench/macbench/cli.py index 38c61b7..82c6d1c 100644 --- a/src/bench/macbench/cli.py +++ b/src/bench/macbench/cli.py @@ -229,15 +229,19 @@ def execute(a, kind, body): try: body(ctx) finally: - core.run_gates(ctx) # every scenario: Coherent, kernel GPU lines and crash reports since the run began + if not run.aborted: # a monitor that ended the run owns the result: no one else writes into it + core.run_gates(ctx) # every scenario: Coherent, kernel GPU lines and crash reports since the run began except core.Stop as e: - ctx.section("stopped", "FAIL", "STOPPED: %s" % e) + if not run.aborted: + ctx.section("stopped", "FAIL", "STOPPED: %s" % e) rc = 2 except (GuestError, DeadlineError) as e: - ctx.section("error", "FAIL", "ERROR: %s: %s" % (type(e).__name__, e)) + if not run.aborted: + ctx.section("error", "FAIL", "ERROR: %s: %s" % (type(e).__name__, e)) rc = 2 finally: - run.cleanup() + if not run.aborted: + run.cleanup() run.join_abort() # when a monitor aborted the run it writes the result and owns the exit code rc = max(rc, finalize(ctx)) fails = [s for s in ctx.doc["sections"] if s["status"] == "FAIL"] diff --git a/src/bench/macbench/core.py b/src/bench/macbench/core.py index b937442..a088556 100644 --- a/src/bench/macbench/core.py +++ b/src/bench/macbench/core.py @@ -113,7 +113,7 @@ def monitor_check(ctx): """While the run waits (a boot, a settle, a window): the host's disk, QEMU and, outside measurement windows (their own calls would be observer load), the Mac. Any breach aborts the run NOW: the monitor thread writes the partial result, runs the one cleanup and exits (8 disk, 9 VM, 7 someone else's simulator, 10 guest memory).""" - a, st = ctx.a, {"dead": 0, "last_guest": 0.0} + a, st = ctx.a, {"dead": 0, "foreign": 0, "last_guest": 0.0} def check(): why = disk_reason(ctx, a.disk_abort_gib, "go on", "") @@ -133,7 +133,11 @@ def check(): st["dead"] = 0 m = ctx.guest.quick(["memcheck"], 60) if m.get("foreign_booted", 0) > 0 and not a.allow_foreign: - return "a simulator that is not the bench's is booted (someone needs the Mac): yielding", 7 + st["foreign"] += 1 # the bench's own boots and shutdowns race the count: two polls in a row + if st["foreign"] >= 2: + return "a simulator that is not the bench's is booted (someone needs the Mac): yielding", 7 + else: + st["foreign"] = 0 why = guest_memory_reason(ctx, m["mem"]) if why: return why, 10 @@ -229,7 +233,7 @@ def make_seed(ctx): ctx.runtime = r.get("runtime") or ctx.runtime g = ctx.doc["env"]["guest"] g["runtime_used"] = r.get("runtime_version") or g.get("runtime_used") - g["seed"] = {k: r.get(k) for k in ("source_type", "reused", "made_s", "services_off", "settle_s", "settle_bg_pct")} + g["seed"] = {k: r.get(k) for k in ("source_type", "reused", "made_s", "services_off", "settle_s", "settle_bg_pct", "icons")} ctx.section("seed", "INFO", "SEED %s: %s services off (Slim), %s" % ( "reused" if r.get("reused") else "made from a Simlock source device in %s s (simslim %s s, settled in %s s at %s%% background)" % ( r.get("made_s"), r.get("slimmed_s"), r.get("settle_s"), r.get("settle_bg_pct")), diff --git a/src/bench/macbench/others.py b/src/bench/macbench/others.py index 34e35be..ee02567 100644 --- a/src/bench/macbench/others.py +++ b/src/bench/macbench/others.py @@ -61,6 +61,10 @@ def run_lifecycle(ctx): mem0 = ctx.guest.call(["memcheck"], 60)["mem"] data = ctx.doc["data"] = {"cycles": cyc, "storm": None} for i in range(a.cycles): + why = core.guard(ctx, 1, "before lifecycle cycle %d" % (i + 1)) + if why: + ctx.section("stopped", "FAIL", "STOPPED: %s" % why) + break r = ctx.guest.call(["cycle", "--keep-min", core.keep_minutes(ctx)] + (["--type", ctx.type_id, "--runtime", ctx.runtime] if ctx.stock else ["--seed", ctx.seed["seed"]]), 900) cyc.append(r["timings"]) m = ctx.guest.call(["memcheck"], 60) diff --git a/src/bench/macbench/runctx.py b/src/bench/macbench/runctx.py index c2bcf5b..da175ae 100644 --- a/src/bench/macbench/runctx.py +++ b/src/bench/macbench/runctx.py @@ -91,6 +91,11 @@ def _watchdog(self): except Exception as e: # the partial result is best effort self.log("partial result failed: %s" % e) self.cleanup() + if self.final: # the cleanup's verdict goes into the result of a run that ran out of time too + try: + self.final() + except Exception as e: + self.log("closing the result failed: %s" % e) os._exit(124) # --- monitors: what must stop the run while a long call is waiting diff --git a/src/bench/macbench/scale.py b/src/bench/macbench/scale.py index 709e546..578ce83 100644 --- a/src/bench/macbench/scale.py +++ b/src/bench/macbench/scale.py @@ -65,7 +65,7 @@ def run_scale(ctx): if first: if mix in ("all-busy", "one-busy"): data["calibration"] = calibrate(ctx, ctx.sims[0]) - core.settle(ctx, "before the input calibration", max_s=60, bg=90) + core.settle(ctx, "before the input calibration", bg=90) if a.golden_phase: ctx.pixel_gate = golden_phase(ctx) core.grow(ctx, n, roles) diff --git a/src/bench/tests/test_dry_guards.py b/src/bench/tests/test_dry_guards.py index 6d51ec5..e10b4b7 100644 --- a/src/bench/tests/test_dry_guards.py +++ b/src/bench/tests/test_dry_guards.py @@ -16,7 +16,7 @@ def results(): if not _R: cases = {"diskstart": (SWEEP, "diskstart"), "disklow": (SWEEP, "disklow@2"), "diskabort": (SWEEP, "diskabort@2"), "diskunreadable": (SWEEP, "diskunreadable"), "qemugone": (SWEEP, "qemugone@2"), - "stockdisk": (["scale", "--n", "2", "--stock-devices"] + FAST[:], "disklow@2"), "clean": (SWEEP, "")} + "transition": (SWEEP, "transition@1"), "stockdisk": (["scale", "--n", "2", "--stock-devices"] + FAST[:], "disklow@2"), "clean": (SWEEP, "")} with concurrent.futures.ThreadPoolExecutor(8) as ex: futs = {k: ex.submit(run_case, a, f) for k, (a, f) in cases.items()} _R.update({k: f.result() for k, f in futs.items()}) @@ -66,6 +66,27 @@ def test_a_restarted_vm_aborts_the_run(self): self.assertIn("QEMU", sections(r["doc"], "abort")[0]["line"]) self.assertEqual(sims_booted(r["state"]), []) + def test_a_bench_device_in_transition_is_not_a_foreign_simulator(self): + r = results()["transition"] # a launchd_sim that names a bench-set device in any state is the bench's own + self.assertEqual(r["rc"], 0, r["out"][-800:]) + self.assertEqual(sections(r["doc"], "abort") + sections(r["doc"], "foreign"), []) + self.assertEqual([s["n"] for s in r["doc"]["data"]["steps"]], [1, 2, 3]) + for s in r["doc"]["data"]["steps"]: + self.assertEqual(s["stability"]["foreign_booted"], 0) + + def test_an_abort_is_not_a_race_with_the_main_thread(self): + """The monitor thread writes the result while the main thread, whose calls it killed, unwinds: the result must always be + whole, the cleanup verified and the exit code the monitor's, every time.""" + cases = [(SWEEP, "diskabort@2", 8), (SWEEP, "qemugone@2", 9)] * 6 + with concurrent.futures.ThreadPoolExecutor(8) as ex: + rs = list(ex.map(lambda c: (run_case(c[0], c[1]), c[2]), cases)) + for r, code in rs: + self.assertEqual(r["rc"], code, r["out"][-600:]) + self.assertIsNotNone(r["doc"], r["out"][-600:]) + self.assertEqual(sections(r["doc"], "cleanup")[0]["status"], "PASS") + self.assertEqual([s["name"] for s in r["doc"]["sections"]].count("abort"), 1) + self.assertEqual([s["name"] for s in r["doc"]["sections"] if s["name"] in ("stopped", "error")], []) + def test_a_clean_run_is_not_aborted(self): r = results()["clean"] self.assertEqual(r["rc"], 0, r["out"][-500:]) diff --git a/src/bench/tests/test_dry_seed.py b/src/bench/tests/test_dry_seed.py index db16cab..139f5b0 100644 --- a/src/bench/tests/test_dry_seed.py +++ b/src/bench/tests/test_dry_seed.py @@ -95,7 +95,7 @@ def job_seedcmd(): "nosourceset": job_fail("nosourceset"), "slimfail": job_fail_then_ok("slimfail"), "slimfew": job_fail_then_ok("slimfew"), - "noicons": job_fail_then_ok("noicons", ONE + ["--seed-settle-max", "2"]), + "noicons": lambda: (lambda st: {"state": st, "r": run_case(ONE, "noicons", state=st)})(new_state()), "noinstall": job_fail_then_ok("noinstall"), "stale": job_stale, "seedcmd": job_seedcmd, @@ -211,8 +211,10 @@ def test_simslim_failure(self): def test_a_seed_that_is_not_slim_enough_fails(self): self.failing("slimfew", "not Slim after simslim: 20 services off") - def test_a_seed_whose_widgets_never_settle_fails(self): - self.failing("noicons", "the seed did not settle") + def test_a_seed_without_the_widgets_icons_folder_is_recorded_not_refused(self): + r = parallel(JOBS)["noicons"]["r"] # the folder is not in the verified recipe: an unknown, never a gate + self.assertEqual(r["rc"], 0, r["out"][-500:]) + self.assertIs(r["doc"]["env"]["guest"]["seed"]["icons"], False) def test_the_app_must_be_in_the_seed(self): self.failing("noinstall", "not installed in the seed") diff --git a/src/bench/tests/test_dry_state.py b/src/bench/tests/test_dry_state.py index 58b09aa..8171e90 100644 --- a/src/bench/tests/test_dry_state.py +++ b/src/bench/tests/test_dry_state.py @@ -2,10 +2,14 @@ Each case is a real `run` process (stub guest, fake host); nothing real is touched.""" import json import os +import signal +import subprocess +import sys import tempfile +import time import unittest -from tests.test_dry import FAST, run_case, sections, simctl_calls +from tests.test_dry import FAST, RUN, run_case, sections, simctl_calls ONE = ["scale", "--n", "1,2"] + FAST @@ -91,3 +95,46 @@ def test_a_refusing_tool_stops_the_run(self): if __name__ == "__main__": unittest.main() + + +def devices_of(state): + with open(os.path.join(state, "guest", "state.json")) as f: + return json.load(f)["devs"] + + +def kill_mid_seed(state): + """A run killed outright (no signal handler, no exit trap, the guest helper gone with it) while its seed is half made: + what an ssh drop or a SIGKILL leaves behind.""" + d = tempfile.mkdtemp(prefix="dry-kill-") + env = dict(os.environ, DRY="1", BENCH_STUB_FAULT="slowslim", BENCH_DRY_KEEP=state, BENCH_DEADLINE_MARGIN="3") + p = subprocess.Popen([sys.executable, RUN] + ONE + ["--out", os.path.join(d, "out")], env=env, stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, start_new_session=True) + t0 = time.time() + try: + while time.time() - t0 < 60 and not any(c[0].endswith("simslim") for c in (calls(state) if os.path.exists(os.path.join(state, "guest", "calls.log")) else [])): + time.sleep(0.2) + finally: + os.killpg(p.pid, signal.SIGKILL) + p.wait() + return any(c[0].endswith("simslim") for c in calls(state)) + + +class HalfMadeSeed(unittest.TestCase): + def test_a_killed_seed_is_refused_then_cleanup_removes_it_and_the_next_run_works(self): + state = tempfile.mkdtemp(prefix="dry-half-") + self.assertTrue(kill_mid_seed(state), "the run never reached simslim") + left = devices_of(state) + self.assertEqual(len(left), 1) # the half-made seed + self.assertEqual(files(state, "sim-keep"), list(left)) + r = run_case(ONE, "", state=state) # the next run refuses to start on it + self.assertNotEqual(r["rc"], 0) + self.assertIn("never completed", r["out"]) + self.assertIn("macbox bench cleanup", r["out"]) + c = run_case(["cleanup"], "", state=state) # a plain cleanup (no --seeds) removes it + self.assertEqual(c["rc"], 0, c["out"][-600:]) + self.assertIn("CLEANUP PASS", c["out"]) + self.assertEqual(devices_of(state), {}) + self.assertEqual(files(state, "sim-keep"), []) + self.assertFalse(os.path.exists(os.path.join(state, "guest", "set", ".bench-run.lock"))) + ok = run_case(ONE, "", state=state) + self.assertEqual(ok["rc"], 0, ok["out"][-600:]) diff --git a/src/bench/tests/test_modules.py b/src/bench/tests/test_modules.py index 27c05c8..cebf5cc 100644 --- a/src/bench/tests/test_modules.py +++ b/src/bench/tests/test_modules.py @@ -108,6 +108,18 @@ def test_count_launchd_sim_sees_every_device_set(self): finally: bg.ps_text = saved + def test_foreign_counts_a_bench_device_that_is_not_booted_as_the_benchs_own(self): + text = PS + " 600 1 20000 0:01.00 /Library/x/launchd_sim /bench/set/CCCC-3/data\n" # CCCC-3 is booting: state Shutdown + devs = [{"udid": "CCCC-3", "state": "Shutdown"}, {"udid": "AAAA-1", "state": "Booted"}, {"udid": "BBBB-2", "state": "Booted"}] + saved = bg.ps_text + bg.ps_text = lambda: text + try: + self.assertEqual(bg.count_launchd_sim(), 2) + self.assertEqual(bg.foreign_count(devs), 0) + self.assertEqual(bg.foreign_count(devs[1:2]), 1) # the same process with no device of the set behind it: foreign + finally: + bg.ps_text = saved + def test_qemu_env_value_reads_the_process_environ(self): with tempfile.TemporaryDirectory() as root: os.makedirs(os.path.join(root, "proc", "77")) diff --git a/src/bench/tests/test_runguards.py b/src/bench/tests/test_runguards.py index 8215139..95a7318 100644 --- a/src/bench/tests/test_runguards.py +++ b/src/bench/tests/test_runguards.py @@ -124,10 +124,25 @@ def test_a_vm_that_stops_answering_twice_in_a_row_ends_the_run_with_9(self): c.guest.up = False self.assertIsNone(check()) - def test_someone_elses_simulator_makes_the_run_yield_with_7(self): - self.assertEqual(monitor_result(Ctx(guest=FakeGuest(foreign=1)))[1], 7) + def test_someone_elses_simulator_makes_the_run_yield_with_7_when_two_polls_in_a_row_see_it(self): + c = Ctx(guest=FakeGuest(foreign=1)) + c.a.watch_guest_every = 0.0 + check = core.monitor_check(c) + self.assertIsNone(check()) # the bench's own boots and shutdowns race the count: one sighting is not a verdict + self.assertEqual(check()[1], 7) self.assertIsNone(monitor_result(Ctx(guest=FakeGuest(foreign=1), allow_foreign=True))) + def test_a_sighting_that_goes_away_resets_the_count(self): + c = Ctx(guest=FakeGuest(foreign=1)) + c.a.watch_guest_every = 0.0 + check = core.monitor_check(c) + self.assertIsNone(check()) + c.guest.foreign = 0 + self.assertIsNone(check()) + c.guest.foreign = 1 + self.assertIsNone(check()) + self.assertEqual(check()[1], 7) + def test_swap_growth_and_the_compressor_end_the_run_with_10(self): self.assertEqual(monitor_result(Ctx(guest=FakeGuest(mem={"swapouts": 9, "compressed_mb": 1})))[1], 10) self.assertEqual(monitor_result(Ctx(guest=FakeGuest(mem={"swapouts": 7, "compressed_mb": 9999})))[1], 10) @@ -141,6 +156,31 @@ def test_a_measurement_window_is_left_alone(self): self.assertEqual(monitor_result(c)[1], 8) # but the host-side checks go on +class LifecycleGuard(unittest.TestCase): + def test_every_lifecycle_cycle_boots_behind_the_guard(self): + from macbench import others + stopped, cycles, seen = [], [], [] + c = Ctx(guest=FakeGuest()) + c.a = cli.parser().parse_args(["lifecycle", "--cycles", "3", "--storm", "0"]) + c.sims, c.seed, c.type_id, c.runtime, c.stock = [], {"seed": "S"}, "t", "r", False + c.marks, c.doc = {}, {"data": {}, "gates": {}, "sections": []} + c.coh = type("Coh", (), {"mark": lambda self: 0, "check": lambda self, m: {}})() + c.host.snap = lambda: {"mem": {}} + c.section = lambda name, status, line, extra=None: stopped.append((name, status, line)) + c.save = lambda: None + orig = (core.setup_guest, core.capture_env, core.guard, core.keep_minutes) + core.setup_guest, core.capture_env, core.keep_minutes = (lambda ctx: {}), (lambda ctx, info: None), (lambda ctx: 30) + core.guard = lambda ctx, k, label: (seen.append(label), "disk is low" if len(seen) == 2 else None)[1] + c.guest.call = lambda args, timeout=60, counted=True: (cycles.append(args[0]) or {"timings": {"clone_s": 1.0}, "mem": {"used_mb": 1, "compressed_mb": 1}}) + try: + others.run_lifecycle(c) + finally: + core.setup_guest, core.capture_env, core.guard, core.keep_minutes = orig + self.assertEqual(seen[:2], ["before lifecycle cycle 1", "before lifecycle cycle 2"]) + self.assertEqual(cycles.count("cycle"), 1) # the second was refused + self.assertIn(("stopped", "FAIL", "STOPPED: disk is low"), stopped) + + class QemuIdentity(unittest.TestCase): def test_reads_a_real_process(self): h = hostprobe.Host(os.getpid()) From 5884b878e91f657651e7ff3a8b120bce84c21501 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Tue, 17 Nov 2026 07:07:18 +0200 Subject: [PATCH 19/23] bench: judge crash reports of the bench and the graphics path only A Mac that runs simulators files crash reports for other daemons all day (widget extensions, searchd, UsageTrackingAgent). One of those failed the first run on a VM. New reports of the bench's own programs and of the processes the measured frames pass through still fail a step and the run; the others are recorded (stability.other_ips, the gate's text). --strict-crashes judges every report, --crash-names adds names. --- src/bench/README.md | 9 ++++++-- src/bench/guest/benchguest_stub.py | 7 ++++--- src/bench/macbench/cli.py | 2 ++ src/bench/macbench/core.py | 11 +++++++--- src/bench/macbench/gates.py | 32 +++++++++++++++++++++++++++++ src/bench/tests/test_dry.py | 12 +++++++++++ src/bench/tests/test_pure.py | 33 +++++++++++++++++++++++++++++- 7 files changed, 97 insertions(+), 9 deletions(-) diff --git a/src/bench/README.md b/src/bench/README.md index 74b78c3..8ef60a0 100644 --- a/src/bench/README.md +++ b/src/bench/README.md @@ -210,7 +210,12 @@ animating simulators in the window. i915 only (xe states cycles, not ns: the fie change over the window, `cgroup_events` (high/max/oom_kill deltas), `pswpin_pages`/`pswpout_pages` (system wide), `psi_mem_some_pct` (system) and `cgroup_psi_some_pct` (the VM's cgroup). -**Stability** (`stability`): new `.ips`/`.crash` files in the guest's DiagnosticReports during the window, +**Stability** (`stability`): new crash reports (`.ips`/`.crash` in the guest's DiagnosticReports) during the window +of the bench's own programs or of the simulator's graphics path (`CRASH_NAMES` in `macbench/gates.py`: GlassBench, +simfps, SpringBoard, backboardd, SimRenderServer, SimMetalHost, CoreSimulatorService, the kernel, jetsam events ...). +A Mac that runs simulators files reports for other daemons all day (widget extensions, searchd, containermanagerd); +those are recorded in `other_ips` and in the gate's text, and fail nothing unless `--strict-crashes` is given. +`--crash-names a,b` (or `BENCH_CRASH_NAMES`) adds names to judge. Also: simulators no longer Booted at its end, kernel GPU error lines (`journalctl -k`: GPU hang, fence expired, context reset, xe job timeouts and faults), QEMU still the same process. Any one makes the step unstable and ends the sweep. @@ -243,7 +248,7 @@ written first. A gate that cannot be read is SKIP, shown as such, never PASS. | `kernel_gpu` | no kernel GPU error line since the step began (SKIP when the journal is not readable) | | `counter` | `simfps` answered with a result for every simulator of the window and every animating simulator's surfaces gave a readable frame counter (a counter that answers nothing or an error fails the window, it is not a missing metric) | | `memory` | the VM's own cgroup swapped no more than 50 MB (SKIP when the cgroup is unreadable: other projects' swapping on the host is not this run's); guest swap-outs fail it with `--fail-on-swap` | -| `run_*` | every scenario, after its body: `run_coherent_census`, `run_coherent_process`, `run_qemu_log`, `run_gpu_hang` and `run_kernel_gpu` since the run began, and `run_crash_reports` (a new `.ips` on the guest since setup). They fail the exit code of `idle`, `lifecycle`, `soak`, `ab` too. | +| `run_*` | every scenario, after its body: `run_coherent_census`, `run_coherent_process`, `run_qemu_log`, `run_gpu_hang` and `run_kernel_gpu` since the run began, and `run_crash_reports` (a new crash report since setup, judged like `stability`'s). They fail the exit code of `idle`, `lifecycle`, `soak`, `ab` too. | | `leak_*` | `lifecycle`: QEMU RSS and guest memory grow less than `--leak-rss-mb` / `--leak-guest-mb` per create/boot/delete cycle (least-squares slope); `soak`: less than `--soak-rss-mb-min` / `--soak-guest-mb-min` per minute | | `observer` | reading the strip does not move the surface frame rate by more than 4% (a second window of the first N without the strip read); otherwise the distinct-frame numbers are lower bounds | diff --git a/src/bench/guest/benchguest_stub.py b/src/bench/guest/benchguest_stub.py index 11142f7..9f9516b 100644 --- a/src/bench/guest/benchguest_stub.py +++ b/src/bench/guest/benchguest_stub.py @@ -389,8 +389,9 @@ def ips(): s = load() calls[0] += 1 cr = fault("crash") - if cr is not None and len(booted(s)) >= cr and calls[0] >= 2 and "GlassBench-stub.ips" not in s["ips"]: - s["ips"].append("GlassBench-stub.ips") # stays on the "disk": the run-wide gate sees it later - save(s) + for cr, name in ((cr, "GlassBench-2026-01-02-030405.ips"), (fault("daemoncrash"), "searchd-2026-01-02-030405.ips")): + if cr is not None and len(booted(s)) >= cr and calls[0] >= 2 and name not in s["ips"]: + s["ips"].append(name) # stays on the "disk": the run-wide gate sees it later + save(s) return set(s["ips"]) mod.list_ips = ips diff --git a/src/bench/macbench/cli.py b/src/bench/macbench/cli.py index 82c6d1c..8bad945 100644 --- a/src/bench/macbench/cli.py +++ b/src/bench/macbench/cli.py @@ -52,6 +52,8 @@ def add_common(p): p.add_argument("--slim-profile", default=os.environ.get("BENCH_SLIM_PROFILE"), help="simslim profile in the guest (default: the only .json in ~/.config/simslim)") p.add_argument("--seed-settle-max", type=float, default=600, help="the seed must settle within this many seconds or it is deleted and the run fails") p.add_argument("--quiesce", action="store_true", help="disable mediaanalysisd in each simulator (recorded; production does not)") + p.add_argument("--strict-crashes", action="store_true", help="any new crash report on the Mac fails the run (default: only reports of the bench's programs and the simulator's graphics path; the others are listed)") + p.add_argument("--crash-names", default=os.environ.get("BENCH_CRASH_NAMES", ""), help="more process names whose crash report fails the run, comma-separated") p.add_argument("--fail-on-swap", action="store_true", help="a guest swap-out in a window fails the memory gate (default: reported, host swap fails)") p.add_argument("--golden-file", default=os.path.join(BENCH_DIR, "baselines", "golden.json")) diff --git a/src/bench/macbench/core.py b/src/bench/macbench/core.py index a088556..8610075 100644 --- a/src/bench/macbench/core.py +++ b/src/bench/macbench/core.py @@ -422,7 +422,8 @@ def build_step(ctx, n, mix, g, s0, s1, t_go, boot=None): died = [s["idx"] for s in sims if g["sims"][s["idx"]].get("state_after") != "Booted"] kern, knote = ctx.host.kernel_gpu(s0["wall"]) foreign = max((g.get("foreign_booted") or {}).values() or [0]) - stab = {"new_ips": len(g["new_ips"]), "ips": g["new_ips"][:5], "sims_died": died, "kernel_gpu_lines": kern, "kernel_note": knote, + judged, listed = gates.crash_split(g["new_ips"], crash_names(a), getattr(a, "strict_crashes", False)) + stab = {"new_ips": len(judged), "ips": judged[:5], "other_ips": listed[:10], "sims_died": died, "kernel_gpu_lines": kern, "kernel_note": knote, "qemu_same": ctx.host.same_qemu(), "observer_errors": [s["idx"] for s in sims if s.get("observer_error")], "foreign_booted": foreign} stab["ok"] = not (stab["new_ips"] or died or kern or not stab["qemu_same"] or (foreign and not a.allow_foreign)) @@ -469,6 +470,11 @@ def step_gates(ctx, step, shots=None): return gs +def crash_names(a): + """Extra process names whose crash report fails a run (--crash-names a,b).""" + return [n for n in (getattr(a, "crash_names", None) or "").split(",") if n.strip()] + + def run_gates(ctx): """The gates that hold for a whole run, whatever the scenario: Coherent and QEMU's log since the run began, kernel GPU error lines since then, and crash reports that appeared on the guest. A scenario that stopped before its setup @@ -489,8 +495,7 @@ def run_gates(ctx): gs["run_kernel_gpu"] = gates.gate("SKIP", "unreadable: %s" % type(e).__name__) try: now = set(ctx.guest.call(["ips"], 60)["ips"]) - new = sorted(now - mk["ips"]) - gs["run_crash_reports"] = gates.gate("FAIL" if new else "PASS", ("%d new crash report(s): %s" % (len(new), ", ".join(new[:3]))) if new else "no new crash report on the guest") + gs["run_crash_reports"] = gates.crash_gate(now - mk["ips"], crash_names(ctx.a), getattr(ctx.a, "strict_crashes", False)) except Exception as e: gs["run_crash_reports"] = gates.gate("SKIP", "guest unreachable: %s" % type(e).__name__) ctx.doc.setdefault("gates", {}).update(gs) diff --git a/src/bench/macbench/gates.py b/src/bench/macbench/gates.py index 8fb1054..2cf3117 100644 --- a/src/bench/macbench/gates.py +++ b/src/bench/macbench/gates.py @@ -12,6 +12,38 @@ def gate(status, detail=""): return {"status": status, "detail": detail} +# Crash reports that fail a run: the bench's own programs and the processes the measured frames pass through. +# macOS files reports for other daemons of a simulator all day (widget extensions, searchd, containermanagerd ...); +# those are listed with the gate, not judged, unless --strict-crashes says so. +CRASH_NAMES = ("GlassBench", "GenieBench", "simfps", "SpringBoard", "backboardd", "SimRenderServer", "SimMetalHost", + "WindowServer", "com.apple.CoreSimulator.CoreSimulatorService", "CoreSimulatorBridge", "launchd_sim", + "Simulator", "simctl", "Kernel", "JetsamEvent") +_CRASH_STAMP = re.compile(r"[-_.]\d{4}-\d{2}-\d{2}[-_]\d{6}.*$") + + +def crash_proc(report): + """The process a crash report's file name names: 'searchd-2026-01-02-030405.ips' -> 'searchd'.""" + n = _CRASH_STAMP.sub("", report) + return re.sub(r"\.(ips|crash)$", "", n) + + +def crash_split(reports, extra=(), strict=False): + """(judged, listed): new crash reports that fail a gate, and those that are only shown.""" + names = {n.lower() for n in CRASH_NAMES} | {n.strip().lower() for n in extra if n.strip()} + judged, listed = [], [] + for r in sorted(reports): + (judged if strict or crash_proc(r).lower() in names else listed).append(r) + return judged, listed + + +def crash_gate(reports, extra=(), strict=False): + judged, listed = crash_split(reports, extra, strict) + also = ("; not judged: %d other report(s): %s" % (len(listed), ", ".join(listed[:5]))) if listed else "" + if judged: + return gate("FAIL", "%d new crash report(s): %s%s" % (len(judged), ", ".join(judged[:3]), also)) + return gate("PASS", "no new crash report of the bench or the simulator's graphics path" + also) + + def worst(gates): """FAIL if any FAIL; else SKIP-only/mixed is still ok: a run PASSes when nothing FAILed (SKIPs stay visible).""" return not any(g["status"] == "FAIL" for g in gates.values()) diff --git a/src/bench/tests/test_dry.py b/src/bench/tests/test_dry.py index 1158858..f51275f 100644 --- a/src/bench/tests/test_dry.py +++ b/src/bench/tests/test_dry.py @@ -66,6 +66,7 @@ def sections(doc, name): "refuse": (["scale", "--n", "1,2"] + FAST, "memfloor@0"), "hostmem": (["scale", "--n", "1,2,3,4,5"] + FAST, "hostmem@3"), "crash": (["scale", "--n", "1,2,3,4"] + FAST, "crash@2"), + "daemoncrash": (["scale", "--n", "1,2"] + FAST, "daemoncrash@2"), "backwards": (["scale", "--n", "1,2,3"] + FAST, "backwards@2"), "kernelgpu": (["scale", "--n", "1,2,3"] + FAST, "kernelgpu@2"), "coherent": (["scale", "--n", "1,2,3"] + FAST, "coherent@2"), @@ -161,6 +162,17 @@ def test_run_wide_gates_catch_what_a_step_gate_does_not_name(self): self.assertEqual(g["run_crash_reports"]["status"], "PASS", k) self.assertIn("run_kernel_gpu", g) + def test_another_daemons_crash_report_is_listed_not_judged(self): + r = self.r["daemoncrash"] + d = r["doc"] + self.assertEqual(r["rc"], 0, r["out"][-600:]) + self.assertEqual([st["n"] for st in d["data"]["steps"]], [1, 2]) + self.assertTrue(all(st["stable"] for st in d["data"]["steps"])) + self.assertEqual(d["data"]["steps"][-1]["stability"]["other_ips"], ["searchd-2026-01-02-030405.ips"]) + g = d["gates"]["run_crash_reports"] + self.assertEqual(g["status"], "PASS") + self.assertIn("not judged: 1 other report(s): searchd-", g["detail"]) + def test_crash_report_is_instability_and_ends_the_sweep(self): d = self.r["crash"]["doc"] self.assertEqual(d["data"]["steps"][-1]["n"], 2) diff --git a/src/bench/tests/test_pure.py b/src/bench/tests/test_pure.py index 5625623..0482324 100644 --- a/src/bench/tests/test_pure.py +++ b/src/bench/tests/test_pure.py @@ -3,10 +3,41 @@ import unittest sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) -from macbench import abrule, analysis, png, stats, strip, targets, tears # noqa: E402 +from macbench import abrule, analysis, gates, png, stats, strip, targets, tears # noqa: E402 from tests import helpers # noqa: E402 +class CrashReports(unittest.TestCase): + NEW = ["searchd-2026-11-17-065234.ips", "SpringBoard-2026-11-17-065301.ips", "GlassBench_2026-11-17-065302_Mac.crash", + "PassbookWidgetsExtension-iPhone-2026-11-17-065303.ips", "com.apple.CoreSimulator.CoreSimulatorService-2026-11-17-070000.ips"] + + def test_process_name_of_a_report(self): + self.assertEqual(gates.crash_proc("searchd-2026-11-17-065234.ips"), "searchd") + self.assertEqual(gates.crash_proc("PassbookWidgetsExtension-iPhone-2026-11-17-065303.ips"), "PassbookWidgetsExtension-iPhone") + self.assertEqual(gates.crash_proc("GlassBench_2026-11-17-065302_Mac.crash"), "GlassBench") + self.assertEqual(gates.crash_proc("odd.ips"), "odd") + + def test_only_the_bench_and_the_graphics_path_are_judged(self): + judged, listed = gates.crash_split(self.NEW) + self.assertEqual([gates.crash_proc(r) for r in judged], ["GlassBench", "SpringBoard", "com.apple.CoreSimulator.CoreSimulatorService"]) + self.assertEqual([gates.crash_proc(r) for r in listed], ["PassbookWidgetsExtension-iPhone", "searchd"]) + + def test_more_names_and_strict(self): + judged, listed = gates.crash_split(self.NEW, extra=["SearchD", " "]) + self.assertIn("searchd-2026-11-17-065234.ips", judged) + judged, listed = gates.crash_split(self.NEW, strict=True) + self.assertEqual((len(judged), listed), (5, [])) + + def test_gate_names_what_it_did_not_judge(self): + g = gates.crash_gate(["searchd-2026-11-17-065234.ips"]) + self.assertEqual(g["status"], "PASS") + self.assertIn("not judged: 1 other report(s): searchd-2026-11-17-065234.ips", g["detail"]) + g = gates.crash_gate(self.NEW) + self.assertEqual(g["status"], "FAIL") + self.assertIn("3 new crash report(s)", g["detail"]) + self.assertEqual(gates.crash_gate([])["status"], "PASS") + + class Stats(unittest.TestCase): def test_percentile_matches_simfps_rule(self): v = list(range(1, 101)) From faefd567dbb601b06725408b62a862edf213f211 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Tue, 17 Nov 2026 08:08:06 +0200 Subject: [PATCH 20/23] bench: one lock per surface in simfps; the observer check's window reads nothing simfps locked every surface twice when it read both the frame strip and the input marker. A lock of a surface the simulator just drew makes it wait for the pixels: with two per surface one 3x simulator composed 54-58 frames/s instead of 104-110 (measured, A B B A, 12 s windows). Strip and marker are now read under one lock (108-110). The observer check's second window never got --no-strip, so it compared two identical windows. It now reads neither strip nor marker and injects nothing: simfps locks no surface there. --- src/bench/guest/benchguest.py | 14 +++++--- src/bench/guest/benchguest_stub.py | 5 ++- src/bench/macbench/core.py | 3 ++ src/bench/simfps.swift | 51 +++++++++++++++--------------- src/bench/tests/test_dry.py | 9 ++++++ 5 files changed, 50 insertions(+), 32 deletions(-) diff --git a/src/bench/guest/benchguest.py b/src/bench/guest/benchguest.py index cbe1faf..c4ae9d6 100644 --- a/src/bench/guest/benchguest.py +++ b/src/bench/guest/benchguest.py @@ -62,10 +62,13 @@ def run(argv, timeout=120, env=None): return 127, "", str(e), time.time() - t0 -def spawn_simfps(udid, secs, label, strip, start_at, raw_path): - """Starts simfps; returns an object whose .wait(timeout) -> (json dict or None, error or None).""" - env = dict(os.environ, SIMFPS_SET=SET, SIMFPS_START_AT="%.3f" % start_at, SIMFPS_STAMPS=raw_path, - SIMFPS_PROBE=PROBE_POINT) +def spawn_simfps(udid, secs, label, strip, start_at, raw_path, probe=True): + """Starts simfps; returns an object whose .wait(timeout) -> (json dict or None, error or None). + strip and probe are what it reads from every surface; with neither it locks no surface (an unviewed screen).""" + env = dict(os.environ, SIMFPS_SET=SET, SIMFPS_START_AT="%.3f" % start_at, SIMFPS_STAMPS=raw_path) + env.pop("SIMFPS_PROBE", None) + if probe: + env["SIMFPS_PROBE"] = PROBE_POINT argv = [os.path.join(HERE, "..", "simfps"), udid, str(secs), label] + (["--strip"] if strip else []) p = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env) @@ -909,7 +912,8 @@ def run_window(a, udids, roles, mix, secs, t_go, tmp, probe_method, probe_every, if mix != "idle": for i, u in enumerate(udids): raws[u] = os.path.join(tmp, "raw-%d.txt" % i) - handles[u] = spawn_simfps(u, secs, "bench-%d" % i, roles[i] != "static" and not no_strip, t_go, raws[u]) + handles[u] = spawn_simfps(u, secs, "bench-%d" % i, roles[i] != "static" and not no_strip, t_go, raws[u], + probe_method != "none" and not no_strip) ips0, ps0, vm0, lvl0 = list_ips(), None, None, pressure_level() d = t_go - now() if d > 0: diff --git a/src/bench/guest/benchguest_stub.py b/src/bench/guest/benchguest_stub.py index 9f9516b..4cb49bd 100644 --- a/src/bench/guest/benchguest_stub.py +++ b/src/bench/guest/benchguest_stub.py @@ -380,7 +380,10 @@ def install(mod): mod.ps_text = stub_ps mod.vmstat_text = stub_vmstat mod.sysctl = lambda name: {"kern.memorystatus_vm_pressure_level": "1", "hw.ncpu": "8", "hw.memsize": str(16 << 30)}.get(name, "") - mod.spawn_simfps = lambda udid, secs, label, strip, start_at, raw: StubSimfps(udid, secs, strip, start_at, raw) + def spawn(udid, secs, label, strip, start_at, raw, probe=True): + _log(["simfps", udid, "strip" if strip else "nostrip", "probe" if probe else "noprobe"]) + return StubSimfps(udid, secs, strip, start_at, raw) + mod.spawn_simfps = spawn calls = [0] diff --git a/src/bench/macbench/core.py b/src/bench/macbench/core.py index 8610075..946e813 100644 --- a/src/bench/macbench/core.py +++ b/src/bench/macbench/core.py @@ -347,6 +347,9 @@ def window(ctx, secs, mix, no_strip=False): "--probe-method", ctx.probe_method or "openurl", "--probe-every", a.probe_every, "--agent-period", a.agent_period, "--probe-timeout", a.probe_timeout] if a.probe_cmd: args += ["--probe-cmd", a.probe_cmd] + if no_strip: # the unviewed screen: no strip, no marker, no injection, so simfps locks no surface + args[args.index("--probe-method") + 1] = "none" + args.append("--no-strip") ctx.in_window = True try: pend = ctx.guest.start(args, secs + lead + 240) diff --git a/src/bench/simfps.swift b/src/bench/simfps.swift index c1d9ab2..1540212 100644 --- a/src/bench/simfps.swift +++ b/src/bench/simfps.swift @@ -110,36 +110,35 @@ let probe: (Double, Double)? = env["SIMFPS_PROBE"].flatMap { s -> (Double, Doubl return p.count == 2 ? (p[0], p[1]) : nil } var raw: [(UInt64, Int, Int)] = [] // (time, counter or -1, probe luma or -1) of every new surface -/// The luma (0-255) at a point given in points of a 402 pt wide screen, or -1. -func readLuma(_ s: IOSurface, _ pt: (Double, Double)) -> Int { +/// One look at a BGRA surface under ONE lock: GlassBench's counter (18 blocks of 20 pt from (21, 120) pt, at the +/// surface's scale; -1 when unreadable or not asked for) and the luma (0-255) at the probe point (-1 without one). +/// A lock of a surface the simulator just drew makes it wait for the pixels: two locks per surface halved the +/// simulator's frame rate (55 against 102 screen frames/s, one 3x simulator), one lock costs a few percent. +func readSurface(_ s: IOSurface, strip: Bool, probe: (Double, Double)?) -> (Int, Int) { + guard strip || probe != nil else { return (-1, -1) } let w = IOSurfaceGetWidth(s), h = IOSurfaceGetHeight(s), bpr = IOSurfaceGetBytesPerRow(s) let scale = Double(w) / 402.0 - guard IOSurfaceLock(s, .readOnly, nil) == kIOReturnSuccess else { return -1 } - defer { IOSurfaceUnlock(s, .readOnly, nil) } - let x = Int(pt.0 * scale), y = Int(pt.1 * scale) - guard x < w, y < h else { return -1 } - let p = IOSurfaceGetBaseAddress(s).assumingMemoryBound(to: UInt8.self) + y * bpr + x * 4 - return (Int(p[0]) + Int(p[1]) + Int(p[2])) / 3 -} -/// GlassBench's counter from a BGRA surface: 18 blocks of 20 pt from (21, 120) pt, at the surface's scale. -func readCounter(_ s: IOSurface) -> Int { - let w = IOSurfaceGetWidth(s), h = IOSurfaceGetHeight(s), bpr = IOSurfaceGetBytesPerRow(s) - let scale = Double(w) / 402.0 - guard IOSurfaceLock(s, .readOnly, nil) == kIOReturnSuccess else { return -1 } + guard IOSurfaceLock(s, .readOnly, nil) == kIOReturnSuccess else { return (-1, -1) } defer { IOSurfaceUnlock(s, .readOnly, nil) } let base = IOSurfaceGetBaseAddress(s).assumingMemoryBound(to: UInt8.self) - func luma(_ i: Int) -> Int { - let x = Int((21 + 20 * Double(i) + 10) * scale), y = Int(130 * scale) - guard x < w, y < h else { return 0 } + func lumaAt(_ px: Double, _ py: Double) -> Int? { + let x = Int(px * scale), y = Int(py * scale) + guard x < w, y < h else { return nil } let p = base + y * bpr + x * 4 return (Int(p[0]) + Int(p[1]) + Int(p[2])) / 3 } - let white = luma(0), black = luma(1) - guard white - black > 100 else { return -1 } - let mid = (white + black) / 2 - var n = 0 - for b in 0..<16 where luma(b + 2) > mid { n |= 1 << b } - return n + var counter = -1 + if strip { + func block(_ i: Int) -> Int { lumaAt(21 + 20 * Double(i) + 10, 130) ?? 0 } + let white = block(0), black = block(1) + if white - black > 100 { + let mid = (white + black) / 2 + var n = 0 + for b in 0..<16 where block(b + 2) > mid { n |= 1 << b } + counter = n + } + } + return (counter, probe.flatMap { lumaAt($0.0, $0.1) } ?? -1) } stamps.reserveCapacity(Int(secs * 200)) var polls = 0 @@ -155,7 +154,7 @@ let poller = Thread { var lastSeed = IOSurfaceGetSeed(surf0) var lastSurf = surf0 // The first line of SIMFPS_STAMPS is the screen as it stands at the start (a static screen makes no new surface). - if rawPath != nil { raw.append((start, readStrip ? readCounter(surf0) : -1, probe.map { readLuma(surf0, $0) } ?? -1)) } + if rawPath != nil { let r = readSurface(surf0, strip: readStrip, probe: probe); raw.append((start, r.0, r.1)) } while true { let t = nowNs() if t >= end { break } @@ -163,9 +162,9 @@ let poller = Thread { let seed = IOSurfaceGetSeed(s) if s !== lastSurf || seed != lastSeed { stamps.append(t); lastSeed = seed; lastSurf = s - let counter = readStrip ? readCounter(s) : -1 + let (counter, luma) = readSurface(s, strip: readStrip, probe: rawPath != nil ? probe : nil) if readStrip { strip.append((t, counter)) } - if rawPath != nil { raw.append((t, counter, probe.map { readLuma(s, $0) } ?? -1)) } + if rawPath != nil { raw.append((t, counter, luma)) } } polls += 1 usleep(500) diff --git a/src/bench/tests/test_dry.py b/src/bench/tests/test_dry.py index f51275f..d614f5e 100644 --- a/src/bench/tests/test_dry.py +++ b/src/bench/tests/test_dry.py @@ -67,6 +67,7 @@ def sections(doc, name): "hostmem": (["scale", "--n", "1,2,3,4,5"] + FAST, "hostmem@3"), "crash": (["scale", "--n", "1,2,3,4"] + FAST, "crash@2"), "daemoncrash": (["scale", "--n", "1,2"] + FAST, "daemoncrash@2"), + "observer": (["scale", "--n", "1"] + [x for x in FAST if x != "--no-observer-check"], ""), "backwards": (["scale", "--n", "1,2,3"] + FAST, "backwards@2"), "kernelgpu": (["scale", "--n", "1,2,3"] + FAST, "kernelgpu@2"), "coherent": (["scale", "--n", "1,2,3"] + FAST, "coherent@2"), @@ -162,6 +163,14 @@ def test_run_wide_gates_catch_what_a_step_gate_does_not_name(self): self.assertEqual(g["run_crash_reports"]["status"], "PASS", k) self.assertIn("run_kernel_gpu", g) + def test_observer_check_window_reads_nothing_from_the_screen(self): + r = self.r["observer"] + self.assertIsNotNone(r["doc"]["data"].get("observer"), r["out"][-600:]) + with open(os.path.join(r["state"], "guest", "calls.log")) as f: + fps = [c[2:] for c in map(json.loads, f) if c[0] == "simfps"] + self.assertIn(["strip", "probe"], fps) # the measured window: strip and marker, one look + self.assertEqual(fps[-1], ["nostrip", "noprobe"]) # the observer check's window: an unviewed screen + def test_another_daemons_crash_report_is_listed_not_judged(self): r = self.r["daemoncrash"] d = r["doc"] From dcbf36fc99bbe7dc59e052c20a99d7508328ccd5 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Tue, 17 Nov 2026 09:41:30 +0200 Subject: [PATCH 21/23] bench: a notification as the input probe, no openurl; the tear band below the bar What the first runs on a VM showed, each measured on one 3x simulator: - simctl openurl is answered by an "Open in GlassBench?" alert on iOS 26. Nobody taps it, it stays over the app through a relaunch, and the screen rate fell from 121 to 80-83 frames/s for the rest of the run. auto no longer tries it. - an appearance switch as the probe redraws the whole screen: 0.5 s to answer, and the window lost a quarter of its distinct frames (43 against 59 a second). The probe is now a Darwin notification GlassBench -probe listens for (simctl spawn notifyutil -p); appearance is the fallback. It is answered 280 ms after simctl spawn is started with one simulator alone, so the responsive bar is 500 ms, not 150. - the tear check flagged a card gap sliding under the navigation bar's glass (3 of 3 runs). Its band starts below the bar; of 50 saved screenshots it can judge it now flags 3 (two looked at: torn). - a torn screenshot fails the step and the run and that N is not sustained, but the sweep goes on (a dry-run test; no VM run met one since). - the observer's cost is INFO, not FAIL: a lock of every picture costs frames (between nothing and 31% over seven runs; why it varies is not known); only a window that did not answer fails. - Simlock's source folder is named by its install: the default is ~/.simlock/devices/ios-sources, --source-set or BENCH_SIMLOCK_SOURCES says otherwise (check and every guest call pass it on). README: what has run on a real VM so far (tools, check, seed, cleanup and the all-busy 3x sweep for N = 1, 2, 3, with its numbers) and what has only run against the stubs. --- src/bench/GlassBench/main.swift | 12 +++++++-- src/bench/README.md | 35 +++++++++++++++++++-------- src/bench/guest/benchguest.py | 10 +++++--- src/bench/guest/benchguest_stub.py | 15 +++++++++--- src/bench/macbench/chart.py | 2 +- src/bench/macbench/cli.py | 9 ++++--- src/bench/macbench/core.py | 2 +- src/bench/macbench/scale.py | 39 ++++++++++++++++++++++-------- src/bench/macbench/targets.py | 4 ++- src/bench/macbench/tears.py | 4 ++- src/bench/macbench/transport.py | 6 +++-- src/bench/tests/helpers.py | 11 ++++++--- src/bench/tests/test_dry.py | 25 +++++++++++++++++++ src/bench/tests/test_pure.py | 9 +++++++ 14 files changed, 142 insertions(+), 41 deletions(-) diff --git a/src/bench/GlassBench/main.swift b/src/bench/GlassBench/main.swift index f8b1bf3..64cc868 100644 --- a/src/bench/GlassBench/main.swift +++ b/src/bench/GlassBench/main.swift @@ -7,7 +7,8 @@ // -probe (an input marker, see below). // // The input marker (-probe): a 30 pt square at (21, 160) pt that flips between white and black on every -// tap and on every glassbench://toggle URL (`simctl openurl`). The bench times from the injection to the +// tap, on the Darwin notification dev.macbox.glassbench.toggle (`simctl spawn notifyutil -p ...`) and +// on every glassbench://toggle URL (`simctl openurl`). The bench times from the injection to the // first surface whose marker flipped (bench/README.md, "input proxy"). // // The frame strip: every display-link tick draws a 16-bit counter as blocks at the top left, so a @@ -279,7 +280,14 @@ final class SceneDelegate: UIResponder, UIWindowSceneDelegate { if noGlass { tabs.tabBar.isHidden = true } let w = BenchWindow(windowScene: ws); w.rootViewController = tabs; w.makeKeyAndVisible() if showStrip { let f = FrameStrip(); w.addSubview(f); frameStrip = f } - if showProbe { let m = ProbeMarker(); w.addSubview(m); probeMarker = m } + if showProbe { + let m = ProbeMarker(); w.addSubview(m); probeMarker = m + // `simctl spawn notifyutil -p dev.macbox.glassbench.toggle`: an event from outside that puts + // nothing on the screen but the marker's flip (an openurl raises an alert, an appearance switch redraws all). + CFNotificationCenterAddObserver(CFNotificationCenterGetDarwinNotifyCenter(), nil, { _, _, _, _, _ in + DispatchQueue.main.async { probeMarker?.toggle() } + }, "dev.macbox.glassbench.toggle" as CFString, nil, .deliverImmediately) + } window = w } diff --git a/src/bench/README.md b/src/bench/README.md index 8ef60a0..eb68534 100644 --- a/src/bench/README.md +++ b/src/bench/README.md @@ -19,6 +19,14 @@ Requirements: Python 3.8+ on the host (standard library only; no Pillow, no nump Unit tests, no VM needed: `cd src/bench && python3 -m unittest discover -s tests -t .` (the pure parts and the run's safety) and `python3 -m unittest tests.test_dry tests.test_dry_state tests.test_dry_seed tests.test_dry_closed tests.test_dry_guards` (the dry runs with injected faults: the sweeps and their faults; what a run leaves and the idle guard, including a run killed mid-seed; the seed; the fail-closed gates; the host-disk, VM and abort guards; several minutes, run one module at a time on a busy host). +**What has run on a real VM so far** (2026-11-17: macOS 26.7.1 guest, iPhone 17 Pro, iOS 26.5, an Arrow Lake iGPU): +`tools`, `check`, `seed`, `cleanup` and `scale --mix all-busy` at 3x for N = 1, 2, 3. One, two and three animating +simulators read 57.6, 39.5 and 23.2 distinct frames/s each (p95 24, 45, 75 ms; input proxy p50 283, 312, 356 ms), +within the noise of a hand-made measurement of the same night (60, 37-41, 22-25). Everything else (`idle`, +`lifecycle`, `soak`, `ab`, the `one-busy` and `agent` mixes, the 1x device type, N above 3) has only run against the +stubs (`DRY=1` and the unit tests): expect to fix things the first time you run them. Only `scale` and +`ab --metric input_p95` inject input events; the other scenarios' windows inject nothing and leave `responsive` unknown. + ## What a run does, and what it never touches The bench has a **device set of its own** (`simctl --set`, a directory with a marker file, the same one for every run so @@ -58,7 +66,8 @@ it finds) the run stops and says so. Two explicit ways out: `--scale-x 3` (the s which Simlock usually has a source for), or **`--stock-devices`**: fresh stock devices, created one at a time with the host-disk guard in front of each, for a device type Simlock has no source for or a scenario that needs a stock device. The result records `env.guest.device_source` (`stock devices` or clones of a seed) and `compare` warns when two results -were made differently. Defaults of the Mac's paths: `~/.simlock04/devices/ios-sources` (`--source-set`), +were made differently. Defaults of the Mac's paths: `~/.simlock/devices/ios-sources` (`--source-set` or `BENCH_SIMLOCK_SOURCES`: the +`devices/ios-sources` folder of your Simlock's state directory, whatever that install calls it), `~/go/bin/simslim` (`--simslim`), the only `.json` in `~/.config/simslim/` (`--slim-profile`; several or none: the run stops and asks for the flag), `~/.local/bin/sim-idle-guard`. ### Safety, in every run that boots simulators @@ -124,7 +133,7 @@ started (explicit pids, never a bare `wait`), shuts down and deletes the bench's Common options: `--out DIR` (results, default `bench-results/`), `--secs` (window length, default 40), `--nominal-fps 60`, `--tag key=value` (recorded, for example `mesa=26.2.4` when Mesa is not a package), -`--probe-method auto|openurl|appearance|cmd|none`, `--stock-devices`, `--rebuild-seed`, `--source-set`, `--simslim`, `--slim-profile`, `--seed-settle-max` (see above), `--disk-path`, `--disk-start-gib`, `--disk-boot-gib`, `--disk-abort-gib`, `--max-compressor-mb`, `--set-name` (the bench's device set, default `bench`), `--quiesce` (disables `mediaanalysisd` in each simulator, as +`--probe-method auto|notify|appearance|openurl|cmd|none`, `--stock-devices`, `--rebuild-seed`, `--source-set`, `--simslim`, `--slim-profile`, `--seed-settle-max` (see above), `--disk-path`, `--disk-start-gib`, `--disk-boot-gib`, `--disk-abort-gib`, `--max-compressor-mb`, `--set-name` (the bench's device set, default `bench`), `--quiesce` (disables `mediaanalysisd` in each simulator, as some old harnesses did; recorded, off by default), `--no-observer-check`, `--no-golden`, `--shots N`. Every run prints one PASS/FAIL/SKIP/INFO line per section and exits 0, 1 (a section failed), 2 (the run could not go on), 20 (the cleanup could not delete everything) or, when a monitor aborted it, 7, 8, 9 or 10 (above). The JSON has the same lines under `sections`. @@ -160,16 +169,22 @@ simulator's framebuffer surface at about 2 kHz and reads the counter from every **Input latency proxy** (`input_p50_ms`, `input_p95_ms`, `input_max_ms`, `input_misses`, `input_call_p50_ms`). It is a proxy and says so. Without Simlock or agent-device, the guest injects an event with `simctl`: by default -`simctl openurl glassbench://toggle`, which flips a 30 pt marker (white/black) in GlassBench `-probe`; the fallback is -`simctl ui appearance dark|light`, which flips the same marker; `--probe-cmd` may run any command that flips it (a -real tap through your own tool). The first N calibrates which method is answered (3 tries each) and the result -records it. For each injection: *t0* = the wall clock just before the command is started in the guest; the answer +`simctl spawn notifyutil -p dev.macbox.glassbench.toggle`, a Darwin notification that flips a 30 pt marker +(white/black) in GlassBench `-probe` and puts nothing else on the screen; the fallback is `simctl ui appearance +dark|light`, which flips the same marker but redraws the whole screen (about 0.5 s to answer, and a window that uses +it lost a quarter of its distinct frames: 43 against 59 a second, one 3x simulator); `--probe-cmd` may run any command +that flips it (a real tap through your own tool). The first N checks which method is answered (3 tries) and the +result records it. `--probe-method openurl` (`simctl openurl glassbench://toggle`) is not tried by +`auto`: iOS 26 answers it with an "Open in GlassBench?" alert nobody taps, which stays over the app, through a +relaunch too, and costs a third of its frame rate (121 against 80-83 screen frames/s, one 3x simulator). For each injection: *t0* = the wall clock just before the command is started in the guest; the answer is the first new surface after t0 whose marker (luma above or below 127 at a fixed point) differs from the last surface before t0; latency = that surface's time - t0. A probe not answered in `--probe-timeout` (2 s) is a **miss and counts at the timeout** in p50/p95, never dropped. The number includes starting `simctl` (its own duration is `input_call_p50_ms`), CoreSimulator, the app's handling and one composite. It excludes the Device panel's stream, the viewer and a real finger's HID path. Read it as a trend over N and between builds, not as an absolute touch latency; -the default `responsive` bar (150 ms) wants one calibration at N=1 on your hardware before you trust it. p95 over +the default `responsive` bar (500 ms) sits about 200 ms above the proxy's floor measured here (the notification is +answered 280 ms after `simctl spawn` is started, one 3x simulator alone; the command itself takes 355 ms): check that +floor at N=1 on your hardware before you trust the bar. p95 over few probes is coarse: a 40 s window gives 16 probes per simulator. **Guest CPU and memory of a simulator** (`guest_cpu_pct`, `guest_rss_mb`). From `ps` snapshots in the guest at the @@ -227,7 +242,7 @@ simulator metrics (`fps p50 p95 p99 max over2x stall input_p50 input_p95 cpu rss that has a value for each condition meets it, with its gates ok and the system stable. The maximum is the largest N such that it and **every smaller measured N** passed: the first failure ends it (a pass above a failure is luck, not sustainability). A condition no simulator has a value for makes the target unknown (`null`), never a pass. -Defaults: `smooth60: fps>=55, p95<=25`, `smooth30: fps>=30, p95<=50`, `responsive: input_p95<=150, stall<0.3`. +Defaults: `smooth60: fps>=55, p95<=25`, `smooth30: fps>=30, p95<=50`, `responsive: input_p95<=500, stall<0.3`. **Standing cost per simulator** (`summary.standing_cost`): the least-squares slope of each system number over N, with its intercept and R². Use `--mix idle` for the pure cost of a booted simulator. @@ -240,7 +255,7 @@ written first. A gate that cannot be read is SKIP, shown as such, never PASS. | gate | PASS when | |---|---| | `strip` | no frame out of order, no counter jump, the counter advanced no more than the app's own tick log says, under 1% of samples unreadable. Lost ticks are the fps metric, not a gate failure. | -| `tears` | no screenshot of the scrolling GlassBench shows a torn band (card spacing and gap heights, the old `tears.py` rule, standard library PNG decoding); SKIP when no screenshot shows cards | +| `tears` | no screenshot of the scrolling GlassBench shows a torn band (card spacing and gap heights, the old `tears.py` rule, standard library PNG decoding); SKIP when no screenshot shows cards. A torn picture fails the step and the run, and that N does not count as sustained, but the sweep goes on to the next N (the stack tears now and then: 3 flagged of 50 judged pictures in one night here) | | `pixels` | GlassBench `-static` screenshots agree with each other and match the stored golden (`baselines/golden.json`, key: macOS, runtime, device type, size), judged once on a freshly launched static app before the first window (a window with input probes flips the marker, so later static screens are not judged); SKIP with the pictures agreeing when no golden is stored: `macbox bench golden` records it | | `coherent_census` / `coherent_process` | in the census reports written since the run began: `overwrite_risk` and `sync_errors` stayed 0, and `late_copies_in_busy`, `late_beside_unplanned`, `plan_left_planned` did not rise. Deltas, not absolute counts. Reports come once a minute, so a short step may have none: SKIP. | | `qemu_log` | no new `command buffer failed` line in QEMU's log (stamp waits are counted in the detail) | @@ -250,7 +265,7 @@ written first. A gate that cannot be read is SKIP, shown as such, never PASS. | `memory` | the VM's own cgroup swapped no more than 50 MB (SKIP when the cgroup is unreadable: other projects' swapping on the host is not this run's); guest swap-outs fail it with `--fail-on-swap` | | `run_*` | every scenario, after its body: `run_coherent_census`, `run_coherent_process`, `run_qemu_log`, `run_gpu_hang` and `run_kernel_gpu` since the run began, and `run_crash_reports` (a new crash report since setup, judged like `stability`'s). They fail the exit code of `idle`, `lifecycle`, `soak`, `ab` too. | | `leak_*` | `lifecycle`: QEMU RSS and guest memory grow less than `--leak-rss-mb` / `--leak-guest-mb` per create/boot/delete cycle (least-squares slope); `soak`: less than `--soak-rss-mb-min` / `--soak-guest-mb-min` per minute | -| `observer` | reading the strip does not move the surface frame rate by more than 4% (a second window of the first N without the strip read); otherwise the distinct-frame numbers are lower bounds | +| `observer` | what looking costs: a second window of the first N reads nothing from the screen and injects nothing. PASS within 4%; beyond it INFO (never FAIL): the distinct-frame numbers are lower bounds. One lock of each picture cost between nothing and 31% of the screen frames in seven runs here (+2, -2.5, -5.6, -12.7, -20, -23, -31%; why it varies is not known); a second lock per picture halved them, so `simfps` reads strip and marker under one | ## A/B discipline diff --git a/src/bench/guest/benchguest.py b/src/bench/guest/benchguest.py index c4ae9d6..4349529 100644 --- a/src/bench/guest/benchguest.py +++ b/src/bench/guest/benchguest.py @@ -38,7 +38,8 @@ IDLE_GUARD = os.path.expanduser("~/.local/bin/sim-idle-guard") # `keep ` (max 240) # Simlock's own source devices (read only: nothing is ever written to, booted, shut down or deleted in this set; the one # command that names it is simlock_clone()) and the Slim tooling the fleet uses on every clone. -SOURCE_SET = os.environ.get("BENCH_SIMLOCK_SOURCES") or os.path.expanduser("~/.simlock04/devices/ios-sources") +# Simlock's state directory is named by its install: give yours with --source-set or BENCH_SIMLOCK_SOURCES. +SOURCE_SET = os.path.expanduser(os.environ.get("BENCH_SIMLOCK_SOURCES") or "~/.simlock/devices/ios-sources") SIMSLIM = os.environ.get("BENCH_SIMSLIM") or os.path.expanduser("~/go/bin/simslim") SLIM_PROFILE = os.environ.get("BENCH_SLIM_PROFILE") # else the one profile in SLIM_DIR SLIM_DIR = os.path.expanduser("~/.config/simslim") @@ -365,6 +366,8 @@ def list_ips(): # --- injection ------------------------------------------------------------------------------------- def inject(method, udid, state, cmd_tpl=None): """One input event; returns the command's rc. state: a per-simulator dict for alternating methods.""" + if method == "notify": + return simctl("spawn", udid, "notifyutil", "-p", "dev.macbox.glassbench.toggle", timeout=20)[0] if method == "openurl": return simctl("openurl", udid, "glassbench://toggle", timeout=20)[0] if method == "appearance": @@ -569,7 +572,8 @@ def find_source(type_id, src_set): try: names = sorted(n for n in os.listdir(src_set) if UUID.match(n)) except OSError: - return None, "Simlock's source device set was not found at the configured path (Simlock is optional)" + return None, ("Simlock's source device set was not found at the configured path (Simlock is optional): give the " + "devices/ios-sources folder of its state directory with --source-set or BENCH_SIMLOCK_SOURCES") found, types = [], set() for n in names: try: @@ -1331,7 +1335,7 @@ def add(name, fn, *opts): add("prepare", cmd_prepare, O("--udids", required=True), O("--roles", required=True), O("--install", action="store_true"), O("--check-app", action="store_true"), O("--quiesce", action="store_true")) add("settle", cmd_settle, O("--max-s", type=float, default=120), O("--bg-pct", type=float, default=40), O("--step", type=float, default=3), O("--consecutive", type=int, default=2)) - add("calibrate", cmd_calibrate, O("--udid", required=True), O("--methods", default="openurl,appearance"), O("--probe-cmd"), O("--time-scale", type=float, default=1.0)) + add("calibrate", cmd_calibrate, O("--udid", required=True), O("--methods", default="notify,appearance"), O("--probe-cmd"), O("--time-scale", type=float, default=1.0)) add("measure", cmd_measure, O("--udids", required=True), O("--roles", required=True), O("--mix", default="all-busy"), O("--secs", type=float, default=40), O("--start-at", type=float, default=0), O("--probe-method", default="openurl"), O("--probe-every", type=float, default=2.5), O("--probe-cmd"), O("--agent-period", type=float, default=12), diff --git a/src/bench/guest/benchguest_stub.py b/src/bench/guest/benchguest_stub.py index 4cb49bd..eb50d5d 100644 --- a/src/bench/guest/benchguest_stub.py +++ b/src/bench/guest/benchguest_stub.py @@ -2,7 +2,7 @@ directory ($BENCHGUEST_STATE) between calls, so `benchguest.py` runs its real control flow with no macOS. Faults ($BENCH_STUB_FAULT, comma separated): memfloor@N (guest free memory below any floor from N booted simulators), crash@N (a new crash report at N booted), backwards@N (a frame shown out of order at N), slow@N -(frame rate collapses from N), hang_simfps (simfps never answers), nomarker (the input marker never flips), +(frame rate collapses from N), hang_simfps (simfps never answers), nomarker (the input marker never flips), nonotify (it does not flip on the notification), tear@N (the screenshots at N booted are torn), cantdelete (simctl delete fails: a cleanup that cannot finish), noguard (no sim-idle-guard on the Mac), keepfail (sim-idle-guard keep fails), nosource (Simlock has no source device of the type), nosourceset (no Simlock set at all), slimfail (simslim fails), slowslim (simslim takes 2 minutes: a run killed while the seed is half made), slimfew (simslim leaves too few services off), noicons (no chronod icons folder: recorded as icons=false, never a failure), @@ -205,8 +205,8 @@ def _stub_run(argv): s["mode"][cmd[1]] = "static" if "-static" in cmd else "busy" elif verb == "terminate": s["mode"].pop(cmd[1], None) - elif verb in ("openurl", "ui"): - if fault("nomarker") is None: + elif verb in ("openurl", "ui") or (verb == "spawn" and "notifyutil" in cmd): + if fault("nomarker") is None and not (verb == "spawn" and fault("nonotify") is not None): s["inj"].append([time.time(), cmd[1]]) elif verb == "io": p = cmd[-1] @@ -380,6 +380,15 @@ def install(mod): mod.ps_text = stub_ps mod.vmstat_text = stub_vmstat mod.sysctl = lambda name: {"kern.memorystatus_vm_pressure_level": "1", "hw.ncpu": "8", "hw.memsize": str(16 << 30)}.get(name, "") + tr = fault("tear") + if tr is not None: # every screenshot taken while exactly `tr` simulators are booted is torn + from macbench import tears as _tears + + def torn(img): + return {"tear": len(booted(load())) == tr, "applicable": True, "odd": [1] if len(booted(load())) == tr else [], + "steps": [], "gaps": [58, 58]} + _tears.detect_tears = torn + def spawn(udid, secs, label, strip, start_at, raw, probe=True): _log(["simfps", udid, "strip" if strip else "nostrip", "probe" if probe else "noprobe"]) return StubSimfps(udid, secs, strip, start_at, raw) diff --git a/src/bench/macbench/chart.py b/src/bench/macbench/chart.py index e6c3475..b8e1b93 100644 --- a/src/bench/macbench/chart.py +++ b/src/bench/macbench/chart.py @@ -99,7 +99,7 @@ def scale_chart(doc): "ms", hlines=[(25, "25 ms")]), panel("input proxy p95 (ms)", xs, [("each sim", [(st["n"], s["input_p95_ms"]) for st in steps for s in st["sims"] if s.get("input_p95_ms") is not None], "dots"), ("worst", [(st["n"], max([s["input_p95_ms"] for s in st["sims"] if s.get("input_p95_ms") is not None] or [None])) for st in steps], "line")], - "ms", hlines=[(150, "150 ms")]), + "ms", hlines=[(500, "500 ms")]), panel("QEMU CPU (cores) and GPU busy (%)", xs, [("QEMU cores", _sys(steps, "qemu_cpu_cores"), "line"), ("GPU busy %", [(x, None if y is None else y / 10.0) for x, y in _sys(steps, "gpu_busy_pct")], "line")], "cores | % / 10"), diff --git a/src/bench/macbench/cli.py b/src/bench/macbench/cli.py index 8bad945..972d3fe 100644 --- a/src/bench/macbench/cli.py +++ b/src/bench/macbench/cli.py @@ -2,6 +2,7 @@ import argparse import json import os +import shlex import shutil import subprocess import sys @@ -36,7 +37,7 @@ def add_common(p): p.add_argument("--scale-x", dest="scale_x", type=int, choices=(1, 2, 3), default=1, help="device pixel scale: 1 = the macbox 1x type, 3 = the stock iPhone (default 1)") p.add_argument("--secs", type=float, default=40, help="length of a measurement window, seconds") p.add_argument("--nominal-fps", type=float, default=60.0) - p.add_argument("--probe-method", default="auto", choices=("auto", "openurl", "appearance", "cmd", "none"), help="input injection (auto: calibrated at the first N)") + p.add_argument("--probe-method", default="auto", choices=("auto", "notify", "appearance", "openurl", "cmd", "none"), help="input injection (auto: notify, else appearance, checked at the first N; openurl raises an alert on iOS 26 that stays over the app)") p.add_argument("--probe-cmd", default=None, help="for --probe-method cmd: a shell command with {udid}; it must flip GlassBench's marker (a tap)") p.add_argument("--probe-every", type=float, default=2.5, help="seconds between input probes of one simulator") p.add_argument("--probe-timeout", type=float, default=2.0, help="an input probe not answered in this time is a miss (counted at this value)") @@ -47,7 +48,7 @@ def add_common(p): p.add_argument("--settle-bg", type=float, default=40, help="the guest counts as settled below this background CPU %% (of one core)") p.add_argument("--stock-devices", action="store_true", help="create FRESH STOCK devices instead of cloning the production seed: about 1.8 GB of guest disk each, not what an agent gets; for a device type Simlock has no source for or a scenario that needs a stock device (recorded in the result)") p.add_argument("--rebuild-seed", action="store_true", help="delete the bench's seed and make a new one (after `macbox bench tools` rebuilt the app, or when it cannot be reused)") - p.add_argument("--source-set", default=os.environ.get("BENCH_SIMLOCK_SOURCES"), help="Simlock's source device set in the guest, read only (default ~/.simlock04/devices/ios-sources)") + p.add_argument("--source-set", default=os.environ.get("BENCH_SIMLOCK_SOURCES"), help="Simlock's source device set in the guest, read only (default ~/.simlock/devices/ios-sources)") p.add_argument("--simslim", default=os.environ.get("BENCH_SIMSLIM"), help="simslim in the guest (default ~/go/bin/simslim)") p.add_argument("--slim-profile", default=os.environ.get("BENCH_SLIM_PROFILE"), help="simslim profile in the guest (default: the only .json in ~/.config/simslim)") p.add_argument("--seed-settle-max", type=float, default=600, help="the seed must settle within this many seconds or it is deleted and the run fails") @@ -383,7 +384,9 @@ def line(good, text): line(hp.mem_avail_gb() is not None, "host available memory readable") ssh = os.environ.get("BENCH_SSH") or os.environ.get("MACBOX_SSH_HOST") or "macbox" try: - out = subprocess.run(["ssh", "-o", "ConnectTimeout=8", "-o", "BatchMode=yes", ssh, "cd ~/.macbox/bench && xcrun python3 guest/benchguest.py info"], + src = os.environ.get("BENCH_SIMLOCK_SOURCES") + out = subprocess.run(["ssh", "-o", "ConnectTimeout=8", "-o", "BatchMode=yes", ssh, "cd ~/.macbox/bench && %sxcrun python3 guest/benchguest.py info" % ( + "BENCH_SIMLOCK_SOURCES=%s " % shlex.quote(src) if src else "")], capture_output=True, text=True, timeout=60).stdout d = json.loads(out.strip().splitlines()[-1]) line(True, "guest answers: macOS %s, %s" % (d.get("macos"), d.get("xcode"))) diff --git a/src/bench/macbench/core.py b/src/bench/macbench/core.py index 946e813..9b10c30 100644 --- a/src/bench/macbench/core.py +++ b/src/bench/macbench/core.py @@ -344,7 +344,7 @@ def window(ctx, secs, mix, no_strip=False): t_go = time.time() + ctx.offset + lead ctx.run.need(secs + lead + 60, "a %d s window" % secs) args = ["measure", "--udids", udids, "--roles", roles, "--mix", mix, "--secs", secs, "--start-at", "%.3f" % t_go, - "--probe-method", ctx.probe_method or "openurl", "--probe-every", a.probe_every, "--agent-period", a.agent_period, "--probe-timeout", a.probe_timeout] + "--probe-method", ctx.probe_method or "none", "--probe-every", a.probe_every, "--agent-period", a.agent_period, "--probe-timeout", a.probe_timeout] if a.probe_cmd: args += ["--probe-cmd", a.probe_cmd] if no_strip: # the unviewed screen: no strip, no marker, no injection, so simfps locks no surface diff --git a/src/bench/macbench/scale.py b/src/bench/macbench/scale.py index 578ce83..17f84a1 100644 --- a/src/bench/macbench/scale.py +++ b/src/bench/macbench/scale.py @@ -10,17 +10,30 @@ STEP_GATES = ("counter", "strip", "kernel_gpu", "coherent_census", "coherent_process", "qemu_log", "gpu_hang", "tears", "pixels", "memory") +# A torn screenshot fails the step and the run (exit code, the gate's line), but the sweep goes on: the pictures of the +# next N are not made worse by it, and a sweep that ends at the first torn picture measures nothing above it. +SWEEP_GOES_ON = ("tears",) + + def calibrate(ctx, sim): """Pick the input injection that is answered (or 'none'): 3 tries of each method on one simulator.""" a = ctx.a if a.probe_method != "auto": ctx.probe_method = a.probe_method return {"chosen": a.probe_method, "calibrated": False} - methods = "openurl,appearance" - r = ctx.guest.call(["calibrate", "--udid", sim["udid"], "--methods", methods, "--time-scale", 0.3 if ctx.host.dry else 1.0] + (["--probe-cmd", a.probe_cmd] if a.probe_cmd else []), 200) - chosen = next((m for m in methods.split(",") if r["methods"].get(m, {}).get("answered", 0) >= 2), "none") + # notify first: a Darwin notification puts nothing on the screen but the marker's flip. appearance redraws the whole + # screen (0.5 s to answer, a quarter of the distinct frames lost in a window that uses it): the fallback only. + # Never openurl: iOS 26 answers `simctl openurl` with an "Open in GlassBench?" alert that nobody taps. It stays over + # the app, through a relaunch too, and its glass cost a third of the frame rate (121 -> 80-83 screen frames/s). + seen, chosen = {}, "none" + for m in ("notify", "appearance"): + r = ctx.guest.call(["calibrate", "--udid", sim["udid"], "--methods", m, "--time-scale", 0.3 if ctx.host.dry else 1.0] + (["--probe-cmd", a.probe_cmd] if a.probe_cmd else []), 200) + seen.update(r["methods"]) + if r["methods"].get(m, {}).get("answered", 0) >= 2: + chosen = m + break ctx.probe_method = chosen - return {"chosen": chosen, "calibrated": True, "methods": r["methods"]} + return {"chosen": chosen, "calibrated": True, "methods": seen} def golden_phase(ctx, record=False): @@ -84,8 +97,8 @@ def run_scale(ctx): _est(ctx, data["steps"]) _finish(ctx, None, partial=True) first = False - if not step["gates_ok"]: - bad = [k for k, v in step["gates"].items() if v["status"] == "FAIL"] + bad = [k for k, v in step["gates"].items() if v["status"] == "FAIL"] + if [k for k in bad if k not in SWEEP_GOES_ON]: stopped, kind = "gate failed at N=%d: %s" % (n, ", ".join(bad)), "gate" break if not step["stable"]: @@ -131,7 +144,8 @@ def _est(ctx, steps): def observer_check(ctx, step, mix): """Decision: if reading the strip moves the surface frame rate by more than the noise (4%), every distinct-frame - number in this result carries that observer cost. A second window of the same N reads no strip.""" + number in this result carries that observer cost. A second window of the same N reads nothing from the screen + and injects nothing.""" g, s0, s1, t_go = core.window(ctx, ctx.a.secs, mix, no_strip=True) with_strip = [s["surface_fps"] for s in step["sims"] if s.get("role") in ("busy", "agent") and s.get("surface_fps")] without = [s["simfps"]["fps"] for s in g["sims"] if s["role"] in ("busy", "agent") and s.get("simfps")] @@ -201,9 +215,14 @@ def _sections(ctx, data, stopped): ctx.section("input", "INFO", "INPUT PROXY method %s %s" % (cal["chosen"], "(calibrated: %s)" % {k: "%d/%d" % (v["answered"], v["n"]) for k, v in cal.get("methods", {}).items()} if cal.get("calibrated") else "(given)")) ob = data.get("observer") if ob: - ctx.section("observer", "PASS" if ob["ok"] else "FAIL", "OBSERVER %s: surface fps %s with the strip read, %s without (%s): %s" % ( - "PASS" if ob["ok"] else "FAIL", ob["surface_fps_with_strip"], ob["surface_fps_without_strip"], - "n/a" if ob["delta"] is None else "%+.1f%%" % (ob["delta"] * 100), "within the noise" if ob["ok"] else "reading the strip changes the system: distinct-fps numbers are lower bounds")) + # a cost of looking is a property of the measurement, not a failure of the system: INFO, never FAIL. A window + # whose counter did not answer (delta None) is the failure. + status = "FAIL" if ob["delta"] is None else "PASS" if ob["ok"] else "INFO" + ctx.section("observer", status, "OBSERVER %s: surface fps %s measured (strip and marker read, events injected), %s on an unviewed screen (%s): %s" % ( + status, ob["surface_fps_with_strip"], ob["surface_fps_without_strip"], + "n/a" if ob["delta"] is None else "%+.1f%%" % (ob["delta"] * 100), + "the unviewed window's counter did not answer" if ob["delta"] is None else "within the noise" if ob["ok"] + else "looking at the screen costs frames (a lock of every picture, the injected events): the distinct-fps numbers are lower bounds")) sc = sm.get("standing_cost") or {} if sc: parts = ["%s %+.2f/sim" % (k, v["per_sim"]) for k, v in sc.items() if k in ("qemu_cpu_cores", "gpu_busy_pct", "guest_mem_used_mb", "guest_sims_cpu_pct", "guest_sims_rss_mb")] diff --git a/src/bench/macbench/targets.py b/src/bench/macbench/targets.py index 5fc1616..cf321fe 100644 --- a/src/bench/macbench/targets.py +++ b/src/bench/macbench/targets.py @@ -16,7 +16,9 @@ DEFAULT_TARGETS = [ "smooth60: fps>=55, p95<=25", "smooth30: fps>=30, p95<=50", - "responsive: input_p95<=150, stall<0.3", + # the proxy's floor is the injecting command itself: `simctl spawn notifyutil` answered in 280 ms (p50, one 3x + # simulator alone, the command took 355 ms). 500 ms leaves about 200 ms for the simulator to be late. + "responsive: input_p95<=500, stall<0.3", ] _COND = re.compile(r"^\s*([a-z0-9_]+)\s*(>=|<=|>|<)\s*([0-9.]+)\s*$") diff --git a/src/bench/macbench/tears.py b/src/bench/macbench/tears.py index fc98e47..89a1a6c 100644 --- a/src/bench/macbench/tears.py +++ b/src/bench/macbench/tears.py @@ -24,7 +24,9 @@ def detect_tears(img): w, h = img.w, img.h s = w / 1206.0 x0, x1 = int(1125 * s), max(int(1125 * s) + 1, int(1150 * s)) - y0, y1 = int(400 * s), min(int(1700 * s), h - 1) + # The band starts below the navigation bar's glass, which ends at 420 px (3x): a gap that scrolls under the glass + # is cut there and looked like a short gap at a wrong distance (a "tear" in every third screenshot). + y0, y1 = int(440 * s), min(int(1700 * s), h - 1) rows = _row_means(img, x0, x1, y0, y1) gap = [min(c) > 225 for c in rows] # the white background between cards margin = int(75 * s) diff --git a/src/bench/macbench/transport.py b/src/bench/macbench/transport.py index e5b682b..eb70bab 100644 --- a/src/bench/macbench/transport.py +++ b/src/bench/macbench/transport.py @@ -38,8 +38,10 @@ def __init__(self, run, ssh_host, set_path, stubs_dir=None): self.fault_ssh_timeout = None # DRY: fail the k-th call (tests) def argv(self, args): - remote = "cd %s && BENCH_SET=\"$HOME/.macbox/bench/sets/%s\" xcrun python3 guest/benchguest.py %s" % ( - GUEST_DIR, self.set_path, " ".join(shlex.quote(str(a)) for a in args)) + src = os.environ.get("BENCH_SIMLOCK_SOURCES") # the guest expands a leading ~ itself + remote = "cd %s && BENCH_SET=\"$HOME/.macbox/bench/sets/%s\" %sxcrun python3 guest/benchguest.py %s" % ( + GUEST_DIR, self.set_path, "BENCH_SIMLOCK_SOURCES=%s " % shlex.quote(src) if src else "", + " ".join(shlex.quote(str(a)) for a in args)) return ["ssh", "-o", "ConnectTimeout=10", "-o", "BatchMode=yes", "-o", "ServerAliveInterval=15", self.ssh_host, remote] def parse(self, what, rc, out, err): diff --git a/src/bench/tests/helpers.py b/src/bench/tests/helpers.py index b178504..8d86c3c 100644 --- a/src/bench/tests/helpers.py +++ b/src/bench/tests/helpers.py @@ -2,18 +2,19 @@ from macbench import png -def card_image(width=603, tear_at=None, with_cards=True): +def card_image(width=603, tear_at=None, with_cards=True, scrolled=0, bar=False): """A screenshot like GlassBench's at scale s = width/1206: white background, cards 336*s apart whose right edge strip (x 1125..1150 * s) is a vertical gradient (1 level per row at 3x), gaps 58*s tall. tear_at=y shifts every - row below y by 40*s rows (a band from another frame).""" + row below y by 40*s rows (a band from another frame). scrolled=N moves the content up by N*s rows; bar=True + draws the navigation bar's glass over everything above 420*s (grey, as the real one tints what is under it).""" s = width / 1206.0 h = int(2622 * s) pitch, gap = 336 * s, 58 * s rows = [] for y in range(h): - yy = y + yy = y + int(scrolled * s) if tear_at is not None and y >= tear_at: - yy = y + int(40 * s) + yy += int(40 * s) k = (yy - int(300 * s)) % pitch if yy >= 300 * s else None row = bytearray([255] * (width * 3)) if with_cards and k is not None and gap / 2 <= k < pitch - gap / 2 and 300 * s < yy < 2300 * s: @@ -23,6 +24,8 @@ def card_image(width=603, tear_at=None, with_cards=True): row[x * 3:x * 3 + 3] = bytes([level, level // 2, 200 - level // 2]) else: row[x * 3:x * 3 + 3] = bytes([60, 60, 200]) + if bar and y < 420 * s: + row = bytearray([200] * (width * 3)) rows.append(bytes(row)) return png.Image(width, h, 3, rows) diff --git a/src/bench/tests/test_dry.py b/src/bench/tests/test_dry.py index d614f5e..e8495cb 100644 --- a/src/bench/tests/test_dry.py +++ b/src/bench/tests/test_dry.py @@ -74,6 +74,8 @@ def sections(doc, name): "slow": (["scale", "--n", "1,2,3,4"] + FAST, "slow@3"), "hang": (["scale", "--n", "1,2,3"] + FAST, "hang_simfps"), "nomarker": (["scale", "--n", "1,2"] + FAST, "nomarker"), + "nonotify": (["scale", "--n", "1"] + FAST, "nonotify"), + "tear": (["scale", "--n", "1,2,3"] + FAST, "tear@2"), "onebusy": (["scale", "--n", "1,2,3", "--mix", "one-busy"] + FAST, ""), "agent": (["scale", "--n", "1,2", "--mix", "agent", "--agent-period", "2"] + FAST, ""), "idlemix": (["scale", "--n", "1,2", "--mix", "idle"] + FAST, ""), @@ -217,6 +219,29 @@ def test_hung_observer_is_reported_not_swallowed(self): self.assertTrue(d["data"]["steps"][0]["stability"]["observer_errors"]) self.assertEqual(sims_booted(self.r["hang"]["state"]), []) + def test_a_torn_screenshot_fails_the_run_but_the_sweep_goes_on(self): + r = self.r["tear"] + d = r["doc"] + self.assertEqual([st["n"] for st in d["data"]["steps"]], [1, 2, 3]) # N=3 is still measured + self.assertEqual([st["gates"]["tears"]["status"] for st in d["data"]["steps"]], ["PASS", "FAIL", "PASS"]) + self.assertNotEqual(r["rc"], 0) + self.assertIsNone(d["data"]["summary"]["stopped"]) + for name, m in d["data"]["summary"]["max_n"].items(): # the torn step is not sustained + self.assertNotIn(m["max_n"], (2, 3), name) + self.assertEqual(sims_booted(r["state"]), []) + + def test_auto_never_sends_openurl(self): + # iOS 26 answers `simctl openurl` with an alert that stays over the app and slows it for the whole run + r = self.r["clean"] + self.assertEqual(list(r["doc"]["data"]["calibration"]["methods"]), ["notify"]) # answered: appearance is not tried + self.assertEqual(r["doc"]["data"]["calibration"]["chosen"], "notify") + self.assertFalse([c for c in simctl_calls(r["state"]) if "openurl" in c]) + + def test_appearance_is_the_fallback_when_the_notification_is_not_answered(self): + cal = self.r["nonotify"]["doc"]["data"]["calibration"] + self.assertEqual(cal["chosen"], "appearance") + self.assertEqual(cal["methods"]["notify"]["answered"], 0) + def test_unanswered_input_probe_is_not_dropped(self): d = self.r["nomarker"]["doc"] cal = d["data"]["calibration"] diff --git a/src/bench/tests/test_pure.py b/src/bench/tests/test_pure.py index 0482324..0df029d 100644 --- a/src/bench/tests/test_pure.py +++ b/src/bench/tests/test_pure.py @@ -142,6 +142,15 @@ def test_torn_screenshot_is_found(self): r = tears.detect_tears(helpers.card_image(603, tear_at=700)) self.assertTrue(r["tear"], r) + def test_a_gap_sliding_under_the_navigation_bar_is_not_a_tear(self): + # the bar's glass ends at 420 px (3x): a gap half under it showed as a short gap at a wrong distance + for scrolled in (190, 200, 216, 230, 245): + r = tears.detect_tears(helpers.card_image(603, scrolled=scrolled, bar=True)) + self.assertTrue(r["applicable"], scrolled) + self.assertFalse(r["tear"], (scrolled, r)) + r = tears.detect_tears(helpers.card_image(603, scrolled=216, bar=True, tear_at=700)) + self.assertTrue(r["tear"]) + def test_no_cards_is_not_applicable(self): r = tears.detect_tears(helpers.card_image(603, with_cards=False)) self.assertFalse(r["applicable"]) From d36210bb174a44a6676e1ddd881c40aba8eb7aad Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Tue, 17 Nov 2026 10:22:36 +0200 Subject: [PATCH 22/23] bench: a second look at a still surface; leaks judged on RSS + swap, and only steady ones The one-busy mix and the lifecycle ran on a VM for the first time: - on a still screen a picture read the moment its surface changes showed the frame before it in 14-15 of 15 probes; 30 ms later the same surface showed the new one. The probes counted as missed (3, 3 and 1 of 15) or were answered by the flip before. simfps now looks at an unchanged surface again after 30, 150 and 600 ms (a line with counter -3, not a frame), and the result counts probes answered that way (input_second_looks). - the lifecycle's leak gate failed on +215 MB of QEMU RSS per cycle from 3 samples: the host bringing swapped pages back, one step then flat. It judges RSS + swap now, FAILs only a steady growth (4 samples, R2 >= 0.8) and says SKIP for a slope above the bar without that; 5 cycles by default. README: both scenarios' numbers in what has run on a VM. --- src/bench/README.md | 21 +++++++++++++-------- src/bench/macbench/analysis.py | 14 +++++++++----- src/bench/macbench/cli.py | 6 +++--- src/bench/macbench/gates.py | 11 +++++++++-- src/bench/macbench/hostprobe.py | 5 +++++ src/bench/macbench/others.py | 14 +++++++------- src/bench/simfps.swift | 14 +++++++++++++- src/bench/tests/test_dry.py | 2 +- src/bench/tests/test_modules.py | 3 +++ src/bench/tests/test_pure.py | 10 ++++++++++ 10 files changed, 73 insertions(+), 27 deletions(-) diff --git a/src/bench/README.md b/src/bench/README.md index eb68534..9aaf7e9 100644 --- a/src/bench/README.md +++ b/src/bench/README.md @@ -20,12 +20,15 @@ Unit tests, no VM needed: `cd src/bench && python3 -m unittest discover -s tests and `python3 -m unittest tests.test_dry tests.test_dry_state tests.test_dry_seed tests.test_dry_closed tests.test_dry_guards` (the dry runs with injected faults: the sweeps and their faults; what a run leaves and the idle guard, including a run killed mid-seed; the seed; the fail-closed gates; the host-disk, VM and abort guards; several minutes, run one module at a time on a busy host). **What has run on a real VM so far** (2026-11-17: macOS 26.7.1 guest, iPhone 17 Pro, iOS 26.5, an Arrow Lake iGPU): -`tools`, `check`, `seed`, `cleanup` and `scale --mix all-busy` at 3x for N = 1, 2, 3. One, two and three animating -simulators read 57.6, 39.5 and 23.2 distinct frames/s each (p95 24, 45, 75 ms; input proxy p50 283, 312, 356 ms), -within the noise of a hand-made measurement of the same night (60, 37-41, 22-25). Everything else (`idle`, -`lifecycle`, `soak`, `ab`, the `one-busy` and `agent` mixes, the 1x device type, N above 3) has only run against the -stubs (`DRY=1` and the unit tests): expect to fix things the first time you run them. Only `scale` and -`ab --metric input_p95` inject input events; the other scenarios' windows inject nothing and leave `responsive` unknown. +`tools`, `check`, `seed`, `cleanup`, `scale` at 3x for N = 1, 2, 3 in the `all-busy` and `one-busy` mixes, and +`lifecycle` (5 cycles, no boot storm). All animating: 57.6, 39.5 and 23.2 distinct frames/s each (p95 24, 45, 75 ms; +input proxy p50 283, 312, 356 ms), within the noise of a hand-made measurement of the same night (60, 37-41, 22-25). +One animating beside still ones: it keeps 60 (59.9, 59.8, 60.0), and the still ones answer the input proxy as fast +as it does (p95 303-311 ms, no miss). Lifecycle medians: clone 2.9 s, first boot 5.6 s, first app frame 1.0 s after a +0.5 s launch call, shutdown 3.2 s, resume boot 5.5 s, delete 0.2 s. Everything else (`idle`, `soak`, `ab`, the `agent` +mix, the lifecycle's boot storm, the 1x device type, N above 3) has only run against the stubs (`DRY=1` and the unit +tests): expect to fix things the first time you run them. Only `scale` and `ab --metric input_p95` inject input +events; the other scenarios' windows inject nothing and leave `responsive` unknown. ## What a run does, and what it never touches @@ -178,7 +181,9 @@ result records it. `--probe-method openurl` (`simctl openurl glassbench://toggle `auto`: iOS 26 answers it with an "Open in GlassBench?" alert nobody taps, which stays over the app, through a relaunch too, and costs a third of its frame rate (121 against 80-83 screen frames/s, one 3x simulator). For each injection: *t0* = the wall clock just before the command is started in the guest; the answer is the first new surface after t0 whose marker (luma above or below 127 at a fixed point) differs from the last -surface before t0; latency = that surface's time - t0. A probe not answered in `--probe-timeout` (2 s) is a **miss +surface before t0; latency = that surface's time - t0. On a still screen `simfps` looks at each surface again 30, 150 +and 600 ms after it appeared (a picture read the moment it appears can still show the frame before it, and nothing +follows to correct that); probes answered only by such a second look are counted in `input_second_looks`. A probe not answered in `--probe-timeout` (2 s) is a **miss and counts at the timeout** in p50/p95, never dropped. The number includes starting `simctl` (its own duration is `input_call_p50_ms`), CoreSimulator, the app's handling and one composite. It excludes the Device panel's stream, the viewer and a real finger's HID path. Read it as a trend over N and between builds, not as an absolute touch latency; @@ -264,7 +269,7 @@ written first. A gate that cannot be read is SKIP, shown as such, never PASS. | `counter` | `simfps` answered with a result for every simulator of the window and every animating simulator's surfaces gave a readable frame counter (a counter that answers nothing or an error fails the window, it is not a missing metric) | | `memory` | the VM's own cgroup swapped no more than 50 MB (SKIP when the cgroup is unreadable: other projects' swapping on the host is not this run's); guest swap-outs fail it with `--fail-on-swap` | | `run_*` | every scenario, after its body: `run_coherent_census`, `run_coherent_process`, `run_qemu_log`, `run_gpu_hang` and `run_kernel_gpu` since the run began, and `run_crash_reports` (a new crash report since setup, judged like `stability`'s). They fail the exit code of `idle`, `lifecycle`, `soak`, `ab` too. | -| `leak_*` | `lifecycle`: QEMU RSS and guest memory grow less than `--leak-rss-mb` / `--leak-guest-mb` per create/boot/delete cycle (least-squares slope); `soak`: less than `--soak-rss-mb-min` / `--soak-guest-mb-min` per minute | +| `leak_*` | `lifecycle`: QEMU's memory (RSS + swap: RSS alone rises when the host brings swapped pages back) and guest memory grow less than `--leak-rss-mb` / `--leak-guest-mb` per create/boot/delete cycle (least-squares slope); `soak`: less than `--soak-rss-mb-min` / `--soak-guest-mb-min` per minute. FAIL needs a steady growth (at least 4 samples, R2 >= 0.8); a slope above the bar without that is SKIP | | `observer` | what looking costs: a second window of the first N reads nothing from the screen and injects nothing. PASS within 4%; beyond it INFO (never FAIL): the distinct-frame numbers are lower bounds. One lock of each picture cost between nothing and 31% of the screen frames in seven runs here (+2, -2.5, -5.6, -12.7, -20, -23, -31%; why it varies is not known); a second lock per picture halved them, so `simfps` reads strip and marker under one | ## A/B discipline diff --git a/src/bench/macbench/analysis.py b/src/bench/macbench/analysis.py index 9c6af3b..9b192d1 100644 --- a/src/bench/macbench/analysis.py +++ b/src/bench/macbench/analysis.py @@ -4,6 +4,9 @@ LUMA_MID = 127 +SECOND_LOOK = -3 # simfps: the same surface looked at again (its marker changed since the first look), not a new picture + + def parse_raw(text): """simfps's SIMFPS_STAMPS file: ' ' per line -> [(t, counter, luma)].""" out = [] @@ -27,7 +30,7 @@ def probe_latencies(raw, injections, timeout_s=2.0): What it includes: starting `simctl` (its own duration is `call_ms`), CoreSimulator, the app's handling and one frame's composite. What it leaves out: the Device panel stream, the viewer, a real finger's HID path. Returns {"n", "misses", "p50_ms", "p95_ms", "max_ms", "call_p50_ms", "latencies_ms": [...]}""" - lat, calls, misses = [], [], 0 + lat, calls, misses, second = [], [], 0, 0 for t0, tret in injections: before = None for t, _, luma in raw: @@ -37,13 +40,14 @@ def probe_latencies(raw, injections, timeout_s=2.0): before = luma > LUMA_MID hit = None if before is not None: - for t, _, luma in raw: + for t, c, luma in raw: if t < t0 or luma < 0: continue if t > t0 + timeout_s: break if (luma > LUMA_MID) != before: hit = t - t0 + second += c == SECOND_LOOK break calls.append((tret - t0) * 1000.0) if hit is None: @@ -51,7 +55,7 @@ def probe_latencies(raw, injections, timeout_s=2.0): lat.append(timeout_s * 1000.0) else: lat.append(hit * 1000.0) - return {"n": len(injections), "misses": misses, "p50_ms": stats.r1(stats.percentile(lat, 0.5)), + return {"n": len(injections), "misses": misses, "second_looks": second, "p50_ms": stats.r1(stats.percentile(lat, 0.5)), "p95_ms": stats.r1(stats.percentile(lat, 0.95)), "max_ms": stats.r1(max(lat)) if lat else None, "call_p50_ms": stats.r1(stats.percentile(calls, 0.5)), "latencies_ms": [round(x, 1) for x in lat]} @@ -63,7 +67,7 @@ def sim_metrics(idx, role, raw, injections, nominal_fps=60.0, timeout_s=2.0, app raw = [r for r in raw if window[0] <= r[0] <= window[1]] or raw[:1] sim = {"idx": idx, "role": role} if role == "busy": - a = strip.analyze_counters([(t, c) for t, c, _ in raw]) + a = strip.analyze_counters([(t, c) for t, c, _ in raw if c != SECOND_LOOK]) fm = stats.frame_stats([t for t, _ in a["unique"]], nominal_fps) sim.update(fm) sim["strip"] = {k: a[k] for k in ("repeats", "lost", "backwards", "jumps", "unreadable", "span_ticks")} @@ -72,7 +76,7 @@ def sim_metrics(idx, role, raw, injections, nominal_fps=60.0, timeout_s=2.0, app sim["app_ticks"] = app_ticks if injections: pl = probe_latencies(raw, injections, timeout_s) - sim.update({"input_n": pl["n"], "input_misses": pl["misses"], "input_p50_ms": pl["p50_ms"], + sim.update({"input_n": pl["n"], "input_misses": pl["misses"], "input_second_looks": pl["second_looks"], "input_p50_ms": pl["p50_ms"], "input_p95_ms": pl["p95_ms"], "input_max_ms": pl["max_ms"], "input_call_p50_ms": pl["call_p50_ms"]}) return sim diff --git a/src/bench/macbench/cli.py b/src/bench/macbench/cli.py index 972d3fe..3835029 100644 --- a/src/bench/macbench/cli.py +++ b/src/bench/macbench/cli.py @@ -80,16 +80,16 @@ def parser(): if name == "idle": p.add_argument("--bar-qemu-cores", type=float, default=0.15, help="PASS bar for QEMU's CPU in cores") if name == "lifecycle": - p.add_argument("--cycles", type=int, default=3) + p.add_argument("--cycles", type=int, default=5, help="create/boot/.../delete cycles (the leak gates need at least 4)") p.add_argument("--storm", type=int, default=4, help="boot this many at once (0: skip)") p.add_argument("--storm-bar-s", type=float, default=300) - p.add_argument("--leak-rss-mb", type=float, default=20, help="FAIL above this QEMU RSS growth per cycle (MB)") + p.add_argument("--leak-rss-mb", type=float, default=20, help="FAIL above this steady growth of QEMU's memory (RSS + swap) per cycle (MB)") p.add_argument("--leak-guest-mb", type=float, default=30, help="FAIL above this guest memory growth per cycle (MB)") if name == "soak": p.add_argument("--soak-n", type=int, default=3) p.add_argument("--soak-minutes", type=float, default=15) p.add_argument("--soak-every", type=float, default=30) - p.add_argument("--soak-rss-mb-min", type=float, default=5, help="FAIL above this QEMU RSS growth (MB/min)") + p.add_argument("--soak-rss-mb-min", type=float, default=5, help="FAIL above this steady growth of QEMU's memory (RSS + swap) (MB/min)") p.add_argument("--soak-guest-mb-min", type=float, default=10, help="FAIL above this guest memory growth (MB/min)") if name == "build": p.add_argument("--xcodebuild", default="", help="arguments for xcodebuild (a project or workspace and scheme) to time cold and warm") diff --git a/src/bench/macbench/gates.py b/src/bench/macbench/gates.py index 2cf3117..244c170 100644 --- a/src/bench/macbench/gates.py +++ b/src/bench/macbench/gates.py @@ -174,9 +174,16 @@ def shots_gates(shots_by_sim, golden=None): def leak_gate(name, xs, ys, bar_per_unit, unit): - """FAIL when the least-squares slope of ys over xs exceeds the bar (e.g. MB per cycle).""" + """FAIL when the least-squares slope of ys over xs exceeds the bar (e.g. MB per cycle) AND the growth is steady: + at least 4 samples on a line (R2 >= 0.8). A slope above the bar from fewer samples or a scatter is one step, not a + leak: SKIP, and it says to run longer.""" from . import stats b, _, r2 = stats.slope(xs, ys) if b is None: return gate("SKIP", "%s: not enough samples" % name) - return gate("FAIL" if b > bar_per_unit else "PASS", "%s: %+.2f %s (bar %g, R2 %.2f, %d samples)" % (name, b, unit, bar_per_unit, r2, len(xs))) + detail = "%s: %+.2f %s (bar %g, R2 %.2f, %d samples)" % (name, b, unit, bar_per_unit, r2, len(xs)) + if b <= bar_per_unit: + return gate("PASS", detail) + if len(xs) < 4 or r2 < 0.8: + return gate("SKIP", detail + ": above the bar but not a steady growth; run more cycles or longer to judge") + return gate("FAIL", detail) diff --git a/src/bench/macbench/hostprobe.py b/src/bench/macbench/hostprobe.py index a5e4bb7..e8db001 100644 --- a/src/bench/macbench/hostprobe.py +++ b/src/bench/macbench/hostprobe.py @@ -153,6 +153,11 @@ def mem_snapshot(pid, root=""): st = _read(root, "/proc/%d/status" % pid) or "" m = re.search(r"VmRSS:\s+(\d+)", st) s["qemu_rss_kb"] = int(m.group(1)) if m else None + sw = re.search(r"VmSwap:\s+(\d+)", st) + s["qemu_swap_kb"] = int(sw.group(1)) if sw else None + # what QEMU holds, wherever the host keeps it: RSS alone rises when the host brings swapped pages back, which is + # not a leak (a VM with 8 GB in swap gained 480 MB of RSS in one create/boot/delete cycle) + s["qemu_mem_kb"] = None if not m else int(m.group(1)) + (int(sw.group(1)) if sw else 0) return s diff --git a/src/bench/macbench/others.py b/src/bench/macbench/others.py index ee02567..d82a0c0 100644 --- a/src/bench/macbench/others.py +++ b/src/bench/macbench/others.py @@ -57,7 +57,7 @@ def run_lifecycle(ctx): core.capture_env(ctx, info) ctx.marks = {"coh": ctx.coh.mark(), "t0": time.time()} cyc, xs, rss, used, comp = [], [], [], [], [] - base = ctx.host.snap()["mem"].get("qemu_rss_kb") + base = ctx.host.snap()["mem"].get("qemu_mem_kb") mem0 = ctx.guest.call(["memcheck"], 60)["mem"] data = ctx.doc["data"] = {"cycles": cyc, "storm": None} for i in range(a.cycles): @@ -71,7 +71,7 @@ def run_lifecycle(ctx): xs.append(i + 1) used.append(m["mem"]["used_mb"] - mem0["used_mb"]) comp.append(m["mem"]["compressed_mb"]) - sn = ctx.host.snap()["mem"].get("qemu_rss_kb") + sn = ctx.host.snap()["mem"].get("qemu_mem_kb") rss.append(None if sn is None or base is None else (sn - base) / 1024.0) ctx.save() keys = sorted({k for c in cyc for k in c}) @@ -83,9 +83,9 @@ def run_lifecycle(ctx): ("resume_first_frame_s", "first frame after resume"), ("clone_boot_s", "clone boot"), ("delete_s", "delete")) ctx.section("lifecycle", "PASS" if ok else "FAIL", "LIFECYCLE %s (%d cycles, medians, s; %s): %s" % ( "PASS" if ok else "FAIL", len(cyc), "fresh stock devices" if ctx.stock else "clones of the seed", ", ".join("%s %s" % (label, med[k]) for k, label in names if k in med))) - leaks = {"qemu_rss": gates.leak_gate("QEMU RSS", xs, rss, a.leak_rss_mb, "MB/cycle"), + leaks = {"qemu_mem": gates.leak_gate("QEMU memory (RSS + swap)", xs, rss, a.leak_rss_mb, "MB/cycle"), "guest_used": gates.leak_gate("guest used memory", xs, used, a.leak_guest_mb, "MB/cycle")} - data["leak"] = {"qemu_rss_mb": rss, "guest_used_mb": used} + data["leak"] = {"qemu_mem_mb": rss, "guest_used_mb": used} for k, g in leaks.items(): ctx.doc["gates"]["leak_" + k] = g ctx.section("leak:" + k, g["status"], "LEAK %s: %s" % (g["status"], g["detail"])) @@ -127,15 +127,15 @@ def run_soak(ctx): ctx.run.need(60, "a soak sample") time.sleep(min(a.soak_every, max(0, t_end - time.time()))) m = ctx.guest.call(["memcheck"], 60) - sn = ctx.host.snap()["mem"].get("qemu_rss_kb") + sn = ctx.host.snap()["mem"].get("qemu_mem_kb") xs.append((time.time() - ctx.marks["t0"]) / 60.0) used.append(m["mem"]["used_mb"]) rss.append(None if sn is None else sn / 1024.0) - ctx.doc["data"]["soak"] = {"minutes": xs, "qemu_rss_mb": rss, "guest_used_mb": used} + ctx.doc["data"]["soak"] = {"minutes": xs, "qemu_mem_mb": rss, "guest_used_mb": used} g, s0, s1, t_go = core.window(ctx, a.secs, "all-busy") res[label] = core.build_step(ctx, n, "all-busy", g, s0, s1, t_go) ctx.doc["data"]["steps"] = [res["start"], res["end"]] - for k, gname, ys, bar in (("QEMU RSS", "leak_qemu_rss", rss, a.soak_rss_mb_min), ("guest used memory", "leak_guest_used", used, a.soak_guest_mb_min)): + for k, gname, ys, bar in (("QEMU memory (RSS + swap)", "leak_qemu_mem", rss, a.soak_rss_mb_min), ("guest used memory", "leak_guest_used", used, a.soak_guest_mb_min)): gt = gates.leak_gate(k, xs, ys, bar, "MB/min") ctx.doc["gates"][gname] = gt ctx.section(gname, gt["status"], "SOAK %s: %s" % (gt["status"], gt["detail"])) diff --git a/src/bench/simfps.swift b/src/bench/simfps.swift index 1540212..821f57b 100644 --- a/src/bench/simfps.swift +++ b/src/bench/simfps.swift @@ -12,6 +12,7 @@ // several simfps (one per simulator) measure the same window // SIMFPS_PROBE=, a point in points (402 pt wide screen): its luma is read from every new surface // SIMFPS_STAMPS= one line per new surface: " " +// (counter -3: the same surface looked at again, its marker changed since the first look) // // --strip also reads GlassBench's frame strip (a 16-bit counter the app draws every tick, see // GlassBench/main.swift) from each new surface: how many app frames reached the screen, how many were @@ -154,7 +155,13 @@ let poller = Thread { var lastSeed = IOSurfaceGetSeed(surf0) var lastSurf = surf0 // The first line of SIMFPS_STAMPS is the screen as it stands at the start (a static screen makes no new surface). - if rawPath != nil { let r = readSurface(surf0, strip: readStrip, probe: probe); raw.append((start, r.0, r.1)) } + // A picture read at the moment it appears can still show the frame before it. On a moving screen the next + // picture corrects that; on a still one nothing follows, and a flipped marker was never seen (a "missed" probe). + // So the same surface is looked at again 30, 150 and 600 ms after it appeared; a marker that changed meanwhile + // is a line with counter -3 (a second look, not a new picture). + let lookAt: [UInt64] = [30_000_000, 150_000_000, 600_000_000] + var lastNew = start, looks = 0, lastLuma = -1 + if rawPath != nil { let r = readSurface(surf0, strip: readStrip, probe: probe); raw.append((start, r.0, r.1)); lastLuma = r.1 } while true { let t = nowNs() if t >= end { break } @@ -165,6 +172,11 @@ let poller = Thread { let (counter, luma) = readSurface(s, strip: readStrip, probe: rawPath != nil ? probe : nil) if readStrip { strip.append((t, counter)) } if rawPath != nil { raw.append((t, counter, luma)) } + lastNew = t; looks = 0; lastLuma = luma + } else if rawPath != nil, let p = probe, looks < lookAt.count, t - lastNew >= lookAt[looks] { + looks += 1 + let l = readSurface(lastSurf, strip: false, probe: p).1 + if l >= 0 && (l > 127) != (lastLuma > 127) { raw.append((t, -3, l)); lastLuma = l } } polls += 1 usleep(500) diff --git a/src/bench/tests/test_dry.py b/src/bench/tests/test_dry.py index e8495cb..3b2120b 100644 --- a/src/bench/tests/test_dry.py +++ b/src/bench/tests/test_dry.py @@ -272,7 +272,7 @@ def test_idle_lifecycle_build_panel(self): lc = self.r["lifecycle"]["doc"] self.assertTrue(sections(lc, "lifecycle")) self.assertTrue(sections(lc, "storm")) - self.assertIn("leak_qemu_rss", lc["gates"]) + self.assertIn("leak_qemu_mem", lc["gates"]) self.assertEqual(sims_booted(self.r["lifecycle"]["state"]), []) self.assertEqual(sections(self.r["build"]["doc"], "xcodebuild")[0]["status"], "SKIP") self.assertEqual(sections(self.r["panel_skip"]["doc"], "panel")[0]["status"], "SKIP") diff --git a/src/bench/tests/test_modules.py b/src/bench/tests/test_modules.py index cebf5cc..29c89e5 100644 --- a/src/bench/tests/test_modules.py +++ b/src/bench/tests/test_modules.py @@ -272,6 +272,9 @@ def test_leak_gate(self): self.assertEqual(gates.leak_gate("x", [1, 2, 3, 4], [0, 50, 100, 150], 20, "MB")["status"], "FAIL") self.assertEqual(gates.leak_gate("x", [1, 2, 3, 4], [0, 1, 2, 3], 20, "MB")["status"], "PASS") self.assertEqual(gates.leak_gate("x", [1], [0], 20, "MB")["status"], "SKIP") + # one step up, then flat (a host bringing swapped pages back): above the bar, but not a steady growth + self.assertEqual(gates.leak_gate("x", [1, 2, 3], [90, 570, 520], 20, "MB")["status"], "SKIP") + self.assertEqual(gates.leak_gate("x", [1, 2, 3, 4, 5], [0, 480, 470, 490, 480], 20, "MB")["status"], "SKIP") class Schema(unittest.TestCase): diff --git a/src/bench/tests/test_pure.py b/src/bench/tests/test_pure.py index 0df029d..bb2eb1b 100644 --- a/src/bench/tests/test_pure.py +++ b/src/bench/tests/test_pure.py @@ -179,6 +179,16 @@ def test_latency_and_misses(self): self.assertEqual(r["max_ms"], 2000.0) self.assertAlmostEqual(r["call_p50_ms"], 120, delta=1) + def test_a_second_look_answers_a_probe_and_is_not_a_frame(self): + # a still screen: the surface that appeared at 103.07 was read too early (old marker), the second look saw the flip + raw = [(100.0, -1, 255), (103.07, -1, 255), (103.10, analysis.SECOND_LOOK, 0)] + r = analysis.probe_latencies(raw, [(103.0, 103.1)]) + self.assertEqual((r["misses"], r["second_looks"]), (0, 1)) + self.assertAlmostEqual(r["latencies_ms"][0], 100, delta=1) + busy = [(100 + i / 60, i % 65536, 255) for i in range(120)] + [(100.5001, analysis.SECOND_LOOK, 0)] + m = analysis.sim_metrics(0, "busy", sorted(busy), []) + self.assertEqual(m["strip"]["unreadable"], 0) # the second look is not an unreadable frame + def test_static_screen_uses_the_start_line(self): raw = [(100.0, -1, 255), (103.07, -1, 0)] # no frames between: a static screen r = analysis.probe_latencies(raw, [(103.0, 103.1)]) From 32f96d65417c2b3672c4dd75e3d8e55cf2669409 Mon Sep 17 00:00:00 2001 From: TUNA_BIN <65464657+TUNA-NOPE@users.noreply.github.com> Date: Tue, 17 Nov 2026 10:26:32 +0200 Subject: [PATCH 23/23] bench: the second look only on still screens; what differs from a lease simfps looked again at an unchanged surface 30 ms after it appeared, also while reading the strip. With 2 or 3 animating simulators a picture stays longer than that, so it would have been a second lock of most pictures (two locks halved the frame rate). It now does so only where it does not read the strip, which is the still screen it was made for and the only place it ran on a VM. README: a bench clone keeps the LaunchServices database a Simlock lease loses, so its first boot is not a lease's; the proxy on a moving screen may carry one picture's interval; the RSS step in the first lifecycle run is likely swap-in, it was not counted. --- src/bench/README.md | 15 +++++++++++---- src/bench/macbench/hostprobe.py | 5 +++-- src/bench/simfps.swift | 11 ++++++----- 3 files changed, 20 insertions(+), 11 deletions(-) diff --git a/src/bench/README.md b/src/bench/README.md index 9aaf7e9..72f6534 100644 --- a/src/bench/README.md +++ b/src/bench/README.md @@ -57,6 +57,10 @@ stopped the VM) and keeps the Home Screen widgets, one of which retries a servic a check that its `disabled.plist` has at least 100 services off (168 today), the bench app installed, a real settle (the guest quiet for two samples in a row, `--seed-settle-max` 600 s; whether the widgets' `chronod/icons` folder exists is recorded as `icons`, not required) and a shutdown. A seed that fails any step is deleted again, never left to be reused. +One difference from a Simlock lease remains: Simlock deletes a clone's LaunchServices database before its first boot and +the bench does not, so a bench clone's first boot (5.6 s here) is not a lease's (17-22 s here); everything after the +boot has settled is the same. + 2. **Every bench simulator is `simctl clone `** (an APFS clone: no disk cost) with the seed's `disabled.plist` copied into its own `/private/var/tmp/com.apple.CoreSimulator.SimDevice.` folder (mode 0700) before its first boot, and checked there; the app is already in it. @@ -181,9 +185,12 @@ result records it. `--probe-method openurl` (`simctl openurl glassbench://toggle `auto`: iOS 26 answers it with an "Open in GlassBench?" alert nobody taps, which stays over the app, through a relaunch too, and costs a third of its frame rate (121 against 80-83 screen frames/s, one 3x simulator). For each injection: *t0* = the wall clock just before the command is started in the guest; the answer is the first new surface after t0 whose marker (luma above or below 127 at a fixed point) differs from the last -surface before t0; latency = that surface's time - t0. On a still screen `simfps` looks at each surface again 30, 150 -and 600 ms after it appeared (a picture read the moment it appears can still show the frame before it, and nothing -follows to correct that); probes answered only by such a second look are counted in `input_second_looks`. A probe not answered in `--probe-timeout` (2 s) is a **miss +surface before t0; latency = that surface's time - t0. A picture read the moment it appears can still show the frame +before it (14-15 of 15 single changes on a still screen here). On a still screen nothing follows to correct that, so +there `simfps` looks at each surface again 30, 150 and 600 ms after it appeared; probes answered only by such a second +look are counted in `input_second_looks`, and their latency is up to 30 ms long. On a moving screen there is no second +look (it would be a second lock of most pictures); if the first look is a picture behind there too (not measured), the +proxy carries one picture's interval, which grows with N (about 10, 21 and 33 ms at 1, 2 and 3 animating simulators). A probe not answered in `--probe-timeout` (2 s) is a **miss and counts at the timeout** in p50/p95, never dropped. The number includes starting `simctl` (its own duration is `input_call_p50_ms`), CoreSimulator, the app's handling and one composite. It excludes the Device panel's stream, the viewer and a real finger's HID path. Read it as a trend over N and between builds, not as an absolute touch latency; @@ -269,7 +276,7 @@ written first. A gate that cannot be read is SKIP, shown as such, never PASS. | `counter` | `simfps` answered with a result for every simulator of the window and every animating simulator's surfaces gave a readable frame counter (a counter that answers nothing or an error fails the window, it is not a missing metric) | | `memory` | the VM's own cgroup swapped no more than 50 MB (SKIP when the cgroup is unreadable: other projects' swapping on the host is not this run's); guest swap-outs fail it with `--fail-on-swap` | | `run_*` | every scenario, after its body: `run_coherent_census`, `run_coherent_process`, `run_qemu_log`, `run_gpu_hang` and `run_kernel_gpu` since the run began, and `run_crash_reports` (a new crash report since setup, judged like `stability`'s). They fail the exit code of `idle`, `lifecycle`, `soak`, `ab` too. | -| `leak_*` | `lifecycle`: QEMU's memory (RSS + swap: RSS alone rises when the host brings swapped pages back) and guest memory grow less than `--leak-rss-mb` / `--leak-guest-mb` per create/boot/delete cycle (least-squares slope); `soak`: less than `--soak-rss-mb-min` / `--soak-guest-mb-min` per minute. FAIL needs a steady growth (at least 4 samples, R2 >= 0.8); a slope above the bar without that is SKIP | +| `leak_*` | `lifecycle`: QEMU's memory (RSS + swap: RSS alone also rises when the host brings swapped pages back) and guest memory grow less than `--leak-rss-mb` / `--leak-guest-mb` per create/boot/delete cycle (least-squares slope); `soak`: less than `--soak-rss-mb-min` / `--soak-guest-mb-min` per minute. FAIL needs a steady growth (at least 4 samples, R2 >= 0.8); a slope above the bar without that is SKIP | | `observer` | what looking costs: a second window of the first N reads nothing from the screen and injects nothing. PASS within 4%; beyond it INFO (never FAIL): the distinct-frame numbers are lower bounds. One lock of each picture cost between nothing and 31% of the screen frames in seven runs here (+2, -2.5, -5.6, -12.7, -20, -23, -31%; why it varies is not known); a second lock per picture halved them, so `simfps` reads strip and marker under one | ## A/B discipline diff --git a/src/bench/macbench/hostprobe.py b/src/bench/macbench/hostprobe.py index e8db001..77f9faa 100644 --- a/src/bench/macbench/hostprobe.py +++ b/src/bench/macbench/hostprobe.py @@ -155,8 +155,9 @@ def mem_snapshot(pid, root=""): s["qemu_rss_kb"] = int(m.group(1)) if m else None sw = re.search(r"VmSwap:\s+(\d+)", st) s["qemu_swap_kb"] = int(sw.group(1)) if sw else None - # what QEMU holds, wherever the host keeps it: RSS alone rises when the host brings swapped pages back, which is - # not a leak (a VM with 8 GB in swap gained 480 MB of RSS in one create/boot/delete cycle) + # what QEMU holds, wherever the host keeps it: RSS alone also rises when the host brings swapped pages back, which + # is not a leak (a VM with 8 GB in swap gained 480 MB of RSS in one create/boot/delete cycle and then stayed flat; + # swap-in is the likely cause, it was not counted in that run) s["qemu_mem_kb"] = None if not m else int(m.group(1)) + (int(sw.group(1)) if sw else 0) return s diff --git a/src/bench/simfps.swift b/src/bench/simfps.swift index 821f57b..72e40cf 100644 --- a/src/bench/simfps.swift +++ b/src/bench/simfps.swift @@ -113,8 +113,8 @@ let probe: (Double, Double)? = env["SIMFPS_PROBE"].flatMap { s -> (Double, Doubl var raw: [(UInt64, Int, Int)] = [] // (time, counter or -1, probe luma or -1) of every new surface /// One look at a BGRA surface under ONE lock: GlassBench's counter (18 blocks of 20 pt from (21, 120) pt, at the /// surface's scale; -1 when unreadable or not asked for) and the luma (0-255) at the probe point (-1 without one). -/// A lock of a surface the simulator just drew makes it wait for the pixels: two locks per surface halved the -/// simulator's frame rate (55 against 102 screen frames/s, one 3x simulator), one lock costs a few percent. +/// Locks are not free for the simulator: two per surface halved its frame rate (55 against 102 screen frames/s, one +/// 3x simulator); one cost between nothing and 31% over seven runs (why it varies is not known). func readSurface(_ s: IOSurface, strip: Bool, probe: (Double, Double)?) -> (Int, Int) { guard strip || probe != nil else { return (-1, -1) } let w = IOSurfaceGetWidth(s), h = IOSurfaceGetHeight(s), bpr = IOSurfaceGetBytesPerRow(s) @@ -157,8 +157,9 @@ let poller = Thread { // The first line of SIMFPS_STAMPS is the screen as it stands at the start (a static screen makes no new surface). // A picture read at the moment it appears can still show the frame before it. On a moving screen the next // picture corrects that; on a still one nothing follows, and a flipped marker was never seen (a "missed" probe). - // So the same surface is looked at again 30, 150 and 600 ms after it appeared; a marker that changed meanwhile - // is a line with counter -3 (a second look, not a new picture). + // So a reader that does not read the strip (a still screen) looks at the same surface again 30, 150 and 600 ms + // after it appeared; a marker that changed meanwhile is a line with counter -3 (a second look, not a new picture). + // Never with --strip: at 30 pictures a second the first second look would be a second lock of most pictures. let lookAt: [UInt64] = [30_000_000, 150_000_000, 600_000_000] var lastNew = start, looks = 0, lastLuma = -1 if rawPath != nil { let r = readSurface(surf0, strip: readStrip, probe: probe); raw.append((start, r.0, r.1)); lastLuma = r.1 } @@ -173,7 +174,7 @@ let poller = Thread { if readStrip { strip.append((t, counter)) } if rawPath != nil { raw.append((t, counter, luma)) } lastNew = t; looks = 0; lastLuma = luma - } else if rawPath != nil, let p = probe, looks < lookAt.count, t - lastNew >= lookAt[looks] { + } else if rawPath != nil, !readStrip, let p = probe, looks < lookAt.count, t - lastNew >= lookAt[looks] { looks += 1 let l = readSurface(lastSurf, strip: false, probe: p).1 if l >= 0 && (l > 127) != (lastLuma > 127) { raw.append((t, -3, l)); lastLuma = l }