diff --git a/.gitignore b/.gitignore index b4fbec5..d59a8c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.bak* __pycache__/ /shots/ +/bench-results/ diff --git a/macbox b/macbox index 5cb6dd3..27eeb0d 100755 --- a/macbox +++ b/macbox @@ -662,6 +662,15 @@ cmd_coherent() { # Coherent's crash guard (docs/performance.md) esac } +cmd_bench() { # the scaling, regression and A/B suite (src/bench/README.md) + case "${1:-}" in + compare|chart|report|tools|check|"") ;; + *) [ "${DRY:-0}" = 1 ] || { vm_running || die "the VM is not running: macbox up first (DRY=1 runs the suite against stubs)"; ssh_ok || die "ssh $MACBOX_SSH_HOST doesn't answer"; } ;; + esac + BENCH_QEMU_PID="$(qemu_pid)" BENCH_BUILD_ID="$(build_id)" BENCH_RUN="$RUN" BENCH_QEMU_LOG="$VMDIR/logs/qemu.log" \ + BENCH_SSH="$MACBOX_SSH_HOST" BENCH_HOME="$MACBOX_HOME" exec "$REPO/src/bench/run" "$@" +} + cmd_backup() { local name=${1:?usage: macbox backup } b="$MACBOX_HOME/backups/$1" disk vm_running && die "the VM is running: macbox down first" @@ -707,6 +716,8 @@ Day to day: macbox coherent [status|reset] Coherent's crash guard (MACBOX_COHERENT=auto|on|off) macbox video-kexts reims-util's video kexts into OpenCore (for MACBOX_VIDEO=on) macbox backup + macbox bench scale|idle|lifecycle|soak|build|panel|ab|golden|seed|cleanup|compare|chart|report|tools|check + the scaling, regression and A/B suite (src/bench/README.md) EOF } @@ -761,6 +772,7 @@ case "$sub" in coherent) cmd_coherent "$@" ;; video-kexts) cmd_video_kexts "$@" ;; backup) cmd_backup "$@" ;; + bench) cmd_bench "$@" ;; help|-h|--help) cmd_help ;; *) cmd_help >&2; exit 2 ;; esac diff --git a/src/bench/GlassBench/main.swift b/src/bench/GlassBench/main.swift index 95612da..64cc868 100644 --- a/src/bench/GlassBench/main.swift +++ b/src/bench/GlassBench/main.swift @@ -2,7 +2,14 @@ // A table of cards scrolls by itself at a fixed speed (time-based, so a dropped // frame skips content rather than slowing the scroll) under a glass navigation bar, // a glass tab bar and a floating glass toolbar with a glass button that keeps pulsing. -// Launch arguments: -noglass (no floating glass, opaque bars), -speed , -nostrip. +// Launch arguments: -noglass (no floating glass, opaque bars), -speed , -nostrip, +// -static (nothing animates: a fixed picture for pixel-exact checks and for a simulator that sits idle), +// -probe (an input marker, see below). +// +// The input marker (-probe): a 30 pt square at (21, 160) pt that flips between white and black on every +// tap, on the Darwin notification dev.macbox.glassbench.toggle (`simctl spawn notifyutil -p ...`) and +// on every glassbench://toggle URL (`simctl openurl`). The bench times from the injection to the +// first surface whose marker flipped (bench/README.md, "input proxy"). // // The frame strip: every display-link tick draws a 16-bit counter as blocks at the top left, so a // frame can be told apart from the one before it all the way to a viewer (simfps --strip at the @@ -14,6 +21,34 @@ import UIKit let launchArgs = ProcessInfo.processInfo.arguments let noGlass = launchArgs.contains("-noglass") let showStrip = !launchArgs.contains("-nostrip") +let isStatic = launchArgs.contains("-static") +let showProbe = launchArgs.contains("-probe") + +/// The input marker: white until toggled, then black, then white again. It follows the system appearance +/// until the first toggle, so `simctl ui appearance` flips it too (the bench's fallback injection). +final class ProbeMarker: UIView { + private var dark = false + init() { + super.init(frame: CGRect(x: 21, y: 160, width: 30, height: 30)) + backgroundColor = .systemBackground + layer.borderWidth = 1; layer.borderColor = UIColor.gray.cgColor + isUserInteractionEnabled = false + } + required init?(coder: NSCoder) { fatalError() } + func toggle() { + dark.toggle() + overrideUserInterfaceStyle = dark ? .dark : .light + } +} +var probeMarker: ProbeMarker? + +/// Any touch on the window flips the marker (the bench's tap injection). +final class BenchWindow: UIWindow { + override func sendEvent(_ event: UIEvent) { + if event.type == .touches, event.allTouches?.contains(where: { $0.phase == .began }) == true { probeMarker?.toggle() } + super.sendEvent(event) + } +} final class FrameStrip: UIView { static let origin = CGPoint(x: 21, y: 120), block: CGFloat = 20 @@ -155,7 +190,7 @@ final class BenchViewController: UITableViewController { override func viewDidAppear(_ animated: Bool) { super.viewDidAppear(animated) if !noGlass, let host = navigationController?.view, pulse.superview == nil { addFloatingGlass(to: host) } - guard link == nil else { return } // viewDidAppear runs again on every return to the app + guard link == nil, !isStatic else { return } // viewDidAppear runs again on every return to the app link = CADisplayLink(target: self, selector: #selector(step)) link?.preferredFrameRateRange = CAFrameRateRange(minimum: 60, maximum: 60, preferred: 60) startTime = CACurrentMediaTime() @@ -243,8 +278,20 @@ final class SceneDelegate: UIResponder, UIWindowSceneDelegate { other.tabBarItem = UITabBarItem(title: "Other", image: UIImage(systemName: "square.grid.2x2"), tag: 1) let tabs = UITabBarController(); tabs.viewControllers = [nav, other] if noGlass { tabs.tabBar.isHidden = true } - let w = UIWindow(windowScene: ws); w.rootViewController = tabs; w.makeKeyAndVisible() + let w = BenchWindow(windowScene: ws); w.rootViewController = tabs; w.makeKeyAndVisible() if showStrip { let f = FrameStrip(); w.addSubview(f); frameStrip = f } + if showProbe { + let m = ProbeMarker(); w.addSubview(m); probeMarker = m + // `simctl spawn notifyutil -p dev.macbox.glassbench.toggle`: an event from outside that puts + // nothing on the screen but the marker's flip (an openurl raises an alert, an appearance switch redraws all). + CFNotificationCenterAddObserver(CFNotificationCenterGetDarwinNotifyCenter(), nil, { _, _, _, _, _ in + DispatchQueue.main.async { probeMarker?.toggle() } + }, "dev.macbox.glassbench.toggle" as CFString, nil, .deliverImmediately) + } window = w } + + func scene(_ scene: UIScene, openURLContexts contexts: Set) { + if contexts.contains(where: { $0.url.scheme == "glassbench" }) { probeMarker?.toggle() } + } } diff --git a/src/bench/README.md b/src/bench/README.md index 13cd5ac..72f6534 100644 --- a/src/bench/README.md +++ b/src/bench/README.md @@ -1,7 +1,338 @@ # Benchmarks -The tools behind the numbers in [docs/performance.md](../../docs/performance.md), for measuring your -own GPU, or a change, before and after. They run inside macOS; build them there once: +How many simulators does this host run at once, each one smooth and responsive? `macbox bench` answers that +and guards every change against it: one suite, one result file per run, one comparison between runs. The older +single-purpose tools it builds on (`simfps`, GlassBench, GenieBench) are described [at the end](#the-tools). + +```sh +macbox bench tools # once: copy the bench to the Mac, build simfps and GlassBench there +macbox bench check # what this host and the VM offer +macbox bench seed # once: the production seed every bench simulator is cloned from (kept between runs) +macbox bench scale --n 1,2,3,4,5,6 # the scaling sweep (the centrepiece) +macbox bench cleanup # after a run that could not clean up (the next run refuses to start until then) +macbox bench compare baseline.json new.json +macbox bench chart new.json # an HTML/SVG chart next to it +DRY=1 macbox bench scale --n 1,2,3 # the whole control flow against stubs: no VM, no simulator, no GPU +``` + +Requirements: Python 3.8+ on the host (standard library only; no Pillow, no numpy), the VM running, `ssh` to it. +Unit tests, no VM needed: `cd src/bench && python3 -m unittest discover -s tests -t .` (the pure parts and the run's safety) +and `python3 -m unittest tests.test_dry tests.test_dry_state tests.test_dry_seed tests.test_dry_closed tests.test_dry_guards` (the dry runs with injected faults: the sweeps and their faults; what a run leaves and the idle guard, including a run killed mid-seed; the seed; the fail-closed gates; the host-disk, VM and abort guards; several minutes, run one module at a time on a busy host). + +**What has run on a real VM so far** (2026-11-17: macOS 26.7.1 guest, iPhone 17 Pro, iOS 26.5, an Arrow Lake iGPU): +`tools`, `check`, `seed`, `cleanup`, `scale` at 3x for N = 1, 2, 3 in the `all-busy` and `one-busy` mixes, and +`lifecycle` (5 cycles, no boot storm). All animating: 57.6, 39.5 and 23.2 distinct frames/s each (p95 24, 45, 75 ms; +input proxy p50 283, 312, 356 ms), within the noise of a hand-made measurement of the same night (60, 37-41, 22-25). +One animating beside still ones: it keeps 60 (59.9, 59.8, 60.0), and the still ones answer the input proxy as fast +as it does (p95 303-311 ms, no miss). Lifecycle medians: clone 2.9 s, first boot 5.6 s, first app frame 1.0 s after a +0.5 s launch call, shutdown 3.2 s, resume boot 5.5 s, delete 0.2 s. Everything else (`idle`, `soak`, `ab`, the `agent` +mix, the lifecycle's boot storm, the 1x device type, N above 3) has only run against the stubs (`DRY=1` and the unit +tests): expect to fix things the first time you run them. Only `scale` and `ab --metric input_p95` inject input +events; the other scenarios' windows inject nothing and leave `responsive` unknown. + +## What a run does, and what it never touches + +The bench has a **device set of its own** (`simctl --set`, a directory with a marker file, the same one for every run so +its seed is kept), so it never sees the pool's simulators or anyone's lease; every `simctl` the guest helper starts goes +through one function that adds `--set` and refuses a set without the marker. The one exception is the clone out of +Simlock's source set (below), which is read-only for Simlock. At the end of a run the bench shuts down and deletes every +device it made except the seed. The 3x device is the stock iPhone Pro; the **1x type** is the macbox 1x device type +`macbox pool` makes (same identifier, reused when it exists; made by the bench when it does not, and then +CoreSimulatorService must restart to load it, which closes every booted simulator: the bench refuses that unless nothing +is booted and `--allow-restart` is given). It starts no tracer. It reads `/proc`, `/sys`, the kernel journal and the VM's +census and log; it writes only its result file, and, with `ab --knob live:KEY=VALUE`, the one line in the VM's live +switch file (restored byte for byte on every way out). + +### Bench simulators are production simulators + +A fresh stock device is not what an agent gets: it runs stock services (a first boot keeps the vCPUs busy for minutes), +costs about 1.8 GB of guest disk (five of them took the guest's disk, then the host's, to the storage guard's limit and +stopped the VM) and keeps the Home Screen widgets, one of which retries a service Slim turns off in a tight loop +(~3 cores per "idle" simulator, for ever). So the bench makes simulators the way Simlock does: + +1. **One seed per device type**, kept shut down in the bench's set between runs: `simctl --set + clone ` (the source is Simlock's own device of that type: widgets trimmed, not Slim; Simlock's + set is only ever cloned from, never written, booted, shut down or deleted, and its config is not read), registered with + `sim-idle-guard`, one boot, `simslim --set on --profile ` (it reboots the device), + a check that its `disabled.plist` has at least 100 services off (168 today), the bench app installed, a real settle + (the guest quiet for two samples in a row, `--seed-settle-max` 600 s; whether the widgets' `chronod/icons` folder exists is recorded as `icons`, not required) and a + shutdown. A seed that fails any step is deleted again, never left to be reused. +One difference from a Simlock lease remains: Simlock deletes a clone's LaunchServices database before its first boot and +the bench does not, so a bench clone's first boot (5.6 s here) is not a lease's (17-22 s here); everything after the +boot has settled is the same. + +2. **Every bench simulator is `simctl clone `** (an APFS clone: no disk cost) with the seed's `disabled.plist` copied + into its own `/private/var/tmp/com.apple.CoreSimulator.SimDevice.` folder (mode 0700) before its first boot, and + checked there; the app is already in it. +3. The seed is **reused** by later runs (its Slim services, its widgets and the app's build are checked: a seed made for + another build of the bench app is refused, not silently reinstalled). `macbox bench seed --rebuild-seed` (or + `--rebuild-seed` on any run) makes a new one, which you need after `macbox bench tools` rebuilt the app. + +Simlock is an **optional** integration. Without a source device of the wanted type (`macbox bench check` lists the types +it finds) the run stops and says so. Two explicit ways out: `--scale-x 3` (the stock iPhone Pro type, +which Simlock usually has a source for), or **`--stock-devices`**: fresh stock devices, created one at a time with the +host-disk guard in front of each, for a device type Simlock has no source for or a scenario that needs a stock device. +The result records `env.guest.device_source` (`stock devices` or clones of a seed) and `compare` warns when two results +were made differently. Defaults of the Mac's paths: `~/.simlock/devices/ios-sources` (`--source-set` or `BENCH_SIMLOCK_SOURCES`: the +`devices/ios-sources` folder of your Simlock's state directory, whatever that install calls it), +`~/go/bin/simslim` (`--simslim`), the only `.json` in `~/.config/simslim/` (`--slim-profile`; several or none: the run stops and asks for the flag), `~/.local/bin/sim-idle-guard`. + +### Safety, in every run that boots simulators + +The **hard deadline** (`--deadline`, default 90 minutes) stops the run 60 s early, writes the result so far and runs +the cleanup; **one exit trap** (normal end, error, SIGTERM/SIGINT, the deadline, a monitor's abort) kills the helpers it +started (explicit pids, never a bare `wait`), shuts down and deletes the bench's simulators and restores every knob. + +- **Idle guard.** The Mac's `sim-idle-guard` shuts down every simulator nobody used for 15 minutes, in any device set. + Every bench simulator (and the seed while it is booted) is registered with `sim-idle-guard keep ` for the + run's length (the deadline plus 5 minutes, at most 240; renewed before every window) and `~/.sim-keep/` is removed + by the cleanup. A guard that is not installed is recorded in the result (`idle_guard`, a section) and the run goes on; a + guard that refuses `keep` stops it. +- **Disk.** `--disk-path` (a path on the filesystem that holds the VM's storage and that your storage guard watches) must + have `--disk-start-gib` 18 free to start, `--disk-boot-gib` 14 to boot another simulator (checked before **every** boot: + simulators boot one after the other, the lifecycle's boot storm is the one exception), and under `--disk-abort-gib` 11 the + run is aborted and cleaned up even while it waits. A disk reading that fails refuses. +- **Guest memory.** Before every boot: the guest's available memory (free + speculative + inactive + purgeable pages) after + the estimated cost of the new simulators stays over `--min-guest-free-gib` (1.5), no memory pressure level 2 or more, + **swap-outs have not grown** since the run began, the **compressor** holds at most `--max-compressor-mb` (2048), the + host's available memory is over `--min-host-free-gib` (6). The sweep stops there; it does not swap the VM to death. +- **A monitor** runs on its own thread while the run waits in a long call (a boot, a settle, a window): the host disk, the + VM's QEMU process (the same pid **and** start time) always; the Mac (answers over ssh twice in a row, no simulator that is + not the bench's, swap-outs, compressor) outside measurement windows, whose own load it must not add to. A breach aborts + the run from that thread: the partial result is written, the cleanup runs, the cleanup's verdict is added and the process + exits with 8 (disk), 9 (VM), 7 (someone else's simulator: **yield**, `--allow-foreign` to measure anyway) or 10 (guest + memory). +- **Settle fails closed.** Before a window the guest must settle: every process except the load and its observers (the + renderers `SimMetalHost`/`backboardd`/`SimRenderServer`, GlassBench, `simfps`, WindowServer, python, ps, ssh, matched by + name against the whole command, and the helper's own tree) uses less than `--settle-bg` % of a core for two samples in a + row. A settle that reaches `--settle-max` (300 s) **fails the run** (a `settle` FAIL section with the top processes by CPU + at that moment); nothing is measured on top of an unsettled guest. +- **The count of booted simulators.** Before every window (scale steps, A/B arms, soak, the observer window, idle) the guest + counts the Booted devices of the bench's own set and the run fails on any mismatch with the expected number (a + simulator the idle guard or someone shut down would otherwise be measured as if it ran). A step's `booted` is that + measured count, `expected_booted` the expected one. +- **The frame counter fails the window.** A simulator for which `simfps` returned nothing or an error, or an animating one + from whose surfaces no counter could be read, fails the `counter` gate (and so the step, the run and the exit code). +- **Leftovers.** A run takes the set with a lock and **refuses to start** while an earlier run left a device other than the + seed, a booted or never-completed seed, a lock, or overrides/keep files of a deleted device; `macbox bench cleanup` removes + them (`--seeds` also deletes the seeds). The cleanup looks again after deleting and says what it could not remove + (`CLEANUP FAIL`, exit code **20**, the lock stays); "cleanup done" is only printed for a verified one. +- A sweep that ran no step at all (a refused guard, foreign simulators) is a FAIL with a non-zero exit. + +## Commands + +| | | +|---|---| +| `seed` | Make the production seed for the device type (or check and report the one that is there); `--rebuild-seed` makes a new one. | +| `cleanup` | Shut down and delete what an earlier run left in the bench's set (verified; exit 20 when something stays); `--seeds` deletes the seeds too. | +| `scale` | N = 1, 2, 3... simulators (`--n 1,2,3,4,6,8`, `--scale-x 1\|3`, `--mix`), per simulator numbers, the maximum sustainable N per quality target. Stops early when a floor is breached (`--floor`, default `fps>=20`), a gate fails or the system turns unstable. | +| `idle` | The VM with no simulator: QEMU, GPU and memory standing cost against `--bar-qemu-cores` (0.15). | +| `lifecycle` | `--cycles` times: create, boot, install, launch, first app frame, shut down, resume, first frame, clone, boot the clone, delete (medians of the timings below); a per-cycle leak gate; `--storm N`: N simulators booted at once. | +| `soak` | `--soak-n` busy simulators for `--soak-minutes`: slope of QEMU's RSS and of the guest's memory, and the frame rate at the start against the end. | +| `build` | Build timing: the bench tools (swiftc) cold and warm; `--xcodebuild ''` times your own project cold and warm (GlassBench is not an Xcode project, so the section is SKIP without it). | +| `panel` | The Device panel's stream: pluggable (see [The panel section](#the-panel-section)); SKIP without `--panel-cmd`. | +| `ab` | A B B A inside one window for one knob: ADOPT or CLOSE against a threshold fixed before the run. | +| `golden` | Records the pixel hash of the static reference screen for this setup in `baselines/golden.json`. | +| `compare A.json B.json` | What moved, per N, beyond the noise (`--noise 0.04`); exit status 1 on a regression. | +| `chart R.json [-o out.html]` | A self-contained HTML page of SVG charts: per-simulator fps and p95 against N, input proxy, QEMU CPU and GPU, guest CPU and memory, GPU ms per frame, fairness. No script, no external file. | +| `report R.json` | The PASS/FAIL lines of a result. | +| `tools [--no-build]` / `check` | Copy the bench to the guest and build it / what is there and what is missing. | + +Common options: `--out DIR` (results, default `bench-results/`), `--secs` (window length, default 40), +`--nominal-fps 60`, `--tag key=value` (recorded, for example `mesa=26.2.4` when Mesa is not a package), +`--probe-method auto|notify|appearance|openurl|cmd|none`, `--stock-devices`, `--rebuild-seed`, `--source-set`, `--simslim`, `--slim-profile`, `--seed-settle-max` (see above), `--disk-path`, `--disk-start-gib`, `--disk-boot-gib`, `--disk-abort-gib`, `--max-compressor-mb`, `--set-name` (the bench's device set, default `bench`), `--quiesce` (disables `mediaanalysisd` in each simulator, as +some old harnesses did; recorded, off by default), `--no-observer-check`, `--no-golden`, `--shots N`. +Every run prints one PASS/FAIL/SKIP/INFO line per section and exits 0, 1 (a section failed), 2 (the run could not +go on), 20 (the cleanup could not delete everything) or, when a monitor aborted it, 7, 8, 9 or 10 (above). The JSON has the same lines under `sections`. + +### Workload mixes (`scale --mix`) + +Agents' simulators are not all animating. + +| mix | what each simulator runs | what it measures | +|---|---|---| +| `all-busy` (default) | GlassBench scrolling, input marker on | every simulator at full rate; frames and input of each | +| `one-busy` | simulator 0 animates; the others show GlassBench `-static` and get an input probe every `--probe-every` s | whether a busy simulator makes the others sluggish (their input proxy), and what an idle-but-ready simulator costs | +| `agent` | per simulator, staggered: launch the app, let it scroll, screenshot, terminate, idle (`--agent-period`, 12 s) | launch to first frame, screenshot time, and the frame rate inside the motion stretches | +| `idle` | booted, nothing launched, no observer | the pure standing cost of a booted simulator: the slope of each resource over N | + +## What is measured: exact definitions + +Everything is per simulator first; system numbers come with it, never instead of it. A result never holds only an +average. + +**Frames** (per animating simulator; GlassBench draws a 16-bit counter strip, see below). `simfps --strip` polls the +simulator's framebuffer surface at about 2 kHz and reads the counter from every new surface. +- *distinct frames*: surfaces whose counter differs from the previous distinct one. `fps` = (n - 1) / (last - first). +- `p50_ms`, `p95_ms`, `p99_ms`, `max_ms`: percentiles of the intervals between consecutive distinct frames + (`sorted[min(n-1, int((n-1)*p))]`, the rule of simfps and GlassBench itself). `over2x`: intervals longer than + 2 / `--nominal-fps`. `longest_gap_s`: the longest interval, seconds (the "stall"). +- `strip`: `repeats` (same counter on another surface: normal, the compositor makes more surfaces than the app + ticks), `lost` (ticks that never reached the screen), `backwards` (an older counter after a newer one), + `jumps` (forward by more than 1000), `unreadable` (references not found), `span_ticks` (ticks the counter moved). +- `app_tick_fps`: GlassBench's own tick rate from its `pace.json` (the app, before the compositor), `surface_fps`: + surface changes per second as simfps counts them. Three layers, so a loss can be placed. +- The agent mix reports the same over the stretches of motion only (a gap above 250 ms ends a stretch). + +**Input latency proxy** (`input_p50_ms`, `input_p95_ms`, `input_max_ms`, `input_misses`, `input_call_p50_ms`). +It is a proxy and says so. Without Simlock or agent-device, the guest injects an event with `simctl`: by default +`simctl spawn notifyutil -p dev.macbox.glassbench.toggle`, a Darwin notification that flips a 30 pt marker +(white/black) in GlassBench `-probe` and puts nothing else on the screen; the fallback is `simctl ui appearance +dark|light`, which flips the same marker but redraws the whole screen (about 0.5 s to answer, and a window that uses +it lost a quarter of its distinct frames: 43 against 59 a second, one 3x simulator); `--probe-cmd` may run any command +that flips it (a real tap through your own tool). The first N checks which method is answered (3 tries) and the +result records it. `--probe-method openurl` (`simctl openurl glassbench://toggle`) is not tried by +`auto`: iOS 26 answers it with an "Open in GlassBench?" alert nobody taps, which stays over the app, through a +relaunch too, and costs a third of its frame rate (121 against 80-83 screen frames/s, one 3x simulator). For each injection: *t0* = the wall clock just before the command is started in the guest; the answer +is the first new surface after t0 whose marker (luma above or below 127 at a fixed point) differs from the last +surface before t0; latency = that surface's time - t0. A picture read the moment it appears can still show the frame +before it (14-15 of 15 single changes on a still screen here). On a still screen nothing follows to correct that, so +there `simfps` looks at each surface again 30, 150 and 600 ms after it appeared; probes answered only by such a second +look are counted in `input_second_looks`, and their latency is up to 30 ms long. On a moving screen there is no second +look (it would be a second lock of most pictures); if the first look is a picture behind there too (not measured), the +proxy carries one picture's interval, which grows with N (about 10, 21 and 33 ms at 1, 2 and 3 animating simulators). A probe not answered in `--probe-timeout` (2 s) is a **miss +and counts at the timeout** in p50/p95, never dropped. The number includes starting `simctl` (its own duration is +`input_call_p50_ms`), CoreSimulator, the app's handling and one composite. It excludes the Device panel's stream, the +viewer and a real finger's HID path. Read it as a trend over N and between builds, not as an absolute touch latency; +the default `responsive` bar (500 ms) sits about 200 ms above the proxy's floor measured here (the notification is +answered 280 ms after `simctl spawn` is started, one 3x simulator alone; the command itself takes 355 ms): check that +floor at N=1 on your hardware before you trust the bar. p95 over +few probes is coarse: a 40 s window gives 16 probes per simulator. + +**Guest CPU and memory of a simulator** (`guest_cpu_pct`, `guest_rss_mb`). From `ps` snapshots in the guest at the +window's start and end: the CPU time of every process in the simulator's tree over the window, in % of one core +(CPU time has a 10 ms resolution; a process that started or ended inside the window is counted only if it +was in both snapshots, so very short-lived processes are undercounted), and the RSS of the tree at the end. The +tree is every process whose command line names the simulator's UDID (`launchd_sim`, the per-device helpers) and +all their descendants, never the bench's own helper, its ssh and shell ancestors, or the `simfps`/`simctl`/`ps` it +started: their CPU is `system.guest.observer_cpu_pct` (the larger of the helper tree's CPU between the two `ps` snapshots and the CPU time of the helper's reaped children, `simfps` and the probes' `simctl`, over the window). RSS counts shared pages in each process: it overstates what leaving a simulator out would +free. `system.guest`: `total_cpu_pct`, `sims_cpu_pct` (the trees), `other_cpu_pct` (everything else), `sims_rss_mb`. + +**Lifecycle timings** (`lifecycle`, seconds, medians over `--cycles`; each cycle makes its own device the way a lease does: +a clone of the seed; with `--stock-devices` a fresh stock device, and then `create_s`, `install_s` and one more clone of the +installed device are timed too): `clone_s` (`simctl clone` of the seed + the Slim disables copied + the idle-guard keep); +`cold_boot_s` (the clone's first `boot` + `bootstatus`); `cold_launch_call_s` (the `launch` call); +`cold_first_frame_s` (from the launch call to the first counter value `simfps` reads on the new surface); `shutdown_s` +(shutting the booted device down), `resume_boot_s` and `resume_first_frame_s` (the same two again: "resume" here is +shutdown then boot, not Simlock's parking); `delete_s` (`simctl delete` of the device and its overrides folder; stock +path only: `clone_boot_s`, the boot of a clone of the installed device, deleted untimed). The boot storm +(`--storm N`) clones N simulators and boots them at once. + +**Guest memory** (`system.guest`): `mem_used_mb` = active + wired + compressor pages; `mem_free_mb` = free + +speculative; `mem_avail_mb` is what the guard uses: free + speculative + inactive + purgeable (`vm_stat` pages; 4 KiB +when the header does not say, 16 KiB on arm64); `compressed_mb`; `swapins_d`, `swapouts_d`, `pageouts_d` = the counters' change over the window; +`pressure_max` = the highest `kern.memorystatus_vm_pressure_level` sampled every 2 s (1 normal, 2 warning, 4 +critical). + +**QEMU CPU** (`system`): from `/proc//task/*/stat` over the window, in cores: `qemu_user_cores` = host-side +user time (utime - guest_time), `qemu_guest_cores` = time in guest mode (the vCPUs running the guest), +`qemu_sys_cores` = kernel time; `qemu_cpu_cores` their sum; `qemu_top_threads` the six busiest thread classes (digits +folded: `CPU N/KVM` is `vCPU`). + +**GPU** (`system`): `gpu_busy_pct` = the change of `drm-engine-render` summed over QEMU's DRM clients (fdinfo, +each client once) divided by the wall time; `gpu_ms_per_frame` = that GPU time divided by the distinct frames of all +animating simulators in the window. i915 only (xe states cycles, not ns: the field is null, never converted by a guess). + +**Host memory and swap** (`system.host`): `mem_avail_gb`, `qemu_rss_mb`, `cgroup_mem_gb`, `cgroup_swap_mb` and its +change over the window, `cgroup_events` (high/max/oom_kill deltas), `pswpin_pages`/`pswpout_pages` (system wide), +`psi_mem_some_pct` (system) and `cgroup_psi_some_pct` (the VM's cgroup). + +**Stability** (`stability`): new crash reports (`.ips`/`.crash` in the guest's DiagnosticReports) during the window +of the bench's own programs or of the simulator's graphics path (`CRASH_NAMES` in `macbench/gates.py`: GlassBench, +simfps, SpringBoard, backboardd, SimRenderServer, SimMetalHost, CoreSimulatorService, the kernel, jetsam events ...). +A Mac that runs simulators files reports for other daemons all day (widget extensions, searchd, containermanagerd); +those are recorded in `other_ips` and in the gate's text, and fail nothing unless `--strict-crashes` is given. +`--crash-names a,b` (or `BENCH_CRASH_NAMES`) adds names to judge. Also: +simulators no longer Booted at its end, kernel GPU error lines (`journalctl -k`: GPU hang, fence expired, context +reset, xe job timeouts and faults), QEMU still the same process. Any one makes the step unstable and ends the sweep. + +**Fairness** (`fairness`, per N): Jain's index over the distinct fps of the animating simulators (1.0 = equal, 1/n = +one gets everything) and min/max. + +**Maximum sustainable N** (`summary.max_n`, one per target). A target is `name: fps>=55, p95<=25` over per +simulator metrics (`fps p50 p95 p99 max over2x stall input_p50 input_p95 cpu rss`). N passes when every simulator +that has a value for each condition meets it, with its gates ok and the system stable. The maximum is the largest N +such that it and **every smaller measured N** passed: the first failure ends it (a pass above a failure is luck, not +sustainability). A condition no simulator has a value for makes the target unknown (`null`), never a pass. +Defaults: `smooth60: fps>=55, p95<=25`, `smooth30: fps>=30, p95<=50`, `responsive: input_p95<=500, stall<0.3`. + +**Standing cost per simulator** (`summary.standing_cost`): the least-squares slope of each system number over N, with +its intercept and R². Use `--mix idle` for the pure cost of a booted simulator. + +## Correctness gates + +They run with every scenario; a FAIL fails the run and stops a sweep, but the result with every smaller N is +written first. A gate that cannot be read is SKIP, shown as such, never PASS. + +| gate | PASS when | +|---|---| +| `strip` | no frame out of order, no counter jump, the counter advanced no more than the app's own tick log says, under 1% of samples unreadable. Lost ticks are the fps metric, not a gate failure. | +| `tears` | no screenshot of the scrolling GlassBench shows a torn band (card spacing and gap heights, the old `tears.py` rule, standard library PNG decoding); SKIP when no screenshot shows cards. A torn picture fails the step and the run, and that N does not count as sustained, but the sweep goes on to the next N (the stack tears now and then: 3 flagged of 50 judged pictures in one night here) | +| `pixels` | GlassBench `-static` screenshots agree with each other and match the stored golden (`baselines/golden.json`, key: macOS, runtime, device type, size), judged once on a freshly launched static app before the first window (a window with input probes flips the marker, so later static screens are not judged); SKIP with the pictures agreeing when no golden is stored: `macbox bench golden` records it | +| `coherent_census` / `coherent_process` | in the census reports written since the run began: `overwrite_risk` and `sync_errors` stayed 0, and `late_copies_in_busy`, `late_beside_unplanned`, `plan_left_planned` did not rise. Deltas, not absolute counts. Reports come once a minute, so a short step may have none: SKIP. | +| `qemu_log` | no new `command buffer failed` line in QEMU's log (stamp waits are counted in the detail) | +| `gpu_hang` | the hang flag did not appear | +| `kernel_gpu` | no kernel GPU error line since the step began (SKIP when the journal is not readable) | +| `counter` | `simfps` answered with a result for every simulator of the window and every animating simulator's surfaces gave a readable frame counter (a counter that answers nothing or an error fails the window, it is not a missing metric) | +| `memory` | the VM's own cgroup swapped no more than 50 MB (SKIP when the cgroup is unreadable: other projects' swapping on the host is not this run's); guest swap-outs fail it with `--fail-on-swap` | +| `run_*` | every scenario, after its body: `run_coherent_census`, `run_coherent_process`, `run_qemu_log`, `run_gpu_hang` and `run_kernel_gpu` since the run began, and `run_crash_reports` (a new crash report since setup, judged like `stability`'s). They fail the exit code of `idle`, `lifecycle`, `soak`, `ab` too. | +| `leak_*` | `lifecycle`: QEMU's memory (RSS + swap: RSS alone also rises when the host brings swapped pages back) and guest memory grow less than `--leak-rss-mb` / `--leak-guest-mb` per create/boot/delete cycle (least-squares slope); `soak`: less than `--soak-rss-mb-min` / `--soak-guest-mb-min` per minute. FAIL needs a steady growth (at least 4 samples, R2 >= 0.8); a slope above the bar without that is SKIP | +| `observer` | what looking costs: a second window of the first N reads nothing from the screen and injects nothing. PASS within 4%; beyond it INFO (never FAIL): the distinct-frame numbers are lower bounds. One lock of each picture cost between nothing and 31% of the screen frames in seven runs here (+2, -2.5, -5.6, -12.7, -20, -23, -31%; why it varies is not known); a second lock per picture halved them, so `simfps` reads strip and marker under one | + +## A/B discipline + +`ab` runs one comparison per lever and nothing else. + +```sh +macbox bench ab --knob live:group_batches=16 --n 3 --metric total_fps --threshold 0.04 +macbox bench ab --set-cmd 'macbox x on' --unset-cmd 'macbox x off' --n 3 --metric worst_p95 +``` + +The rule is printed and stored **before** the run. Simulators boot once; then the arms run interleaved A B B A in +one window (`--arm-settle` between them, `--secs` each). Pair 1 = the first A with the first B, pair 2 = the second +B with the second A. **ADOPT** only if the mean effect of the two pairs exceeds the threshold (default 4%, the +run-to-run noise on this stack; fixed on the command line before the run), **both pairs agree in sign**, and no tail +metric (default `worst_p95`, `worst_p99` for the fps metrics) worsens by more than `--tail-tol` (4%). Anything else, +including a gate failure or instability in any arm, is **CLOSE**, with no re-run. Metrics: `total_fps` (sum of the +simulators' distinct fps), `min_fps`, `worst_p95`, `worst_p99`, `input_p95`, `gpu_ms_per_frame`, `qemu_cores`. The +decision line (`AB : ADOPT|CLOSE | metric A a1/a2 B b1/b2 | effect, pairs, bar, reasons`) is a section of the result +file. A knob that needs a VM restart cannot be A/B'd here (the arms share one boot, by design). +`--knob live:KEY=VALUE` edits one line of the VM's live switch file and puts the file back whole on every exit. + +## The panel section + +The Device panel needs Simlock and T3 Code, which the bench does not depend on. `panel --panel-cmd 'cmd'` runs your +command with `BENCH_PANEL_SECS` set and reads one JSON object from its last stdout line: +`{"frames": [[t_seconds, counter], ...]}`: every frame a viewer received through the panel and the counter +GlassBench's strip showed in it (the strip survives H.264). The bench then applies the same distinct-frame metrics and +strip gate and judges `--panel-fps` / `--panel-p95` (55 / 25). Without a command the section is SKIP. + +## The result file + +`/-.json`, `schema` 1: `kind`, `id`, `created`, `params` (the command line), `env` (`build_id`, +`switches` read from the live switch file, `qemu_env` (REIMS_VGPU_* names and values), `guest` macOS / Xcode / +runtime / device type, `host` kernel / Mesa / CPU, free-form `tags`), `sections` (the PASS/FAIL lines), `gates` (the +run's roll-up), `data` (for `scale`: `mix`, `scale_x`, `steps[]` with `sims[]`, `system`, `stability`, `gates`, +`targets`, `fairness`, and `summary`). The file is written after every step, so a run that dies keeps its data. Nothing in it +names a host, an account or a path: strings are scrubbed on write (home paths to `~`, host and ssh names to ``, +IPv4 addresses to ``), and `publishable` says whether anything slipped through (`problems` lists it). + +`baselines/` holds results worth comparing against (see its README); `golden.json` holds the pixel hashes. + +## Limits of the measurements + +- The observer is not free: `simfps` polls at 2 kHz and `--strip` reads each new surface with the CPU, which also + touches Coherent's surfaces. `observer` measures that at the first N; the simulators' guest CPU does not include it + (the helper's own tree is excluded from every simulator's tree and reported as `observer_cpu_pct`), but the + observer still takes CPU the simulators would otherwise have. +- Probes run `simctl` in the guest every 2.5 s per simulator, a small steady load that agents also make. +- Windows are 40 s: slow drifts need `soak`; a stall rarer than once per window is invisible. +- GPU numbers need DRM fdinfo of QEMU's process to be readable by the user running the bench. + +## The tools + +The tools behind the numbers in [docs/performance.md](../../docs/performance.md). `macbox bench tools` copies and builds +them; by hand, inside macOS, once: ```sh tar -C bench -cf - . | ssh macbox 'mkdir -p ~/.macbox/bench && tar -C ~/.macbox/bench -xf -' @@ -10,12 +341,13 @@ ssh macbox 'cd ~/.macbox/bench && ./build.sh' | | | |---|---| -| `simfps [label] [--strip]` | counts the frames a simulator's screen produces, straight from CoreSimulator's framebuffer (no encoder in the way); one JSON line with fps and frame intervals. `--strip` also reads GlassBench's frame counter from each new picture (its synced reads slow the 2 kHz poll, so count frames without it) | -| `GlassBench.app` | the iOS load: a table scrolling by itself under Liquid Glass bars and a floating glass toolbar. It draws a **frame strip** at the top left: a 16-bit counter that advances every display-link tick, as 18 squares of 20 pt from (21, 120) pt (white and black references, then bits 0-15), large enough to survive H.264, so any viewer can tell unique frames from repeats and count the ones lost on the way. Its own tick intervals go to `Documents/pace.json` in its container every 5 s. `-nostrip` leaves the strip out | +| `simfps [label] [--strip]` | counts the frames a simulator's screen produces, straight from CoreSimulator's framebuffer (no encoder in the way); one JSON line with fps and frame intervals. `--strip` also reads GlassBench's frame counter from each new picture (its synced reads slow the 2 kHz poll, so count frames without it). Environment, used by the bench: `SIMFPS_SET` (a device set of its own), `SIMFPS_START_AT` (attach, then count from this unix second, so several simfps share one window), `SIMFPS_PROBE=x,y` (read this point's luma, in points of a 402 pt screen), `SIMFPS_STAMPS=path` (one line per new surface: unix seconds, counter or -1, probe luma or -1; the first line is the screen as it stands at the start) | +| `GlassBench.app` | the iOS load: a table scrolling by itself under Liquid Glass bars and a floating glass toolbar. It draws a **frame strip** at the top left: a 16-bit counter that advances every display-link tick, as 18 squares of 20 pt from (21, 120) pt (white and black references, then bits 0-15), large enough to survive H.264, so any viewer can tell unique frames from repeats and count the ones lost on the way. Its own tick intervals go to `Documents/pace.json` in its container every 5 s. `-nostrip` leaves the strip out, `-static` animates nothing (a fixed picture), `-probe` adds the input marker: a 30 pt square at (21, 160) pt that flips white/black on a tap or a `glassbench://toggle` URL | | `GenieBench.app` | the desktop load: a window that minimizes and restores, for counting the Genie animation's frames | | `scroll-local.html` | a page for Safari that scrolls itself under a blurred floating bar, with no network | -A run on the iOS Simulator, from the host: +A by-hand run on the iOS Simulator (a **fresh stock device**: about 1.8 GB of guest disk and a minutes-long first boot, not what an +agent's simulator is; `macbox bench scale` makes production simulators and is what to use for numbers): ```sh # a simulator of its own for the bench, so the pool is left alone (1x: the macbox 1x device type) diff --git a/src/bench/baselines/README.md b/src/bench/baselines/README.md new file mode 100644 index 0000000..ddc8322 --- /dev/null +++ b/src/bench/baselines/README.md @@ -0,0 +1,24 @@ +# Baselines + +Results worth comparing against, committed here so that a change can be held to them: + +```sh +macbox bench scale --n 1,2,3,4,5,6 --scale-x 1 --mix all-busy --out /tmp/r +macbox bench compare src/bench/baselines/.json /tmp/r/scale-.json +``` + +Format: a result file of the bench as it wrote it (`schema` 1, `"publishable": true`), nothing edited by hand. +Name it `--x-.json` (for example `scale-all-busy-1x-.json`), so a +comparison says what it compares. Only results with `"publishable": true` belong here: the bench scrubs host +names, accounts, addresses and paths on write, and `publishable` says whether anything slipped through. + +A baseline is only a baseline for the setup that made it (`env`: macOS and runtime, device type, host kernel and +Mesa, the live switches). `compare` prints the mix and scale of both and warns when they differ; it does not +compare across setups for you. Keep one baseline per (kind, mix, scale), replace it when the setup changes on +purpose, and say why in the commit. + +`golden.json` is different: the pixel hashes of GlassBench's static screen per (macOS, runtime, device type, size), +written by `macbox bench golden` and read by the `pixels` gate. Re-record it after a macOS or runtime update that +changes how text or glass renders; a hash that changed without such an update is a rendering regression. + +There are none yet: the first ones come from a run on the VM that has the build you want to hold. diff --git a/src/bench/build.sh b/src/bench/build.sh index b1279c3..25898a6 100755 --- a/src/bench/build.sh +++ b/src/bench/build.sh @@ -25,6 +25,10 @@ cat > "$APP/Info.plist" <<'PLIST' MinimumOSVersion26.0 UIDeviceFamily1 UILaunchScreen + CFBundleURLTypes + CFBundleURLNamedev.macbox.glassbench + CFBundleURLSchemesglassbench + UIApplicationSceneManifest UIApplicationSupportsMultipleScenes diff --git a/src/bench/guest/__init__.py b/src/bench/guest/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/bench/guest/benchguest.py b/src/bench/guest/benchguest.py new file mode 100644 index 0000000..4349529 --- /dev/null +++ b/src/bench/guest/benchguest.py @@ -0,0 +1,1366 @@ +#!/usr/bin/env python3 +"""benchguest: the bench's helper inside macOS (run by `macbox bench`; `xcrun python3`, standard library only, +Python 3.9 compatible). Every subcommand prints ONE JSON object on its last stdout line. + +Every simulator command goes through simctl() below, which always passes --set +and refuses a set without the bench's marker file: the pool and Simlock's devices are never touched. + + info | init-set | ensure-type | create | boot | prepare | settle | calibrate | measure | shots | + cycle | teardown | memcheck | build +""" +import argparse +import base64 +import hashlib +import json +import math +import os +import plistlib +import re +import resource +import shutil +import subprocess +import sys +import tempfile +import threading +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.dirname(HERE)) +BUNDLE_ID = "dev.macbox.glassbench" +MARKER = ".macbox-bench-set" +MANIFEST = ".bench-manifest.json" # what the bench made in its set: seeds (complete or not) and the devices it must clean up +LOCK = ".bench-run.lock" # present while a run owns the set; a leftover lock = a run that did not clean up +SEED_PREFIX = "bench-seed-" +# Things outside the device set that a bench simulator leaves behind or depends on. Module variables so the DRY stub +# can point them under its own state directory (nothing here is touched on the host). +OVERRIDES = "/private/var/tmp/com.apple.CoreSimulator.SimDevice" # + "." + udid: the Slim disables of one device +KEEP_DIR = os.path.expanduser("~/.sim-keep") # sim-idle-guard's keep files, one per udid +IDLE_GUARD = os.path.expanduser("~/.local/bin/sim-idle-guard") # `keep ` (max 240) +# Simlock's own source devices (read only: nothing is ever written to, booted, shut down or deleted in this set; the one +# command that names it is simlock_clone()) and the Slim tooling the fleet uses on every clone. +# Simlock's state directory is named by its install: give yours with --source-set or BENCH_SIMLOCK_SOURCES. +SOURCE_SET = os.path.expanduser(os.environ.get("BENCH_SIMLOCK_SOURCES") or "~/.simlock/devices/ios-sources") +SIMSLIM = os.environ.get("BENCH_SIMSLIM") or os.path.expanduser("~/go/bin/simslim") +SLIM_PROFILE = os.environ.get("BENCH_SLIM_PROFILE") # else the one profile in SLIM_DIR +SLIM_DIR = os.path.expanduser("~/.config/simslim") +MIN_SLIM_SERVICES = 100 # a Slim device has 168 services off today +PROBE_POINT = "36,175" # the input marker's centre, in points (GlassBench -probe) +SET = None +STUB = os.environ.get("BENCHGUEST_STUB") +CALLS = [] + + +# --- primitives (replaced by benchguest_stub.py under BENCHGUEST_STUB) --------------------------------- +def run(argv, timeout=120, env=None): + """(returncode, stdout, stderr, seconds). A timeout kills the process and returns rc 124.""" + t0 = time.time() + try: + p = subprocess.run(argv, capture_output=True, text=True, timeout=timeout, env=env) + return p.returncode, p.stdout, p.stderr, time.time() - t0 + except subprocess.TimeoutExpired as e: + return 124, (e.stdout or b"").decode() if isinstance(e.stdout, bytes) else (e.stdout or ""), "timeout", time.time() - t0 + except OSError as e: + return 127, "", str(e), time.time() - t0 + + +def spawn_simfps(udid, secs, label, strip, start_at, raw_path, probe=True): + """Starts simfps; returns an object whose .wait(timeout) -> (json dict or None, error or None). + strip and probe are what it reads from every surface; with neither it locks no surface (an unviewed screen).""" + env = dict(os.environ, SIMFPS_SET=SET, SIMFPS_START_AT="%.3f" % start_at, SIMFPS_STAMPS=raw_path) + env.pop("SIMFPS_PROBE", None) + if probe: + env["SIMFPS_PROBE"] = PROBE_POINT + argv = [os.path.join(HERE, "..", "simfps"), udid, str(secs), label] + (["--strip"] if strip else []) + p = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=env) + + class H: + def wait(self, timeout): + try: + out, err = p.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + p.kill() + p.communicate() + return None, "simfps hung (killed after %ds)" % timeout + try: + return json.loads(out.strip().splitlines()[-1]), None + except (ValueError, IndexError): + return None, (err or out or "no output").strip()[-200:] + + def kill(self): + try: + p.kill() + except OSError: + pass + return H() + + +def ps_text(): + return run(["ps", "-axww", "-o", "pid=,ppid=,rss=,time=,command="], 30)[1] + + +def vmstat_text(): + return run(["vm_stat"], 15)[1] + + +def sysctl(name): + return run(["sysctl", "-n", name], 15)[1].strip() + + +def now(): + return time.time() + + + +# --- simctl ---------------------------------------------------------------------------------------- +class GuestError(Exception): + pass + + +def check_set(): + if not SET: + raise GuestError("--set is required") + if not os.path.exists(os.path.join(SET, MARKER)): + raise GuestError("not a bench device set (no %s): refusing" % MARKER) + + +def simctl(*args, timeout=180): + """xcrun simctl --set ...; the only way this file starts simctl.""" + check_set() + argv = ["xcrun", "simctl", "--set", SET] + [str(a) for a in args] + CALLS.append(argv) + return run(argv, timeout) + + +def simctl_json(*args): + rc, out, err, _ = simctl(*args, "-j") + if rc != 0: + raise GuestError("simctl %s: %s" % (" ".join(map(str, args)), err.strip()[:200])) + return json.loads(out) + + +def devices(): + out = [] + for rt, devs in simctl_json("list", "devices").get("devices", {}).items(): + for d in devs: + d = dict(d) + d["runtime"] = rt + out.append(d) + return out + + +def is_seed(d): + return d.get("name", "").startswith(SEED_PREFIX) + + +def overrides_dir(udid): + return "%s.%s" % (OVERRIDES, udid) + + +def load_manifest(): + try: + with open(os.path.join(SET, MANIFEST)) as f: + m = json.load(f) + except (OSError, ValueError): + m = {} + m.setdefault("seeds", {}) + m.setdefault("devices", {}) + return m + + +def save_manifest(m): + p = os.path.join(SET, MANIFEST) + with open(p + ".tmp", "w") as f: + json.dump(m, f, indent=1, sort_keys=True) + os.replace(p + ".tmp", p) + + +def idle_guard_keep(udid, minutes): + """Registers a simulator with the Mac's sim-idle-guard (it shuts down any simulator nobody used for 15 minutes, in + every device set). {"state": "kept"|"absent"|"failed", "minutes", "err"}. 'absent' is recorded by the caller and the + run goes on; 'failed' (the tool is there and refused) is an error: the simulator would be shut down under the run.""" + minutes = max(1, min(int(math.ceil(minutes)), 240)) + if not (os.path.isfile(IDLE_GUARD) and os.access(IDLE_GUARD, os.X_OK)): + return {"state": "absent", "minutes": 0, "err": None} + rc, out, err, _ = run([IDLE_GUARD, "keep", udid, str(minutes)], 30) + if rc != 0: + return {"state": "failed", "minutes": minutes, "err": (err or out).strip()[-120:]} + return {"state": "kept", "minutes": minutes, "err": None} + + +def keep_or_fail(udid, minutes): + k = idle_guard_keep(udid, minutes) + if k["state"] == "failed": + raise GuestError("sim-idle-guard keep %s failed: %s (the Mac would shut the simulator down under the run)" % (udid, k["err"])) + return k + + +def drop_keep(udid): + """Removes ~/.sim-keep/; returns None or what went wrong (a missing file is fine).""" + try: + os.remove(os.path.join(KEEP_DIR, udid)) + except FileNotFoundError: + return None + except OSError as e: + return "keep file of %s: %s" % (udid, e) + return None + + +def drop_overrides(udid): + d = overrides_dir(udid) + shutil.rmtree(d, ignore_errors=True) + return "overrides folder of %s still there" % udid if os.path.exists(d) else None + + +# --- parsers (unit-tested) ------------------------------------------------------------------------- +def parse_vm_stat(text): + """vm_stat -> {"page": bytes, "free": pages, ...}. Keys lower-cased, spaces to underscores.""" + d = {} + m = re.search(r"page size of (\d+) bytes", text) + d["page"] = int(m.group(1)) if m else (4096 if os.uname()[4] == "x86_64" else 16384) + for line in text.splitlines(): + m = re.match(r"^\"?([A-Za-z ()-]+?)\"?:\s+(\d+)\.?\s*$", line) + if m: + d[re.sub(r"[^a-z0-9]+", "_", m.group(1).lower()).strip("_")] = int(m.group(2)) + return d + + +def mem_summary(vm): + """The fields a report needs, in MB, from parse_vm_stat().""" + p = vm.get("page", 16384) / 1048576.0 + g = lambda k: vm.get(k, 0) + used = (g("pages_active") + g("pages_wired_down") + g("pages_occupied_by_compressor")) * p + return {"free_mb": round((g("pages_free") + g("pages_speculative")) * p, 1), "used_mb": round(used, 1), + # what the guest can hand out without swapping: free, speculative, inactive (file cache) and purgeable pages + "avail_mb": round((g("pages_free") + g("pages_speculative") + g("pages_inactive") + g("pages_purgeable")) * p, 1), + "compressed_mb": round(g("pages_occupied_by_compressor") * p, 1), + "swapins": g("swapins"), "swapouts": g("swapouts"), "pageins": g("pageins"), "pageouts": g("pageouts")} + + +def parse_cputime(s): + """ps TIME: [d-][h:]m:s[.cc] -> seconds""" + days = 0 + if "-" in s: + d, s = s.split("-", 1) + days = int(d) + parts = s.split(":") + secs = 0.0 + for p in parts: + secs = secs * 60 + float(p) + return days * 86400 + secs + + +def parse_ps(text): + """[{pid, ppid, rss_kb, cpu_s, cmd}]""" + out = [] + for line in text.splitlines(): + p = line.split(None, 4) + if len(p) < 5: + continue + try: + out.append({"pid": int(p[0]), "ppid": int(p[1]), "rss_kb": int(p[2]), "cpu_s": parse_cputime(p[3]), "cmd": p[4]}) + except ValueError: + pass + return out + + +def own_pids(procs, me=None): + """The helper itself, its ancestors (ssh, shells) and everything it started (simfps, simctl, ps): the observer.""" + me = os.getpid() if me is None else me + by_pid = {p["pid"]: p for p in procs} + by_ppid = {} + for p in procs: + by_ppid.setdefault(p["ppid"], []).append(p["pid"]) + mine, stack = {me}, [me] + while stack: + for c in by_ppid.get(stack.pop(), []): + if c not in mine: + mine.add(c) + stack.append(c) + q = me + while q in by_pid and by_pid[q]["ppid"] > 1: + q = by_pid[q]["ppid"] + mine.add(q) + return mine + + +def sim_trees(procs, udids, exclude=()): + """{udid: set of pids}: processes whose command line names the simulator's UDID (launchd_sim, SimMetalHost, + ...) and every descendant of them, never the observer (`exclude`: the helper's own tree, which names every + UDID on its command line). A process that names no simulator is nobody's.""" + by_ppid = {} + for p in procs: + by_ppid.setdefault(p["ppid"], []).append(p["pid"]) + out = {} + for u in udids: + roots = [p["pid"] for p in procs if u in p["cmd"] and p["pid"] not in exclude] + seen, stack = set(roots), list(roots) + while stack: + for c in by_ppid.get(stack.pop(), []): + if c not in seen and c not in exclude: + seen.add(c) + stack.append(c) + out[u] = seen + return out + + +def attribute(before, after, udids, secs, exclude=()): + """Per simulator and in total: CPU % of one core over the window (from the processes' CPU time deltas) and RSS MB + at the end. A process that started or ended inside the window counts from its first sample or not at all: + short-lived processes are undercounted (declared in the README).""" + b = {p["pid"]: p for p in before} + trees = sim_trees(after, udids, exclude) + per = {} + for u in udids: + cpu = sum(max(0.0, p["cpu_s"] - b[p["pid"]]["cpu_s"]) for p in after if p["pid"] in trees[u] and p["pid"] in b) + rss = sum(p["rss_kb"] for p in after if p["pid"] in trees[u]) / 1024.0 + per[u] = {"cpu_pct": round(100 * cpu / secs, 1), "rss_mb": round(rss, 1), "procs": len(trees[u])} + obs = sum(max(0.0, p["cpu_s"] - b[p["pid"]]["cpu_s"]) for p in after if p["pid"] in b and p["pid"] in exclude) + tot = sum(max(0.0, p["cpu_s"] - b[p["pid"]]["cpu_s"]) for p in after if p["pid"] in b and p["pid"] not in exclude) + mine = set().union(*trees.values()) if trees else set() + sims_cpu = sum(max(0.0, p["cpu_s"] - b[p["pid"]]["cpu_s"]) for p in after if p["pid"] in mine and p["pid"] in b) + return per, {"observer_cpu_pct": round(100 * obs / secs, 1), "total_cpu_pct": round(100 * tot / secs, 1), "sims_cpu_pct": round(100 * sims_cpu / secs, 1), + "other_cpu_pct": round(100 * (tot - sims_cpu) / secs, 1), + "sims_rss_mb": round(sum(v["rss_mb"] for v in per.values()), 1)} + + +def pressure_level(): + s = sysctl("kern.memorystatus_vm_pressure_level") + return int(s) if s.isdigit() else None + + +def parse_pace(text): + """GlassBench's Documents/pace.json (one JSON object per line) -> [(at, counter, fps, ticks)]""" + out = [] + for line in text.splitlines(): + try: + d = json.loads(line) + out.append((float(d["at"]), int(d["counter"]), float(d.get("fps", 0)), int(d.get("ticks", 0)))) + except (ValueError, KeyError, TypeError): + pass + return out + + +def read_pace(udid): + rc, out, _, _ = simctl("get_app_container", udid, BUNDLE_ID, "data") + if rc != 0: + return [] + try: + with open(os.path.join(out.strip(), "Documents", "pace.json")) as f: + return parse_pace(f.read()) + except OSError: + return [] + + +def list_ips(): + names = set() + for d in (os.path.expanduser("~/Library/Logs/DiagnosticReports"), "/Library/Logs/DiagnosticReports"): + try: + for n in os.listdir(d): + if n.endswith((".ips", ".crash")): + names.add(n) + except OSError: + pass + return names + + +# --- injection ------------------------------------------------------------------------------------- +def inject(method, udid, state, cmd_tpl=None): + """One input event; returns the command's rc. state: a per-simulator dict for alternating methods.""" + if method == "notify": + return simctl("spawn", udid, "notifyutil", "-p", "dev.macbox.glassbench.toggle", timeout=20)[0] + if method == "openurl": + return simctl("openurl", udid, "glassbench://toggle", timeout=20)[0] + if method == "appearance": + state["dark"] = not state.get("dark", False) + return simctl("ui", udid, "appearance", "dark" if state["dark"] else "light", timeout=20)[0] + if method == "cmd" and cmd_tpl: + return run(["/bin/sh", "-c", cmd_tpl.replace("{udid}", udid)], 20)[0] + return 2 + + +def probe_loop(udid, method, cmd_tpl, t_first, period, t_stop, out, stop_evt): + state = {} + t = t_first + while t < t_stop and not stop_evt.is_set(): + d = t - now() + if d > 0: + stop_evt.wait(d) + if stop_evt.is_set(): + break + t0 = now() + rc = inject(method, udid, state, cmd_tpl) + out.append([t0, now(), rc]) + t += period + + +# --- actions of the agent mix ---------------------------------------------------------------------- +def agent_loop(udid, idx, n, t_go, t_stop, period, out, stop_evt): + """launch app -> scroll (the app animates by itself) -> screenshot -> terminate -> idle, staggered by sim.""" + t = t_go + (idx * period / max(n, 1)) + tmp = tempfile.mkdtemp(prefix="benchshot") + try: + while t + period <= t_stop + 0.5 and not stop_evt.is_set(): + d = t - now() + if d > 0 and stop_evt.wait(d): + break + t0 = now() + rc = simctl("launch", udid, BUNDLE_ID, "-probe", timeout=60)[0] + out.append({"op": "launch", "t0": t0, "t1": now(), "rc": rc}) + stop_evt.wait(0.45 * period) + t0 = now() + rc = simctl("io", udid, "screenshot", "--type=png", os.path.join(tmp, "s.png"), timeout=60)[0] + out.append({"op": "screenshot", "t0": t0, "t1": now(), "rc": rc}) + stop_evt.wait(0.15 * period) + t0 = now() + rc = simctl("terminate", udid, BUNDLE_ID, timeout=30)[0] + out.append({"op": "terminate", "t0": t0, "t1": now(), "rc": rc}) + t += period + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +# --- subcommands ----------------------------------------------------------------------------------- +def out_json(obj): + sys.stdout.write(json.dumps(obj) + "\n") + sys.stdout.flush() + + +def _has_profile(): + try: + return os.path.isfile(resolve_profile()) + except GuestError: + return False + + +def cmd_info(a): + sw = run(["sw_vers"], 10)[1] + ver = re.search(r"ProductVersion:\s*(\S+)", sw) + xc = run(["xcodebuild", "-version"], 30)[1].replace("\n", " ").strip() + rts = [] + try: + rts = [{"name": r.get("name"), "version": r.get("version"), "build": r.get("buildversion")} + for r in json.loads(run(["xcrun", "simctl", "list", "runtimes", "-j"], 30)[1]).get("runtimes", []) + if r.get("isAvailable", True)] + except ValueError: + pass + vm = parse_vm_stat(vmstat_text()) + out_json({"macos": ver.group(1) if ver else None, "xcode": xc, "runtimes": rts, "arch": run(["uname", "-m"], 5)[1].strip(), + "cpus": sysctl("hw.ncpu"), "mem_mb": int(sysctl("hw.memsize") or 0) // 1048576, "mem": mem_summary(vm), + "simlock_types": source_types(SOURCE_SET), "has_simslim": os.path.isfile(SIMSLIM) and os.access(SIMSLIM, os.X_OK), + "has_slim_profile": _has_profile(), "has_idle_guard": os.path.isfile(IDLE_GUARD) and os.access(IDLE_GUARD, os.X_OK), + "has_simfps": bool(STUB) or os.path.exists(os.path.join(HERE, "..", "simfps")), + "has_app": bool(STUB) or os.path.isdir(os.path.join(HERE, "..", "GlassBench.app")), "stub": bool(STUB)}) + + +def cmd_init_set(a): + global SET + os.makedirs(a.set, exist_ok=True) + with open(os.path.join(a.set, MARKER), "w") as f: + f.write("macbox bench device set\n") + SET = a.set + out_json({"ok": True, "devices": len(devices())}) + + +def stock_pro_type(): + """(runtime id, runtime version, 'iPhone NN Pro' type name, id, bundle): macbox-guest's choice of the stock type.""" + best = None + rts = json.loads(run(["xcrun", "simctl", "list", "runtimes", "-j"], 30)[1]).get("runtimes", []) + ios = [r for r in rts if r.get("platform", "iOS") == "iOS" and r.get("isAvailable", True)] + if not ios: + raise GuestError("no iOS Simulator runtime") + ios.sort(key=lambda r: [int(x) for x in re.findall(r"\d+", r.get("version", "0"))]) + rt = ios[-1] + for d in rt.get("supportedDeviceTypes", []): + m = re.match(r"^iPhone (\d+) Pro$", d["name"]) + if m and d["identifier"].startswith("com.apple.") and (best is None or int(m.group(1)) > best[0]): + best = (int(m.group(1)), d) + if not best: + raise GuestError("the runtime supports no 'iPhone NN Pro'") + d = best[1] + return rt["identifier"], rt.get("version"), d["name"], d["identifier"], d.get("bundlePath", "") + + +def cmd_ensure_type(a): + """The device type for a pixel scale: the stock iPhone NN Pro at 3x, a macbox type (the same one `macbox pool` + makes, same identifier, reused when it exists) at 1x and 2x. Making a NEW type needs CoreSimulatorService to + restart, which shuts down every simulator: done only with --allow-restart and only when none is booted.""" + rt, rtv, name, stock_id, bundle = stock_pro_type() + scale = int(a.scale) + res = {"runtime": rt, "runtime_version": rtv, "stock_name": name, "scale": scale} + if scale == 3: + res.update({"type": stock_id, "created": False, "needs_restart": False}) + return out_json(res) + tid = "dev.macbox.CoreSimulator.SimDeviceType.%s-%dx" % (stock_id.rsplit(".", 1)[-1], scale) + known = {t["identifier"] for t in json.loads(run(["xcrun", "simctl", "list", "devicetypes", "-j"], 30)[1]).get("devicetypes", [])} + if tid in known: + res.update({"type": tid, "created": False, "needs_restart": False}) + return out_json(res) + types_dir = os.path.expanduser("~/Library/Developer/CoreSimulator/Profiles/DeviceTypes") + dest = os.path.join(types_dir, "%s (%dx).simdevicetype" % (os.path.basename(bundle)[:-len(".simdevicetype")], scale)) + if not os.path.isdir(bundle): + raise GuestError("no device type bundle at the stock type's path") + os.makedirs(types_dir, exist_ok=True) + tmp = dest + ".tmp" + shutil.rmtree(tmp, ignore_errors=True) + shutil.copytree(bundle, tmp, symlinks=True) + pp = os.path.join(tmp, "Contents", "Resources", "profile.plist") + with open(pp, "rb") as f: + prof = plistlib.load(f) + s = int(prof["mainScreenScale"]) + pw, ph = int(prof["mainScreenWidth"]) // s, int(prof["mainScreenHeight"]) // s + prof["mainScreenScale"] = float(scale) if isinstance(prof["mainScreenScale"], float) else scale + prof["mainScreenWidth"], prof["mainScreenHeight"] = pw * scale, ph * scale + prof["mainScreenWidthDPI"] = (int(prof["mainScreenWidthDPI"]) * scale + s // 2) // s + prof["mainScreenHeightDPI"] = (int(prof["mainScreenHeightDPI"]) * scale + s // 2) // s + prof.pop("createByDefaultForRuntimeVersions", None) # else CoreSimulator would make and pair devices of it itself + prof.pop("pairByDefaultForRuntimeVersions", None) + with open(pp, "wb") as f: + plistlib.dump(prof, f) + ip = os.path.join(tmp, "Contents", "Info.plist") + with open(ip, "rb") as f: + info = plistlib.load(f) + info["CFBundleIdentifier"] = tid + info["CFBundleName"] = "%s (%dx)" % (name, scale) + with open(ip, "wb") as f: + plistlib.dump(info, f) + shutil.rmtree(dest, ignore_errors=True) + os.rename(tmp, dest) + res.update({"type": tid, "created": True, "needs_restart": True}) + if a.allow_restart: + if count_launchd_sim() or [d for d in devices() if d.get("state") == "Booted"]: + res["restart"] = "refused: a simulator is running (in any device set)" + else: + run(["pkill", "-f", "CoreSimulatorService"], 10) + time.sleep(3) + res["restart"] = "done" + res["needs_restart"] = False + out_json(res) + + +def cmd_create(a): + """A fresh STOCK device (about 1.8 GB of guest disk each): only behind --stock-devices on the host.""" + made = [] + man = load_manifest() + for i in range(a.start, a.start + a.n): + name = "%s-%d" % (a.prefix, i) + rc, out, err, secs = simctl("create", name, a.type, a.runtime) + if rc != 0: + raise GuestError("create %s: %s" % (name, err.strip()[:200])) + udid = out.strip().splitlines()[-1] + man["devices"][udid] = {"name": name, "kind": "stock"} + save_manifest(man) + k = keep_or_fail(udid, a.keep_min) + made.append({"idx": i, "udid": udid, "create_s": round(secs, 2), "idle_guard": k["state"]}) + out_json({"sims": made}) + + +def simlock_clone(src_set, src_udid, name): + """THE one command that names Simlock's device set: `simctl --set clone `. + It reads the source and writes into the bench set; Simlock's set is never written, booted, shut down or deleted.""" + check_set() + argv = ["xcrun", "simctl", "--set", src_set, "clone", src_udid, name, SET] + CALLS.append(argv) + return run(argv, 900) + + +UUID = re.compile(r"^[0-9A-Fa-f]{8}(-[0-9A-Fa-f]{4}){3}-[0-9A-Fa-f]{12}$") + + +def find_source(type_id, src_set): + """Simlock's source device of this device type: (udid, device.plist dict), or (None, why). Reads the source folders' + device.plist only (never config.json); prefers a source that is shut down.""" + try: + names = sorted(n for n in os.listdir(src_set) if UUID.match(n)) + except OSError: + return None, ("Simlock's source device set was not found at the configured path (Simlock is optional): give the " + "devices/ios-sources folder of its state directory with --source-set or BENCH_SIMLOCK_SOURCES") + found, types = [], set() + for n in names: + try: + with open(os.path.join(src_set, n, "device.plist"), "rb") as f: + pl = plistlib.load(f) + except (OSError, ValueError, plistlib.InvalidFileException): + continue + types.add(str(pl.get("deviceType")).rsplit(".", 1)[-1]) + if pl.get("deviceType") == type_id: + found.append((0 if pl.get("state") == 1 else 1, n, pl)) + if not found: + return None, "Simlock has no source device of type %s (it has: %s)" % (type_id.rsplit(".", 1)[-1], ", ".join(sorted(types)) or "none") + found.sort(key=lambda x: (x[0], x[1])) + return found[0][1], found[0][2] + + +def source_types(src_set): + """The device types Simlock has a source device for (names only), [] when Simlock's set is not there.""" + out = set() + try: + for n in os.listdir(src_set): + try: + with open(os.path.join(src_set, n, "device.plist"), "rb") as f: + out.add(str(plistlib.load(f).get("deviceType")).rsplit(".", 1)[-1]) + except (OSError, ValueError, plistlib.InvalidFileException): + pass + except OSError: + pass + return sorted(out) + + +def runtime_version(runtime_id): + try: + for r in json.loads(run(["xcrun", "simctl", "list", "runtimes", "-j"], 30)[1]).get("runtimes", []): + if r.get("identifier") == runtime_id: + return r.get("version") + except ValueError: + pass + return None + + +def top_text(top): + return ", ".join("%s %g%%" % (t["cmd"], t["cpu_pct"]) for t in top[:5]) or "nothing above 0%" + + +def delete_device(udid): + """A device of the bench's set: shut down, delete, remove what lives outside it (and from the manifest); returns problems.""" + probs = [] + simctl("shutdown", udid, timeout=120) + rc, _, err, _ = simctl("delete", udid, timeout=120) + if rc != 0: + probs.append("delete %s: %s" % (udid, err.strip()[:100])) + for e in (drop_keep(udid), drop_overrides(udid)): + if e: + probs.append(e) + man = load_manifest() + man["devices"].pop(udid, None) + if not probs: + man["seeds"] = {t: x for t, x in man["seeds"].items() if x["udid"] != udid} + save_manifest(man) + return probs + + +def cmd_seed(a): + """ONE settled production seed per device type in the bench's set: a clone of Simlock's own source device of that type + (widgets trimmed, as production), one boot, simslim with the fleet's profile (Simlock's sources are not Slim), the bench + app, a real settle, shutdown. Kept (shut down) between runs and reused; every bench simulator is a clone of it. A seed + that was never completed is deleted again, never left to be reused.""" + tid = a.type + name = SEED_PREFIX + tid.rsplit(".", 1)[-1] + man = load_manifest() + cur = man["seeds"].get(tid) + here = {d["udid"]: d for d in devices()} + stamp = app_stamp() + if cur and (a.rebuild or cur["udid"] not in here): + if cur["udid"] in here: + probs = delete_device(cur["udid"]) + if probs: + raise GuestError("the old seed could not be deleted: %s" % "; ".join(probs)) + man = load_manifest() + man["seeds"].pop(tid, None) + save_manifest(man) + cur = None + if cur: + if not cur.get("complete"): + raise GuestError("the seed in the bench's set was never completed: macbox bench cleanup --seeds") + u = cur["udid"] + if here[u].get("state") != "Shutdown": + simctl("shutdown", u, timeout=120) + n = slim_count(u) + why = ("only %d services off" % n) if n < MIN_SLIM_SERVICES else None + if not why and cur.get("app_stamp") != stamp: + why = "the bench app was rebuilt since (GlassBench differs)" + if why: + raise GuestError("the existing seed cannot be reused (%s): rebuild it with --rebuild-seed" % why) + return out_json(dict(cur, seed=u, reused=True, services_off=n, source_type=tid.rsplit(".", 1)[-1], icons=os.path.isdir(icons_path(u)))) + src, pl = find_source(tid, os.path.expanduser(a.source_set or SOURCE_SET)) + if src is None: + return out_json({"seed": None, "no_source": pl, "udid": None}) + t0 = now() + rc, out, err, _ = simlock_clone(os.path.expanduser(a.source_set or SOURCE_SET), src, name) + if rc != 0: + raise GuestError("simctl clone out of Simlock's source device failed: %s" % (err or out).strip()[-200:]) + u = out.strip().splitlines()[-1] + runtime = pl.get("runtime") + man["seeds"][tid] = {"udid": u, "name": name, "complete": False, "type": tid, "runtime": runtime} + man["devices"][u] = {"name": name, "kind": "seed"} + save_manifest(man) # written before anything else is done to it: a crash leaves an incomplete seed, which is refused + try: + keep_or_fail(u, a.keep_min) + if simctl("boot", u, timeout=300)[0] != 0: + raise GuestError("the seed did not boot") + if simctl("bootstatus", u, timeout=600)[0] != 0: + raise GuestError("the seed never finished booting (bootstatus)") + sl = os.path.expanduser(a.simslim or SIMSLIM) + prof = resolve_profile(a.slim_profile) + # Simlock's source devices are widget-trimmed but NOT Slim: simslim shuts the device down, rewrites its services and reboots it + rc, out, err, secs = run([sl, "--set", SET, "on", u, "--profile", prof], 900) + if rc != 0: + raise GuestError("simslim failed on the seed (rc %s): %s" % (rc, (err or out).strip()[-200:])) + if simctl("bootstatus", u, timeout=600)[0] != 0: + raise GuestError("the seed never finished booting after simslim (bootstatus)") + n = slim_count(u) + if n < MIN_SLIM_SERVICES: + raise GuestError("the seed is not Slim after simslim: %d services off, need >= %d" % (n, MIN_SLIM_SERVICES)) + app = os.path.join(HERE, "..", "GlassBench.app") + if simctl("install", u, app, timeout=180)[0] != 0: + raise GuestError("installing the bench app in the seed failed") + if simctl("get_app_container", u, BUNDLE_ID, timeout=30)[0] != 0: + raise GuestError("the bench app is not installed in the seed after install") + st = settle_wait(a.settle_max, a.bg_pct, a.step, a.consecutive) + icons = os.path.isdir(icons_path(u)) # recorded, never a gate: whether a widget-trimmed seed grows this folder is unverified + if not st["settled"]: + raise GuestError("the seed did not settle in %g s (background %.0f%% of a core; top: %s)" % ( + a.settle_max, st["bg_pct"], top_text(st["top"]))) + if simctl("shutdown", u, timeout=120)[0] != 0: + raise GuestError("the seed did not shut down") + except BaseException: + delete_device(u) + man = load_manifest() + man["seeds"].pop(tid, None) + save_manifest(man) + raise + drop_keep(u) + res = {"seed": u, "udid": u, "name": name, "type": tid, "runtime": runtime, "runtime_version": runtime_version(runtime), "complete": True, "app_stamp": stamp, + "source_type": tid.rsplit(".", 1)[-1], "reused": False, "made_s": round(now() - t0, 1), "services_off": n, "settle_s": st["waited_s"], "icons": icons, + "settle_bg_pct": st["bg_pct"], "slimmed_s": round(secs, 1)} + man = load_manifest() + man["seeds"][tid] = {k: res[k] for k in ("udid", "name", "type", "runtime", "runtime_version", "complete", "app_stamp", "made_s", "services_off")} + save_manifest(man) + out_json(res) + + +def cmd_clone(a): + """Bench simulators = APFS clones of the seed (no disk cost) with the seed's Slim disables copied to each UDID's folder + before its first boot.""" + here = {d["udid"]: d for d in devices()} + if a.seed not in here or not is_seed(here[a.seed]): + raise GuestError("the seed is not in the bench's set") + if here[a.seed].get("state") != "Shutdown": + simctl("shutdown", a.seed, timeout=120) # a clone is taken of a shut-down device + man = load_manifest() + made = [] + for i in range(a.start, a.start + a.n): + name = "%s-%d" % (a.prefix, i) + rc, out, err, secs = simctl("clone", a.seed, name, timeout=300) + if rc != 0: + raise GuestError("clone of the seed %s: %s" % (name, (err or out).strip()[:200])) + u = out.strip().splitlines()[-1] + man["devices"][u] = {"name": name, "kind": "clone"} + save_manifest(man) + t = now() + off = copy_overrides(a.seed, u) + k = keep_or_fail(u, a.keep_min) + made.append({"idx": i, "udid": u, "clone_s": round(secs + now() - t, 2), "services_off": off, "idle_guard": k["state"]}) + out_json({"sims": made}) + + +def _boot_one(udid, res, i): + t0 = now() + rc, _, err, _ = simctl("boot", udid, timeout=300) + t1 = now() + ok = rc == 0 or "current state: Booted" in err + rc2 = simctl("bootstatus", udid, timeout=600)[0] if ok else rc + res[i] = {"udid": udid, "ok": ok and rc2 == 0, "boot_cmd_s": round(t1 - t0, 2), "usable_s": round(now() - t0, 2), + "err": None if ok and rc2 == 0 else err.strip()[-120:]} + + +def slim_count(udid): + """Services Slim keeps disabled for a simulator: `=> true` lines of its disabled.plist (what the fleet's check counts; + 168 today). 0 = stock or no file.""" + f = os.path.join(overrides_dir(udid), "disabled.plist") + if not os.path.exists(f): + return 0 + return run(["plutil", "-p", f], 15)[1].count("=> true") + + +def resolve_profile(arg=None): + """simslim's profile: --slim-profile, else $BENCH_SLIM_PROFILE, else the only *.json in ~/.config/simslim (ambiguous or + none: an error that says to pass --slim-profile).""" + chosen = arg or SLIM_PROFILE + if chosen: + return os.path.expanduser(chosen) + try: + found = sorted(n for n in os.listdir(SLIM_DIR) if n.endswith(".json")) + except OSError: + found = [] + if len(found) != 1: + raise GuestError("no simslim profile: %s; pass --slim-profile (or set BENCH_SLIM_PROFILE)" % ( + "%d profiles in the simslim config folder" % len(found) if found else "none in the simslim config folder")) + return os.path.join(SLIM_DIR, found[0]) + + +def copy_overrides(seed_udid, udid, min_services=MIN_SLIM_SERVICES): + """The Slim disables live OUTSIDE the device, keyed by UDID: a clone boots stock unless the seed's file is in its own + folder (mode 0700) before its first boot. Fails when the seed has none or the copy did not take.""" + n = slim_count(seed_udid) + if n < min_services: + raise GuestError("the seed's disabled.plist has %d services off, a production simulator needs >= %d: rebuild the seed (macbox bench seed --rebuild-seed)" % (n, min_services)) + d = overrides_dir(udid) + os.makedirs(d, exist_ok=True) + os.chmod(d, 0o700) + shutil.copyfile(os.path.join(overrides_dir(seed_udid), "disabled.plist"), os.path.join(d, "disabled.plist")) + got = slim_count(udid) + if got != n: + raise GuestError("the overrides copy for %s holds %d services off, the seed's %d: it would boot stock" % (udid, got, n)) + return got + + +def icons_path(udid): + return os.path.join(SET, udid, "data", "Library", "chronod", "icons") # exists once the widgets have settled + + +def app_stamp(): + """A hash of the installed bench app (GlassBench.app of this checkout): a seed made with another build is stale.""" + root = os.path.join(HERE, "..", "GlassBench.app") + h = hashlib.sha256() + for dp, dn, fn in os.walk(root): + dn.sort() + for f in sorted(fn): + p = os.path.join(dp, f) + h.update(os.path.relpath(p, root).encode()) + try: + with open(p, "rb") as fh: + h.update(fh.read()) + except OSError: + pass + return h.hexdigest()[:16] + + +def cmd_boot(a): + us = a.udids.split(",") + res = [None] * len(us) + t0 = now() + if a.parallel: + th = [threading.Thread(target=_boot_one, args=(u, res, i)) for i, u in enumerate(us)] + [t.start() for t in th] + [t.join(700) for t in th] + else: + for i, u in enumerate(us): + _boot_one(u, res, i) + out_json({"sims": [r or {"ok": False, "err": "no result"} for r in res], "all_s": round(now() - t0, 2)}) + + +def cmd_prepare(a): + """status bar fixed, app installed, launched with the arguments of the simulator's role.""" + us, roles = a.udids.split(","), a.roles.split(",") + app = os.path.join(HERE, "..", "GlassBench.app") + res = [] + for u, role in zip(us, roles): + t0 = now() + rc_i = simctl("install", u, app, timeout=120)[0] if a.install else 0 + if a.check_app and simctl("get_app_container", u, BUNDLE_ID, timeout=30)[0] != 0: + raise GuestError("the bench app is not installed in %s: the seed is stale, rebuild it (--rebuild-seed)" % u) + simctl("status_bar", u, "override", "--time", "9:41", "--batteryState", "charged", "--batteryLevel", "100", + "--cellularBars", "4", "--wifiBars", "3", "--operatorName", "", timeout=30) + if a.quiesce: + simctl("spawn", u, "launchctl", "disable", "system/com.apple.mediaanalysisd", timeout=30) + simctl("spawn", u, "launchctl", "bootout", "system/com.apple.mediaanalysisd", timeout=30) + simctl("ui", u, "appearance", "light", timeout=30) + simctl("terminate", u, BUNDLE_ID, timeout=30) + args = ["-probe"] + (["-static"] if role == "static" else []) + rc_l = 0 if role in ("agent", "idle") else simctl("launch", u, BUNDLE_ID, *args, timeout=60)[0] + res.append({"udid": u, "role": role, "install_rc": rc_i, "launch_rc": rc_l, "s": round(now() - t0, 2)}) + out_json({"sims": res}) + + +def parse_name_cpu(text): + return text + + +# the load itself and its observers, matched against the WHOLE command (a runtime path contains spaces) +SETTLE_SKIP = re.compile(r"(^|/)(SimMetalHost|backboardd|GlassBench|SimRenderServer|kernel_task|simfps|WindowServer|python3?|Python|ps|top|ssh|sshd)( |$)") + + +def bg_sample(secs=3.0): + """(CPU % of one core used by everything that is not the load or its observers, the top processes by CPU in the same + interval as [{"cmd": basename, "cpu_pct", "load": True when it is part of the load}]): the settle condition (the + guest has stopped doing its own housekeeping) and what to show when it never does.""" + a = parse_ps(ps_text()) + mine = own_pids(a) + t0 = now() + time.sleep(secs) + b = parse_ps(ps_text()) + dt = max(now() - t0, 0.001) + am = {p["pid"]: p for p in a} + rows = [(max(0.0, p["cpu_s"] - am[p["pid"]]["cpu_s"]), p["cmd"]) for p in b if p["pid"] in am and p["pid"] not in mine] + tot = sum(d for d, c in rows if not SETTLE_SKIP.search(c)) + rows.sort(key=lambda r: -r[0]) + top = [{"cmd": os.path.basename(c.split(" /")[0].strip()) or c[:30], "cpu_pct": round(100 * d / dt, 1), "load": bool(SETTLE_SKIP.search(c))} + for d, c in rows[:6] if d > 0] + return round(100 * tot / dt, 1), top + + +def bg_cpu_pct(secs=3.0): + return bg_sample(secs)[0] + + +def settle_wait(max_s, bg_pct, step, consecutive=2): + """Waits until the background CPU is under bg_pct for `consecutive` samples IN A ROW. {"settled", "bg_pct" (last + sample), "waited_s", "series", "top" (at the last sample)}. Reaching max_s is settled False: the caller fails the run, + it never measures on top of this.""" + t0, series, top = now(), [], [] + while True: + v, top = bg_sample(step) + series.append(v) + if len(series) >= consecutive and all(x < bg_pct for x in series[-consecutive:]): + return {"settled": True, "bg_pct": v, "waited_s": round(now() - t0, 1), "series": series[-8:], "top": top} + if now() - t0 > max_s: + return {"settled": False, "bg_pct": v, "waited_s": round(now() - t0, 1), "series": series[-8:], "top": top} + + +def cmd_settle(a): + out_json(settle_wait(a.max_s, a.bg_pct, a.step, a.consecutive)) + + +def run_window(a, udids, roles, mix, secs, t_go, tmp, probe_method, probe_every, cmd_tpl, no_strip=False, probe_timeout=2.0): + """simfps on each simulator, the probe and agent threads, and the guest's samples around the window.""" + raws, handles, stop_evt = {}, {}, threading.Event() + if mix != "idle": + for i, u in enumerate(udids): + raws[u] = os.path.join(tmp, "raw-%d.txt" % i) + handles[u] = spawn_simfps(u, secs, "bench-%d" % i, roles[i] != "static" and not no_strip, t_go, raws[u], + probe_method != "none" and not no_strip) + ips0, ps0, vm0, lvl0 = list_ips(), None, None, pressure_level() + d = t_go - now() + if d > 0: + time.sleep(d) + ps0, vm0 = parse_ps(ps_text()), parse_vm_stat(vmstat_text()) + t_start = now() + ru0 = resource.getrusage(resource.RUSAGE_CHILDREN) + t_stop = t_go + secs + inj, acts, threads = {u: [] for u in udids}, {u: [] for u in udids}, [] + lvls = [] + if mix in ("all-busy", "one-busy") and probe_method != "none": + targets = list(udids) # the busy simulator too: it is what a one-busy fleet's user waits on + for k, u in enumerate(targets): + t = threading.Thread(target=probe_loop, args=(u, probe_method, cmd_tpl, t_go + min(1.0, secs * 0.15) + k * probe_every / max(len(targets), 1), + probe_every, t_stop - probe_timeout - 0.2, inj[u], stop_evt)) + threads.append(t) + if mix == "agent": + for i, u in enumerate(udids): + threads.append(threading.Thread(target=agent_loop, args=(u, i, len(udids), t_go, t_stop, a.agent_period, acts[u], stop_evt))) + + def sample_pressure(): + while not stop_evt.is_set() and now() < t_stop: + lvls.append(pressure_level()) + stop_evt.wait(2.0) + threads.append(threading.Thread(target=sample_pressure)) + [t.start() for t in threads] + res = {} + try: + for u, h in handles.items(): + res[u] = h.wait(max(5, t_stop - now() + 40)) + while now() < t_stop: + time.sleep(0.2) + finally: + stop_evt.set() + [t.join(30) for t in threads] + for h in handles.values(): + h.kill() + if probe_method == "appearance" and mix in ("all-busy", "one-busy"): + for u in udids: # the appearance probe leaves a simulator dark: fps and tears are judged in light + simctl("ui", u, "appearance", "light", timeout=30) + ps1, vm1 = parse_ps(ps_text()), parse_vm_stat(vmstat_text()) + t_end = now() + ru1 = resource.getrusage(resource.RUSAGE_CHILDREN) + if mix != "idle" and not STUB: + time.sleep(6) # GlassBench writes pace.json every 5 s: wait for the record that closes the window + per, tot = attribute(ps0, ps1, udids, max(t_end - t_start, 0.001), own_pids(ps1)) + # simfps and the probes' simctl have exited (and been reaped) by now, so the two ps snapshots cannot see them: the + # CPU time of the helper's reaped children over the window is the observer's real cost + reaped = (ru1.ru_utime + ru1.ru_stime) - (ru0.ru_utime + ru0.ru_stime) + tot["observer_cpu_pct"] = round(max(tot["observer_cpu_pct"], 100 * reaped / max(t_end - t_start, 0.001)), 1) + dev_list = devices() + st = {d["udid"]: d.get("state") for d in dev_list} + foreign = {"before": foreign_count([dict(d, state="Booted") if d["udid"] in udids else d for d in dev_list], ps0), + "after": foreign_count(dev_list, ps1)} + sims = [] + for i, u in enumerate(udids): + sj, err = res.get(u, (None, None)) + raw = "" + if u in raws and os.path.exists(raws[u]): + with open(raws[u]) as f: + raw = f.read() + sims.append({"udid": u, "role": roles[i], "simfps": sj, "simfps_error": err, "raw": raw, "injections": inj[u], + "actions": acts[u], "proc": per.get(u), "state_after": st.get(u), + "pace": read_pace(u) if mix != "idle" and roles[i] != "static" else []}) + m0, m1 = mem_summary(vm0), mem_summary(vm1) + return {"t_go": t_go, "t_start": t_start, "t_end": t_end, "secs": round(t_end - t_start, 2), "sims": sims, + "guest": dict(tot, mem=m1, mem_before=m0, swapins_d=m1["swapins"] - m0["swapins"], swapouts_d=m1["swapouts"] - m0["swapouts"], + pageouts_d=m1["pageouts"] - m0["pageouts"], pressure_before=lvl0, + pressure_max=max([x for x in lvls + [pressure_level()] if x] or [None]) if any(lvls) else pressure_level()), + "new_ips": sorted(list_ips() - ips0), "procs_total": len(ps1), "foreign_booted": foreign} + + +def cmd_measure(a): + udids, roles = [u for u in a.udids.split(",") if u], [r for r in a.roles.split(",") if r] + tmp = tempfile.mkdtemp(prefix="benchwin") + try: + t_go = a.start_at or now() + 3 + len(udids) + out_json(run_window(a, udids, roles, a.mix, a.secs, t_go, tmp, a.probe_method, a.probe_every, a.probe_cmd, a.no_strip, a.probe_timeout)) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def cmd_calibrate(a): + """Which injection method is answered? 3 injections of each method on one simulator, watched by simfps.""" + from macbench import analysis + tmp = tempfile.mkdtemp(prefix="benchcal") + res = {} + try: + for method in a.methods.split(","): + f = a.time_scale + t_go = now() + 3 * f + raw = os.path.join(tmp, "raw.txt") + h = spawn_simfps(a.udid, 9 * f, "cal", False, t_go, raw) + inj, state = [], {} + for k in range(3): + d = t_go + (1.5 + k * 2.4) * f - now() + if d > 0: + time.sleep(d) + t0 = now() + rc = inject(method, a.udid, state, a.probe_cmd) + inj.append([t0, now(), rc]) + sj, err = h.wait(40 + 9 * f) + txt = open(raw).read() if os.path.exists(raw) else "" + r = analysis.probe_latencies(analysis.parse_raw(txt), [(x[0], x[1]) for x in inj], 2.0) + res[method] = {"answered": r["n"] - r["misses"], "n": r["n"], "p50_ms": r["p50_ms"], "call_p50_ms": r["call_p50_ms"], + "error": err} + out_json({"methods": res}) + finally: + simctl("ui", a.udid, "appearance", "light", timeout=30) # the appearance method leaves it dark + shutil.rmtree(tmp, ignore_errors=True) + + +def cmd_shots(a): + """K screenshots of each simulator: pixel hash, tear check, strip counter. PNGs stay here.""" + from macbench import png, strip, tears + tmp = tempfile.mkdtemp(prefix="benchshots") + res = [] + try: + for u in a.udids.split(","): + one = {"udid": u, "shots": []} + for k in range(a.count): + p = os.path.join(tmp, "%s-%d.png" % (u[:8], k)) + rc = simctl("io", u, "screenshot", "--type=png", p, timeout=60)[0] + if rc != 0 or not os.path.exists(p): + one["shots"].append({"ok": False}) + continue + with open(p, "rb") as f: + img = png.read_png(f.read()) + t = tears.detect_tears(img) + one["shots"].append({"ok": True, "w": img.w, "h": img.h, "hash": img.pixel_hash(), + "counter": strip.read_counter_image(img), "tear": t["tear"], "tear_applicable": t["applicable"]}) + os.unlink(p) + time.sleep(0.4) + res.append(one) + out_json({"sims": res}) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def cmd_cycle(a): + """One lifecycle of a simulator, timings in seconds. Production path (--seed): clone the seed (+ the Slim disables and the + idle-guard keep), boot (a clone's first boot), launch, first app frame, shut down, resume (boot again), first frame + again, delete. With --type/--runtime (the explicit stock path): create a fresh stock device instead, install the app, + and clone the installed device once more.""" + from macbench import analysis + tmp = tempfile.mkdtemp(prefix="benchcycle") + t = {} + udid = None + man = load_manifest() + try: + name = "%s-cycle" % a.prefix + t0 = now() + if a.seed: + if {d["udid"]: d for d in devices()}.get(a.seed, {}).get("state") != "Shutdown": + simctl("shutdown", a.seed, timeout=120) + rc, out, err, _ = simctl("clone", a.seed, name, timeout=300) + if rc != 0: + raise GuestError("clone of the seed: %s" % (err or out).strip()[:200]) + udid = out.strip().splitlines()[-1] + man["devices"][udid] = {"name": name, "kind": "clone"} + save_manifest(man) + copy_overrides(a.seed, udid) + keep_or_fail(udid, a.keep_min) + t["clone_s"] = round(now() - t0, 2) + else: + udid = simctl("create", name, a.type, a.runtime)[1].strip().splitlines()[-1] + man["devices"][udid] = {"name": name, "kind": "stock"} + save_manifest(man) + keep_or_fail(udid, a.keep_min) + t["create_s"] = round(now() - t0, 2) + for phase in ("cold", "resume"): + if phase == "resume": + t0 = now() + simctl("shutdown", udid, timeout=120) + t["shutdown_s"] = round(now() - t0, 2) + t0 = now() + simctl("boot", udid, timeout=300) + simctl("bootstatus", udid, timeout=600) + t["%s_boot_s" % phase] = round(now() - t0, 2) + if phase == "cold" and not a.seed: + t0 = now() + simctl("install", udid, os.path.join(HERE, "..", "GlassBench.app"), timeout=120) + t["install_s"] = round(now() - t0, 2) + raw = os.path.join(tmp, "raw-%s.txt" % phase) + h = spawn_simfps(udid, 14, "cycle", True, now() + 2, raw) + time.sleep(2.5) + t_l = now() + simctl("launch", udid, BUNDLE_ID, "-probe", timeout=60) + t["%s_launch_call_s" % phase] = round(now() - t_l, 2) + sj, err = h.wait(60) + frames = analysis.parse_raw(open(raw).read()) if os.path.exists(raw) else [] + first = next((x[0] for x in frames if x[0] >= t_l and x[1] >= 0 and x[1] != 65535), None) + t["%s_first_frame_s" % phase] = None if first is None else round(first - t_l, 2) + if not a.seed: + # clone of the shut-down, installed device (what a pool does for a new lease), then its boot + simctl("shutdown", udid, timeout=120) + t0 = now() + cl = simctl("clone", udid, "%s-clone" % name, timeout=300)[1].strip().splitlines()[-1] + t["clone_s"] = round(now() - t0, 2) + man = load_manifest() + man["devices"][cl] = {"name": "%s-clone" % name, "kind": "stock"} + save_manifest(man) + try: + t0 = now() + simctl("boot", cl, timeout=300) + simctl("bootstatus", cl, timeout=600) + t["clone_boot_s"] = round(now() - t0, 2) + finally: + simctl("shutdown", cl, timeout=120) + simctl("delete", cl, timeout=120) + else: + simctl("shutdown", udid, timeout=120) + t0 = now() + problems = delete_device(udid) + t["delete_s"] = round(now() - t0, 2) + udid = None + if problems: + raise GuestError("; ".join(problems)) + out_json({"timings": t}) + finally: + if udid: + delete_device(udid) + shutil.rmtree(tmp, ignore_errors=True) + + +def leftovers(ignore_lock=False): + """What an earlier run left in the bench's set: [text]. Empty = the set is clean (settled seeds, shut down, are not + leftovers: they are kept on purpose). The one definition: `acquire` refuses on it and `teardown` verifies against it.""" + out, man, devs = [], load_manifest(), devices() + complete = {s["udid"] for s in man["seeds"].values() if s.get("complete")} + for d in devs: + if is_seed(d): + if d["udid"] not in complete: + out.append("a seed that was never completed (%s %s)" % (d["name"], d["udid"])) + elif d.get("state") != "Shutdown": + out.append("the seed %s is %s" % (d["name"], d.get("state"))) + else: + out.append("device %s %s (%s)" % (d["name"], d["udid"], d.get("state"))) + here = {d["udid"] for d in devs} + for u in man["devices"]: + if u not in here and (os.path.exists(overrides_dir(u)) or os.path.exists(os.path.join(KEEP_DIR, u))): + out.append("overrides or keep file of the deleted device %s" % u) + lock = os.path.join(SET, LOCK) + if os.path.exists(lock) and not ignore_lock: + try: + with open(lock) as f: + out.append("the run lock of an earlier run (%s)" % f.read().strip()[:80]) + except OSError: + out.append("the run lock of an earlier run") + return out + + +def cmd_acquire(a): + """Takes the set for one run: refuses (ok false, with the list) while an earlier run left anything, then writes the + run lock (exclusive create: two runs cannot share the set).""" + left = leftovers() + if left: + return out_json({"ok": False, "leftovers": left}) + try: + fd = os.open(os.path.join(SET, LOCK), os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + except FileExistsError: + return out_json({"ok": False, "leftovers": ["the run lock was taken just now by another run"]}) + with os.fdopen(fd, "w") as f: + f.write("run %s pid %d at %d\n" % (a.run, os.getpid(), int(now()))) + out_json({"ok": True, "leftovers": []}) + + +def cmd_teardown(a): + """Shuts down every device of the bench's set and deletes every one that is not a COMPLETE seed (every seed with --seeds; + a seed that was never completed is a leftover like any other device), then removes what lives outside the device: its + overrides folder and its ~/.sim-keep file. It looks again at the end with the same definition of 'leftover' that + `acquire` uses and reports what is still there in `failed`/`left`/`booted`; the run lock goes only when nothing is.""" + failed = [] + man = load_manifest() + complete = {s["udid"] for s in man["seeds"].values() if s.get("complete")} + simctl("shutdown", "all", timeout=300) # a failure shows in the second look below + for d in devices(): + if d["udid"] in complete and not a.seeds: + continue + rc, _, err, _ = simctl("delete", d["udid"], timeout=120) + if rc != 0: + failed.append("delete %s %s: %s" % (d["name"], d["udid"], (err or "failed").strip()[:100])) + after = devices() + present = {d["udid"] for d in after} + for u in sorted(set(man["devices"]) | {s["udid"] for s in man["seeds"].values()} | present): + e = drop_keep(u) + if e: + failed.append(e) + if u not in present: + e = drop_overrides(u) + if e: + failed.append(e) + else: + man["devices"].pop(u, None) + man["seeds"] = {t: s for t, s in man["seeds"].items() if s["udid"] in present and s["udid"] in complete} + save_manifest(man) + keep_left = [u for u in present | set(man["devices"]) if os.path.exists(os.path.join(KEEP_DIR, u))] + if keep_left: + failed.append("%d keep file(s) still in the idle guard's folder" % len(keep_left)) + remaining = leftovers(ignore_lock=True) + failed += ["still there: %s" % x for x in remaining] + booted = [d for d in after if d.get("state") == "Booted"] + if not (failed or remaining or booted): + try: + os.remove(os.path.join(SET, LOCK)) + except FileNotFoundError: + pass + except OSError as e: + failed.append("run lock: %s" % e) + out_json({"left": len(remaining), "booted": len(booted), "failed": failed, "seeds": len([d for d in after if is_seed(d)]), "calls": [c[1:] for c in CALLS][-3:]}) + + +def count_launchd_sim(): + """Simulators running on this Mac in any device set (one launchd_sim each): the pool's, a lease's, the bench's.""" + return len([p for p in parse_ps(ps_text()) if re.search(r"(^|/)launchd_sim( |$)", p["cmd"])]) + + +def foreign_count(devs=None, procs=None): + """Simulators that are not the bench's: the `launchd_sim` processes minus the bench's own. A device of the bench's set + in ANY state counts as its own when the process's command line names its UDID (a device that is booting, shutting + down or being rebooted by simslim has a launchd_sim and is not Booted yet or any more); where the command line names + no device (HYPOTHESIS: it does carry the device's path), the number of Booted devices is the fallback.""" + devs = devices() if devs is None else devs + udids = [d["udid"] for d in devs] + procs = [p for p in (parse_ps(ps_text()) if procs is None else procs) if re.search(r"(^|/)launchd_sim( |$)", p["cmd"])] + named = len([p for p in procs if any(u in p["cmd"] for u in udids)]) + return max(0, len(procs) - max(named, len([d for d in devs if d.get("state") == "Booted"]))) + + +def cmd_precheck(a): + """Before a measurement window: how many simulators of the bench's own set are Booted (counted from the set's own + device list, the seed included), which of the expected ones are not, and how many simulators outside the bench run on + this Mac (a `launchd_sim` that is not the bench's). Renews the idle-guard keep of the expected simulators.""" + expect = [u for u in a.udids.split(",") if u] + booted = sorted(d["udid"] for d in devices() if d.get("state") == "Booted") + kept = {} + for u in expect: + if a.keep_min: + k = keep_or_fail(u, a.keep_min) + kept[k["state"]] = kept.get(k["state"], 0) + 1 + out_json({"booted": len(booted), "booted_udids": booted, "missing": [u for u in expect if u not in booted], + "extra": [u for u in booted if u not in expect], "foreign_booted": foreign_count(), "keep": kept}) + + +def cmd_ips(a): + out_json({"ips": sorted(list_ips())}) + + +def cmd_memcheck(a): + vm = mem_summary(parse_vm_stat(vmstat_text())) + own = len([d for d in devices() if d.get("state") == "Booted"]) + out_json({"mem": vm, "pressure": pressure_level(), "booted": own, "foreign_booted": foreign_count(), + "ips": len(list_ips())}) + + +def cmd_build(a): + """Builds the bench tools in a scratch copy: cold (a module cache that does not exist yet), then warm (the same + cache again); optionally an xcodebuild of a project given on the command line (cold: new DerivedData; warm: again).""" + root = os.path.abspath(os.path.join(HERE, "..")) + tmp = tempfile.mkdtemp(prefix="benchbuild") + res = {} + try: + for f in ("simfps.swift", "build.sh"): + shutil.copy(os.path.join(root, f), tmp) + for d in ("GlassBench", "GenieBench"): + shutil.copytree(os.path.join(root, d), os.path.join(tmp, d)) + mc = os.path.join(tmp, "modcache") + env = dict(os.environ, CLANG_MODULE_CACHE_PATH=mc) + for phase in ("cold", "warm"): + rc, _, err, secs = run(["/bin/bash", os.path.join(tmp, "build.sh")], 1800, env=env) + res["tools_%s_s" % phase] = round(secs, 1) + res["tools_%s_rc" % phase] = rc + if a.xcodebuild: + dd = os.path.join(tmp, "dd") + for phase in ("cold", "warm"): + rc, _, err, secs = run(["xcodebuild"] + a.xcodebuild.split() + ["-derivedDataPath", dd], 3600) + res["xcodebuild_%s_s" % phase] = round(secs, 1) + res["xcodebuild_%s_rc" % phase] = rc + out_json(res) + finally: + shutil.rmtree(tmp, ignore_errors=True) + + +def cmd_calls(a): + out_json({"calls": CALLS}) + + +def main(argv=None): + global SET + if STUB: # after every function is defined: the stub replaces the primitives (run, ps_text, list_ips...) + from guest import benchguest_stub + benchguest_stub.install(sys.modules[__name__]) + ap = argparse.ArgumentParser(prog="benchguest") + ap.add_argument("--set", default=os.environ.get("BENCH_SET")) + sp = ap.add_subparsers(dest="cmd", required=True) + + def add(name, fn, *opts): + p = sp.add_parser(name) + p.set_defaults(fn=fn) + for o in opts: + p.add_argument(*o[0], **o[1]) + return p + O = lambda *n, **k: (n, k) + add("info", cmd_info) + add("init-set", cmd_init_set) + add("ensure-type", cmd_ensure_type, O("--scale", default="1"), O("--allow-restart", action="store_true")) + add("create", cmd_create, O("--n", type=int, required=True), O("--type", required=True), O("--runtime", required=True), + O("--prefix", default="bench"), O("--start", type=int, default=0), O("--keep-min", type=float, default=60)) + add("seed", cmd_seed, O("--type", required=True), O("--source-set"), O("--simslim"), O("--slim-profile"), O("--keep-min", type=float, default=60), + O("--rebuild", action="store_true"), O("--settle-max", type=float, default=600), O("--bg-pct", type=float, default=40), + O("--step", type=float, default=10), O("--consecutive", type=int, default=2)) + add("clone", cmd_clone, O("--seed", required=True), O("--n", type=int, required=True), O("--prefix", default="bench"), O("--start", type=int, default=0), + O("--keep-min", type=float, default=60)) + add("boot", cmd_boot, O("--udids", required=True), O("--parallel", action="store_true")) + add("prepare", cmd_prepare, O("--udids", required=True), O("--roles", required=True), O("--install", action="store_true"), + O("--check-app", action="store_true"), O("--quiesce", action="store_true")) + add("settle", cmd_settle, O("--max-s", type=float, default=120), O("--bg-pct", type=float, default=40), O("--step", type=float, default=3), O("--consecutive", type=int, default=2)) + add("calibrate", cmd_calibrate, O("--udid", required=True), O("--methods", default="notify,appearance"), O("--probe-cmd"), O("--time-scale", type=float, default=1.0)) + add("measure", cmd_measure, O("--udids", required=True), O("--roles", required=True), O("--mix", default="all-busy"), + O("--secs", type=float, default=40), O("--start-at", type=float, default=0), O("--probe-method", default="openurl"), + O("--probe-every", type=float, default=2.5), O("--probe-cmd"), O("--agent-period", type=float, default=12), + O("--no-strip", action="store_true"), O("--probe-timeout", type=float, default=2.0)) + add("shots", cmd_shots, O("--udids", required=True), O("--count", type=int, default=2)) + add("cycle", cmd_cycle, O("--seed"), O("--type"), O("--runtime"), O("--prefix", default="bench"), O("--keep-min", type=float, default=60)) + add("acquire", cmd_acquire, O("--run", default="?")) + add("teardown", cmd_teardown, O("--seeds", action="store_true", help="delete the seeds too")) + add("build", cmd_build, O("--xcodebuild", default="")) + add("memcheck", cmd_memcheck) + add("precheck", cmd_precheck, O("--udids", default=""), O("--keep-min", type=float, default=0)) + add("ips", cmd_ips) + add("calls", cmd_calls) + a = ap.parse_args(argv) + SET = a.set + try: + if a.cmd != "init-set": + if a.cmd not in ("info", "ensure-type"): + check_set() + a.fn(a) + except GuestError as e: + out_json({"error": str(e)}) + return 3 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/bench/guest/benchguest_stub.py b/src/bench/guest/benchguest_stub.py new file mode 100644 index 0000000..eb50d5d --- /dev/null +++ b/src/bench/guest/benchguest_stub.py @@ -0,0 +1,409 @@ +"""The guest's stand-in for DRY=1 and the tests: a fake simctl, ps, vm_stat and simfps that keep their state in a +directory ($BENCHGUEST_STATE) between calls, so `benchguest.py` runs its real control flow with no macOS. +Faults ($BENCH_STUB_FAULT, comma separated): memfloor@N (guest free memory below any floor from N booted +simulators), crash@N (a new crash report at N booted), backwards@N (a frame shown out of order at N), slow@N +(frame rate collapses from N), hang_simfps (simfps never answers), nomarker (the input marker never flips), nonotify (it does not flip on the notification), tear@N (the screenshots at N booted are torn), +cantdelete (simctl delete fails: a cleanup that cannot finish), noguard (no sim-idle-guard on the Mac), keepfail +(sim-idle-guard keep fails), nosource (Simlock has no source device of the type), nosourceset (no Simlock set at all), +slimfail (simslim fails), slowslim (simslim takes 2 minutes: a run killed while the seed is half made), slimfew (simslim leaves too few services off), noicons (no chronod icons folder: recorded as icons=false, never a failure), +noinstall (simctl install does nothing), unsettled@N (a stray process burns 1.7 cores from N booted simulators), +lostsim@N (the Mac's idle guard shuts the newest simulator down once N are booted), counterfail@N (the frame counter +of the last simulator returns nothing from N booted), foreign@N (a simulator outside the bench boots when N are booted), transition@N (a bench device that is not Booted still has its launchd_sim: it must not read as foreign), swapgrow@N (the guest's swap-outs +grow from N booted), compressor@N (the compressor holds 3 GB from N booted). +The files a simulator leaves outside its device (Slim overrides, ~/.sim-keep, the idle guard) live under +$BENCHGUEST_STATE too, so the bench's real file code runs against them.""" +import json +import os +import random +import sys +import time +import uuid + +STATE = os.environ.get("BENCHGUEST_STATE", "/tmp/benchguest-stub") +FAULTS = [f for f in os.environ.get("BENCH_STUB_FAULT", "").split(",") if f] +T0 = float(os.environ.get("BENCH_STUB_T0", "0")) or time.time() + + +def _p(name): + return os.path.join(STATE, name) + + +def load(): + try: + with open(_p("state.json")) as f: + return json.load(f) + except (OSError, ValueError): + return {"devs": {}, "inj": [], "mode": {}, "ips": [], "pid": 1000} + + +def save(s): + os.makedirs(STATE, exist_ok=True) + with open(_p("state.json.tmp"), "w") as f: + json.dump(s, f) + os.replace(_p("state.json.tmp"), _p("state.json")) + + +def fault(name): + for f in FAULTS: + if f == name or f.startswith(name + "@"): + return int(f.split("@")[1]) if "@" in f else 0 + return None + + +def booted(s): + return [u for u, d in s["devs"].items() if d["state"] == "Booted"] + + +def _log(argv): + os.makedirs(STATE, exist_ok=True) + with open(os.environ.get("BENCHGUEST_CALLS", _p("calls.log")), "a") as f: + f.write(json.dumps(argv) + "\n") + + +RUNTIMES = {"runtimes": [{"identifier": "stub.runtime.iOS-26-5", "name": "iOS 26.5", "version": "26.5", "buildversion": "stub", + "platform": "iOS", "isAvailable": True, + "supportedDeviceTypes": [{"name": "iPhone 17 Pro", "identifier": "com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro", + "bundlePath": "/nonexistent/iPhone 17 Pro.simdevicetype"}]}]} +TYPES = {"devicetypes": [{"identifier": "com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro", "name": "iPhone 17 Pro"}, + {"identifier": "dev.macbox.CoreSimulator.SimDeviceType.iPhone-17-Pro-1x", "name": "iPhone 17 Pro (1x)"}]} + + +class _Lock: + """One stub process at a time through the state file (threads and processes alike).""" + def __enter__(self): + import fcntl + os.makedirs(STATE, exist_ok=True) + self.f = open(_p("lock"), "w") + fcntl.flock(self.f, fcntl.LOCK_EX) + + def __exit__(self, *a): + self.f.close() + + +def stub_run(argv, timeout=120, env=None): + with _Lock(): + return _stub_run(argv) + + +SRC_UDIDS = ["5A5A5A5A-0000-4000-8000-00000000000%d" % i for i in range(4)] +SRC_TYPES = ["com.apple.CoreSimulator.SimDeviceType.iPhone-17", "com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro", + "com.apple.CoreSimulator.SimDeviceType.iPhone-17-Pro", "dev.macbox.CoreSimulator.SimDeviceType.iPhone-17-Pro-1x"] + + +def make_source_set(path): + """Simlock's source set as the bench sees it: one folder per device with a device.plist (an iPhone 17 Pro shut down and one booted, its 1x type, another type).""" + import plistlib + os.makedirs(path, exist_ok=True) + for i, (typ, state) in enumerate(zip(SRC_TYPES, (1, 1, 3, 1))): + d = os.path.join(path, SRC_UDIDS[i]) + if os.path.exists(d): + continue # made once: a run must find it exactly as it was + os.makedirs(d, exist_ok=True) + with open(os.path.join(d, "device.plist"), "wb") as f: + plistlib.dump({"UDID": SRC_UDIDS[i], "deviceType": typ, "runtime": "stub.runtime.iOS-26-5", "name": "src%d" % i, "state": state}, f) + + +def _stub_source(argv, s): + """`xcrun simctl --set ...`: ONLY clone into another set is allowed; anything else is recorded as a violation.""" + cmd = argv[4:] + if cmd[0] != "clone" or len(cmd) != 4 or cmd[1] not in SRC_UDIDS: + s.setdefault("source_violations", []).append(cmd) + return 1, "", "stub: Simlock's set may only be cloned from" + u = str(uuid.uuid4()).upper() + s["devs"][u] = {"name": cmd[2], "state": "Shutdown", "type": SRC_TYPES[SRC_UDIDS.index(cmd[1])], "apps": []} + return 0, u + "\n", "" + + +def _stub_run(argv): + _log(argv) + s = load() + out, err, rc = "", "", 0 + if argv and argv[0] == _MOD.IDLE_GUARD: + # the Mac's idle guard: `keep ` (max 240) writes ~/.sim-keep/ + if fault("keepfail") is not None: + return 1, "", "keep: refused (injected)", 0.01 + if argv[1] != "keep" or len(argv) != 4 or not argv[3].isdigit() or not 1 <= int(argv[3]) <= 240: + return 2, "", "usage: sim-idle-guard keep ", 0.01 + os.makedirs(_MOD.KEEP_DIR, exist_ok=True) + with open(os.path.join(_MOD.KEEP_DIR, argv[2]), "w") as f: + f.write("%s\n" % argv[3]) + return 0, "", "", 0.01 + if argv[:3] == ["xcrun", "simctl", "--set"] and argv[3] == _MOD.SOURCE_SET: + rc, out, err = _stub_source(argv, s) + elif argv and argv[0] == _MOD.SIMSLIM: + # simslim --set on --profile : shuts the device down, rewrites its services, reboots it + u = argv[4] + if fault("slowslim") is not None: + time.sleep(120) # a run that is killed while its seed is half made + if fault("slimfail") is not None: + rc, err = 1, "simslim: injected failure" + elif s["devs"].get(u, {}).get("state") != "Booted": + rc, err = 1, "simslim: device is not booted" + else: + import plistlib + os.makedirs(_MOD.overrides_dir(u), exist_ok=True) + n = 20 if fault("slimfew") is not None else 168 + with open(os.path.join(_MOD.overrides_dir(u), "disabled.plist"), "wb") as f: + plistlib.dump({"com.apple.stub.service%d" % i: True for i in range(n)}, f) + elif argv and argv[0] == "plutil": + import plistlib + try: + with open(argv[2], "rb") as f: + d = plistlib.load(f) + out = "{\n" + "".join(' "%s" => %s\n' % (k, "true" if v is True else "false") for k, v in d.items()) + "}\n" + except (OSError, ValueError): + rc, err = 1, "plutil: no such file" + elif argv[:3] == ["xcrun", "simctl", "--set"]: + cmd = argv[4:] + verb = cmd[0] + if verb == "list": + if cmd[1] == "devices": + ls = fault("lostsim") + if ls is not None and len(booted(s)) >= ls and not s.get("lost") and "-j" in cmd: + s["devs"][booted(s)[-1]]["state"] = "Shutdown" # the idle guard took it + s["lost"] = 1 + devs = [{"udid": u, "name": d["name"], "state": d["state"], "isAvailable": True} for u, d in s["devs"].items()] + out = json.dumps({"devices": {"stub.runtime.iOS-26-5": devs}}) + else: + out = json.dumps(RUNTIMES if cmd[1] == "runtimes" else TYPES) + elif verb == "create": + u = str(uuid.uuid4()).upper() + s["devs"][u] = {"name": cmd[1], "state": "Shutdown", "type": cmd[2], "apps": []} + out = u + "\n" + elif verb == "clone": + src = s["devs"][cmd[1]] + if src["state"] != "Shutdown": + rc, err = 1, "Unable to clone a device in state: %s" % src["state"] # HYPOTHESIS about real simctl + else: + u = str(uuid.uuid4()).upper() + s["devs"][u] = {"name": cmd[2], "state": "Shutdown", "type": src.get("type"), "apps": list(src.get("apps", []))} + if os.path.isdir(os.path.join(argv[3], cmd[1], "data")): + import shutil + shutil.copytree(os.path.join(argv[3], cmd[1], "data"), os.path.join(argv[3], u, "data")) # the clone has the seed's data + out = u + "\n" + elif verb == "boot": + time.sleep(0.05) + s["devs"][cmd[1]]["state"] = "Booted" + if fault("noicons") is None: + os.makedirs(os.path.join(argv[3], cmd[1], "data", "Library", "chronod", "icons"), exist_ok=True) + elif verb == "install": + if fault("noinstall") is None: + s["devs"][cmd[1]].setdefault("apps", []).append("dev.macbox.glassbench") + elif verb == "shutdown": + for u in (list(s["devs"]) if cmd[1] == "all" else [cmd[1]]): + if u in s["devs"]: + s["devs"][u]["state"] = "Shutdown" + elif verb == "delete": + if fault("cantdelete") is not None: + err, rc = "Unable to delete (injected)", 1 + else: + import shutil + for u in (list(s["devs"]) if cmd[1] == "all" else [cmd[1]]): + s["devs"].pop(u, None) + shutil.rmtree(os.path.join(argv[3], u), ignore_errors=True) + elif verb == "launch": + s["mode"][cmd[1]] = "static" if "-static" in cmd else "busy" + elif verb == "terminate": + s["mode"].pop(cmd[1], None) + elif verb in ("openurl", "ui") or (verb == "spawn" and "notifyutil" in cmd): + if fault("nomarker") is None and not (verb == "spawn" and fault("nonotify") is not None): + s["inj"].append([time.time(), cmd[1]]) + elif verb == "io": + p = cmd[-1] + from macbench import png + rows = [bytes([(x * 3 + y) % 251 for x in range(402) for _ in range(3)][:402 * 3]) for y in range(874)] + with open(p, "wb") as f: + f.write(png.write_png(rows, 402, 874, 3)) + elif verb == "get_app_container": + if "dev.macbox.glassbench" not in s["devs"].get(cmd[1], {}).get("apps", []): + save(s) + return 1, "", "No such app", 0.01 + d = _p("container") + os.makedirs(os.path.join(d, "Documents"), exist_ok=True) + u = cmd[1] + now = time.time() + with open(os.path.join(d, "Documents", "pace.json"), "w") as f: + for k in range(8): + f.write(json.dumps({"at": now - 40 + 5 * k, "counter": int((now - 40 + 5 * k - T0) * 60) % 65536, "fps": 60.0, "ticks": 300}) + "\n") + out = d + "\n" + elif argv[:3] == ["xcrun", "simctl", "list"]: + out = json.dumps(RUNTIMES if argv[3] == "runtimes" else TYPES) if argv[3] in ("runtimes", "devicetypes") else "" + elif argv[0] in ("sw_vers",): + out = "ProductName: macOS\nProductVersion: 26.0\n" + elif argv[0] == "xcodebuild": + out = "Xcode 26.0\nBuild version stub\n" + elif argv[0] == "uname": + out = "x86_64\n" + else: + out = "" + save(s) + return rc, out, err, 0.01 + + +def stub_ps(): + s = load() + now = time.time() + lines = [" 1 0 4000 %s /sbin/launchd" % _cpu(now - T0, 0.01), " 100 1 60000 %s /usr/sbin/WindowServer" % _cpu(now - T0, 0.2)] + pid = 200 + for i, u in enumerate(sorted(booted(s))): + busy = s["mode"].get(u) == "busy" + base = "/stub/set/%s/data" % u + lines.append("%5d 1 20000 %s launchd_sim %s" % (pid, _cpu(now - T0, 0.003), base)) + lines.append("%5d %5d 150000 %s SpringBoard" % (pid + 1, pid, _cpu(now - T0, 0.03 if busy else 0.01))) + lines.append("%5d %5d 300000 %s GlassBench" % (pid + 2, pid, _cpu(now - T0, 0.5 if busy else 0.01))) + lines.append("%5d 1 90000 %s SimMetalHost -d %s" % (pid + 3, _cpu(now - T0, 0.3 if busy else 0.02), u)) + pid += 10 + n = len(booted(s)) + us = fault("unsettled") + if us is not None and n >= us: # the Maps widget's retry loop: a process that never quiets down + lines.append("%5d 1 40000 %s /usr/libexec/diagnosticd" % (900, _cpu(now - T0, 1.7))) + tr = fault("transition") + if tr is not None and n >= tr: # a bench device (the seed) that is shut down, its launchd_sim still alive: not foreign + for u, d in sorted(s["devs"].items()): + if d["state"] != "Booted": + lines.append("%5d 1 20000 %s launchd_sim /stub/set/%s/data" % (960, _cpu(now - T0, 0.003), u)) + break + fo = fault("foreign") + if fo is not None and n >= fo: # someone else's simulator on the Mac, in another device set + lines.append("%5d 1 20000 %s launchd_sim /stub/other-set/FOREIGN/data" % (950, _cpu(now - T0, 0.01))) + return "\n".join(lines) + "\n" + + +def _cpu(elapsed, frac): + secs = max(0.0, elapsed) * frac + return "%d:%05.2f" % (int(secs // 60), secs % 60) + + +def stub_vmstat(): + s = load() + n = len(booted(s)) + page = 16384 + total_pages = 16 * 1024 * 1024 * 1024 // page + used = (3000 + 1100 * n) * 1048576 // page + free = max(0, total_pages - used) + sw = fault("swapgrow") + swapouts = 500 if sw is not None and n >= sw else 0 + cp = fault("compressor") + comp = 200000 if cp is not None and n >= cp else 5000 + 300 * n + mf = fault("memfloor") + if mf is not None and n >= mf: + free = 20 * 1048576 // page + return ("Mach Virtual Memory Statistics: (page size of %d bytes)\nPages free: %d.\nPages active: %d.\nPages inactive: 1000.\n" + "Pages speculative: 500.\nPages wired down: %d.\nPages occupied by compressor: %d.\nPageins: 1000.\nPageouts: 0.\n" + "Swapins: 0.\nSwapouts: %d.\n" % (page, free, used // 2, used // 4, comp, swapouts)) + + +class StubSimfps: + def __init__(self, udid, secs, strip, start_at, raw_path): + self.udid, self.secs, self.strip, self.start_at, self.raw = udid, secs, strip, start_at, raw_path + + def wait(self, timeout): + cf = fault("counterfail") + if cf is not None and len(booted(load())) >= cf and self.udid == sorted(booted(load()))[-1]: + return None, "simfps: no surface for the simulator (injected)" + if fault("hang_simfps") is not None: + return None, "simfps hung (killed after %ds)" % int(timeout) + d = self.start_at + self.secs - time.time() + if d > 0: + time.sleep(d) + s = load() + n = len(booted(s)) + mode = s["mode"].get(self.udid, "busy") + rng = random.Random(hash(self.udid) & 0xffff) + fps = 60.0 if n <= 2 else max(8.0, 60.0 - 7.0 * (n - 2)) + sl = fault("slow") + if sl is not None and n >= sl: + fps = 14.0 + inj = sorted(t for t, u in s["inj"] if u == self.udid and self.start_at - 1 <= t <= self.start_at + self.secs) + lines, t, c, lum, k = [], self.start_at, 100, 255, 0 + back = fault("backwards") + lines.append("%.6f %d %d" % (t, c if self.strip else -1, lum)) + end = self.start_at + self.secs + delay = [0.06 + rng.random() * 0.05 for _ in inj] + while t < end: + if mode == "busy": + t += (1.0 / fps) * (0.9 + 0.2 * rng.random()) + c = (c + 1) % 65536 + if back is not None and n >= back and k == 50: + c = (c - 3) % 65536 + k += 1 + else: + t += 0.05 + while inj and t >= inj[0] + delay[0]: + lum = 0 if lum == 255 else 255 + inj.pop(0) + delay.pop(0) + if mode != "busy": + lines.append("%.6f %d %d" % (t, -1, lum)) + if mode == "busy": + lines.append("%.6f %d %d" % (t, c if self.strip else -1, lum)) + with open(self.raw, "w") as f: + f.write("\n".join(lines) + "\n") + n_frames = len(lines) + return {"fps": round(n_frames / self.secs, 1), "frames": n_frames, "secs": self.secs, "udid": self.udid}, None + + def kill(self): + pass + + +_MOD = None + + +def install(mod): + global _MOD + _MOD = mod + mod.OVERRIDES = _p("overrides/SimDevice") + mod.KEEP_DIR = _p("sim-keep") + mod.IDLE_GUARD = _p("bin/sim-idle-guard") + if fault("noguard") is None: + os.makedirs(_p("bin"), exist_ok=True) + with open(mod.IDLE_GUARD, "w") as f: + f.write("#!/bin/sh\n") + os.chmod(mod.IDLE_GUARD, 0o755) + mod.SOURCE_SET = _p("simlock-sources") + if fault("nosourceset") is None: + make_source_set(mod.SOURCE_SET) + if fault("nosource") is not None: + import shutil + for u in SRC_UDIDS[1:]: + shutil.rmtree(os.path.join(mod.SOURCE_SET, u)) + mod.SLIM_DIR = _p("simslim-config") + os.makedirs(mod.SLIM_DIR, exist_ok=True) + with open(os.path.join(mod.SLIM_DIR, "profile.json"), "w") as f: + f.write("{}") + mod.SIMSLIM = _p("bin/simslim") + os.makedirs(_p("bin"), exist_ok=True) + with open(mod.SIMSLIM, "w") as f: + f.write("#!/bin/sh\n") + os.chmod(mod.SIMSLIM, 0o755) + mod.run = stub_run + mod.ps_text = stub_ps + mod.vmstat_text = stub_vmstat + mod.sysctl = lambda name: {"kern.memorystatus_vm_pressure_level": "1", "hw.ncpu": "8", "hw.memsize": str(16 << 30)}.get(name, "") + tr = fault("tear") + if tr is not None: # every screenshot taken while exactly `tr` simulators are booted is torn + from macbench import tears as _tears + + def torn(img): + return {"tear": len(booted(load())) == tr, "applicable": True, "odd": [1] if len(booted(load())) == tr else [], + "steps": [], "gaps": [58, 58]} + _tears.detect_tears = torn + + def spawn(udid, secs, label, strip, start_at, raw, probe=True): + _log(["simfps", udid, "strip" if strip else "nostrip", "probe" if probe else "noprobe"]) + return StubSimfps(udid, secs, strip, start_at, raw) + mod.spawn_simfps = spawn + + calls = [0] + + def ips(): + # a crash report appears during a measurement (the second look of a window), not before it + s = load() + calls[0] += 1 + cr = fault("crash") + for cr, name in ((cr, "GlassBench-2026-01-02-030405.ips"), (fault("daemoncrash"), "searchd-2026-01-02-030405.ips")): + if cr is not None and len(booted(s)) >= cr and calls[0] >= 2 and name not in s["ips"]: + s["ips"].append(name) # stays on the "disk": the run-wide gate sees it later + save(s) + return set(s["ips"]) + mod.list_ips = ips diff --git a/src/bench/macbench/__init__.py b/src/bench/macbench/__init__.py new file mode 100644 index 0000000..0488359 --- /dev/null +++ b/src/bench/macbench/__init__.py @@ -0,0 +1 @@ +"""macbox bench: the scaling, regression and A/B suite (see ../README.md). Python 3.8+, standard library only.""" diff --git a/src/bench/macbench/ab.py b/src/bench/macbench/ab.py new file mode 100644 index 0000000..b6af917 --- /dev/null +++ b/src/bench/macbench/ab.py @@ -0,0 +1,88 @@ +"""`ab`: one knob, arms interleaved A B B A inside one window, only the within-window difference judged.""" +import time + +from . import abrule, core, gates, scale, stats +from .core import Stop +from .runctx import CmdKnob, LiveKnob, shell_runner + +METRICS = { + "total_fps": (lambda st: sum(s["fps"] for s in st["sims"] if s.get("fps") is not None), True), + "min_fps": (lambda st: min([s["fps"] for s in st["sims"] if s.get("fps") is not None] or [None]), True), + "worst_p95": (lambda st: max([s["p95_ms"] for s in st["sims"] if s.get("p95_ms") is not None] or [None]), False), + "worst_p99": (lambda st: max([s["p99_ms"] for s in st["sims"] if s.get("p99_ms") is not None] or [None]), False), + "input_p95": (lambda st: max([s["input_p95_ms"] for s in st["sims"] if s.get("input_p95_ms") is not None] or [None]), False), + "gpu_ms_per_frame": (lambda st: st["system"].get("gpu_ms_per_frame"), False), + "qemu_cores": (lambda st: st["system"].get("qemu_cpu_cores"), False), +} +DEFAULT_TAILS = {"total_fps": ["worst_p95", "worst_p99"], "min_fps": ["worst_p95", "worst_p99"], "gpu_ms_per_frame": ["worst_p95"], + "qemu_cores": ["min_fps"], "worst_p95": ["min_fps"], "worst_p99": ["min_fps"], "input_p95": ["min_fps"]} + + +def make_knob(a, ctx): + if a.knob and a.knob.startswith("live:"): + k, v = a.knob[5:].split("=", 1) + if not ctx.live_conf: + raise Stop("--knob live:KEY=VALUE needs the VM's live switch file (not found)") + return LiveKnob(ctx.live_conf, k, v, ctx.run) + if a.set_cmd and a.unset_cmd: + return CmdKnob(a.set_cmd, a.unset_cmd, ctx.run, shell_runner()) + raise Stop("a knob is needed: --knob live:KEY=VALUE, or --set-cmd and --unset-cmd") + + +def run_ab(ctx): + a = ctx.a + metric = a.metric + hb = METRICS[metric][1] + tails = [t for t in (a.tails.split(",") if a.tails else DEFAULT_TAILS.get(metric, [])) if t] + rule = "AB rule (fixed before the run): knob %s; metric %s (%s better); ADOPT only if the mean effect of the 2 pairs exceeds %.1f%% and both pairs agree in sign; tails %s may not worsen by more than %.1f%%; order A B B A; no re-run" % ( + a.knob or "cmd", metric, "higher" if hb else "lower", a.threshold * 100, ",".join(tails) or "none", a.tail_tol * 100) + ctx.log(rule) + ctx.doc["params"]["rule"] = rule + knob = make_knob(a, ctx) + info = core.setup_guest(ctx) + core.capture_env(ctx, info) + ctx.marks = {"coh": ctx.coh.mark(), "t0": time.time()} + n, mix = a.n, a.mix + reason = core.guard(ctx, n, "before the A/B") + if reason: + raise Stop(reason) + ctx.host.n = n + core.grow(ctx, n, core.roles_for(mix, n)) + ctx.probe_method = scale.calibrate(ctx, ctx.sims[0])["chosen"] if mix in ("all-busy", "one-busy") and metric == "input_p95" else "none" + core.settle(ctx, "before the A/B") + ctx.doc["data"] = {"knob": knob.describe(), "metric": metric, "arms": [], "n": n, "mix": mix} + vals = {"A": [], "B": []} + steps = {"A": [], "B": []} + bad = None + ctx.run.on_deadline = lambda reason=None: (_save(ctx), None)[1] + for arm in abrule.abba_order(): + (knob.set if arm == "B" else knob.unset)() + time.sleep(a.arm_settle) + g, s0, s1, t_go = core.window(ctx, a.secs, mix) + step = core.build_step(ctx, n, mix, g, s0, s1, t_go) + core.step_gates(ctx, step) + steps[arm].append(step) + v = {k: f(step) for k, (f, _) in METRICS.items()} + vals[arm].append(v) + ctx.doc["data"]["arms"].append({"arm": arm, "values": v, "gates_ok": step["gates_ok"], "stable": step["stable"]}) + _save(ctx) + if not (step["gates_ok"] and step["stable"]) and bad is None: + bad = "arm %s failed its gates or stability: %s" % (arm, [k for k, g in step["gates"].items() if g["status"] == "FAIL"] or step["stability"]) + knob.unset() + a1, a2 = vals["A"][0][metric], vals["A"][1][metric] + b1, b2 = vals["B"][0][metric], vals["B"][1][metric] + tl = {t: (vals["A"][0][t], vals["B"][0][t], vals["A"][1][t], vals["B"][1][t], METRICS[t][1]) for t in tails} + res = abrule.decide(a1, b1, a2, b2, hb, a.threshold, tl, a.tail_tol) + if bad: + res["decision"] = "CLOSE" + res["why"].append(bad) + line = abrule.decision_line(knob.describe() if a.knob else "cmd", metric, ((a1, a2), (b1, b2)), res, a.threshold) + ctx.doc["data"]["decision"] = res + ctx.doc["data"]["line"] = line + ctx.doc["data"]["pairs"] = [{"pair": 1, "A": a1, "B": b1}, {"pair": 2, "A": a2, "B": b2}] + ctx.section("ab", "PASS" if not bad else "FAIL", line) + ctx.save() + + +def _save(ctx): + ctx.save() diff --git a/src/bench/macbench/abrule.py b/src/bench/macbench/abrule.py new file mode 100644 index 0000000..2b216c8 --- /dev/null +++ b/src/bench/macbench/abrule.py @@ -0,0 +1,60 @@ +"""The A/B decision rule (docs: README "A/B discipline"). + +Arms run interleaved A B B A inside one window; only the difference inside that window counts. Two pairs: +pair 1 = the first A with the first B, pair 2 = the second B with the second A. The rule is fixed BEFORE the run +(threshold on the command line) and has no re-run: a result below the bar, or with pairs that disagree, CLOSES +the lever.""" +from . import stats + +DEFAULT_THRESHOLD = 0.04 # the run-to-run noise on this stack + + +def pair_effects(a1, b1, a2, b2, higher_better=True): + """Relative improvement of B over A in each pair (positive = B better). None where A is 0 or missing.""" + def eff(a, b): + d = stats.rel_diff(a, b) + return None if d is None else (d if higher_better else -d) + return eff(a1, b1), eff(a2, b2) + + +def decide(a1, b1, a2, b2, higher_better=True, threshold=DEFAULT_THRESHOLD, tails=None, tail_tol=DEFAULT_THRESHOLD): + """tails: {name: (a1, b1, a2, b2, higher_better)} metrics that must not get worse (p95, max, gaps...). + Returns {"decision": "ADOPT"|"CLOSE", "effect", "pairs", "why": [...], "tails": {...}}""" + e1, e2 = pair_effects(a1, b1, a2, b2, higher_better) + why = [] + if e1 is None or e2 is None: + return {"decision": "CLOSE", "effect": None, "pairs": [e1, e2], "why": ["a pair has no usable value"], "tails": {}} + effect = (e1 + e2) / 2 + agree = (e1 > 0 and e2 > 0) or (e1 < 0 and e2 < 0) + adopt = True + if not agree: + adopt = False + why.append("the pairs disagree (%+.1f%%, %+.1f%%)" % (e1 * 100, e2 * 100)) + if effect <= threshold: + adopt = False + why.append("effect %+.1f%% does not exceed %.1f%%" % (effect * 100, threshold * 100)) + tail_out = {} + for name, (ta1, tb1, ta2, tb2, hb) in (tails or {}).items(): + t1, t2 = pair_effects(ta1, tb1, ta2, tb2, hb) + te = None if t1 is None or t2 is None else (t1 + t2) / 2 + worse = te is not None and te < -tail_tol + tail_out[name] = {"effect": te, "worse": worse} + if worse: + adopt = False + why.append("tail %s worsens %+.1f%%" % (name, te * 100)) + return {"decision": "ADOPT" if adopt else "CLOSE", "effect": effect, "pairs": [e1, e2], "why": why, "tails": tail_out} + + +def abba_order(): + return ["A", "B", "B", "A"] + + +def decision_line(knob, metric, arms, res, threshold): + """One human line, the one stored in the result.""" + f = lambda x: "n/a" if x is None else "%.2f" % x + pairs = ", ".join("n/a" if p is None else "%+.1f%%" % (p * 100) for p in res["pairs"]) + eff = "n/a" if res["effect"] is None else "%+.1f%%" % (res["effect"] * 100) + why = ("; " + "; ".join(res["why"])) if res["why"] else "" + return "AB %s: %s | %s A %s/%s B %s/%s | effect %s (pairs %s) | bar %.1f%%%s" % ( + knob, res["decision"], metric, f(arms[0][0]), f(arms[0][1]), f(arms[1][0]), f(arms[1][1]), eff, pairs, + threshold * 100, why) diff --git a/src/bench/macbench/analysis.py b/src/bench/macbench/analysis.py new file mode 100644 index 0000000..9b192d1 --- /dev/null +++ b/src/bench/macbench/analysis.py @@ -0,0 +1,183 @@ +"""Pure analysis of one measurement window: raw per-frame lines and probes in, per-simulator metrics out.""" +from . import stats, strip, targets + +LUMA_MID = 127 + + +SECOND_LOOK = -3 # simfps: the same surface looked at again (its marker changed since the first look), not a new picture + + +def parse_raw(text): + """simfps's SIMFPS_STAMPS file: ' ' per line -> [(t, counter, luma)].""" + out = [] + for line in text.splitlines(): + p = line.split() + if len(p) == 3: + try: + out.append((float(p[0]), int(p[1]), int(p[2]))) + except ValueError: + pass + return out + + +def probe_latencies(raw, injections, timeout_s=2.0): + """Input-latency PROXY for one simulator. + + raw: [(t, counter, luma)] ascending (the first line is the screen at the start); injections: [(t0, t_return)] + guest wall-clock seconds, t0 taken just before the injecting command was started. A probe is answered by the + first surface after t0 whose marker (luma above/below 127) differs from the last one before t0. No such + surface within timeout_s is a MISS; it counts at the timeout in the percentiles (never dropped). + What it includes: starting `simctl` (its own duration is `call_ms`), CoreSimulator, the app's handling and + one frame's composite. What it leaves out: the Device panel stream, the viewer, a real finger's HID path. + Returns {"n", "misses", "p50_ms", "p95_ms", "max_ms", "call_p50_ms", "latencies_ms": [...]}""" + lat, calls, misses, second = [], [], 0, 0 + for t0, tret in injections: + before = None + for t, _, luma in raw: + if t >= t0: + break + if luma >= 0: + before = luma > LUMA_MID + hit = None + if before is not None: + for t, c, luma in raw: + if t < t0 or luma < 0: + continue + if t > t0 + timeout_s: + break + if (luma > LUMA_MID) != before: + hit = t - t0 + second += c == SECOND_LOOK + break + calls.append((tret - t0) * 1000.0) + if hit is None: + misses += 1 + lat.append(timeout_s * 1000.0) + else: + lat.append(hit * 1000.0) + return {"n": len(injections), "misses": misses, "second_looks": second, "p50_ms": stats.r1(stats.percentile(lat, 0.5)), + "p95_ms": stats.r1(stats.percentile(lat, 0.95)), "max_ms": stats.r1(max(lat)) if lat else None, + "call_p50_ms": stats.r1(stats.percentile(calls, 0.5)), "latencies_ms": [round(x, 1) for x in lat]} + + +def sim_metrics(idx, role, raw, injections, nominal_fps=60.0, timeout_s=2.0, app_ticks=None, window=None): + """One simulator's entry in a step. A busy simulator gets frame metrics from its strip counters (distinct + frames), every simulator with probes gets the input proxy. `window` = (t_start, t_end) clips the raw lines.""" + if window: + raw = [r for r in raw if window[0] <= r[0] <= window[1]] or raw[:1] + sim = {"idx": idx, "role": role} + if role == "busy": + a = strip.analyze_counters([(t, c) for t, c, _ in raw if c != SECOND_LOOK]) + fm = stats.frame_stats([t for t, _ in a["unique"]], nominal_fps) + sim.update(fm) + sim["strip"] = {k: a[k] for k in ("repeats", "lost", "backwards", "jumps", "unreadable", "span_ticks")} + ok, why = strip.strip_gate(a, app_ticks) + sim["strip_gate"] = {"ok": ok, "why": why} + sim["app_ticks"] = app_ticks + if injections: + pl = probe_latencies(raw, injections, timeout_s) + sim.update({"input_n": pl["n"], "input_misses": pl["misses"], "input_second_looks": pl["second_looks"], "input_p50_ms": pl["p50_ms"], + "input_p95_ms": pl["p95_ms"], "input_max_ms": pl["max_ms"], "input_call_p50_ms": pl["call_p50_ms"]}) + return sim + + +def fairness(sims): + """Jain's index and min/max over the distinct fps of the simulators that were meant to animate.""" + f = [s["fps"] for s in sims if s.get("role") in ("busy", "agent") and s.get("fps") is not None] + return {"jain": stats.r1(stats.jain(f), 3), "min_max": stats.r1(stats.min_max_ratio(f), 3), "n": len(f)} + + +STANDING = ("qemu_cpu_cores", "gpu_busy_pct", "guest_cpu_total_pct", "guest_mem_used_mb", "guest_compressed_mb", + "host_mem_used_mb", "gpu_ms_per_frame", "sims_cpu_pct", "sims_rss_mb") + + +def flat_system(step): + """The step's system numbers as one flat {name: value} (nested dicts joined with no prefix clash).""" + out = {} + s = step.get("system", {}) + for k, v in s.items(): + if isinstance(v, (int, float)): + out[k] = v + elif isinstance(v, dict): + for k2, v2 in v.items(): + if isinstance(v2, (int, float)): + out[k + "_" + k2 if not k2.startswith(k) else k2] = v2 + return out + + +def standing_cost(steps, names=None): + """Per-simulator standing cost: the least-squares slope of each system number over N (units per simulator), + its intercept (the cost with no simulator) and R^2. Steps from `idle` give the pure standing cost.""" + xs = [s["n"] for s in steps] + flats = [flat_system(s) for s in steps] + keys = names or sorted({k for f in flats for k in f}) + out = {} + for k in keys: + b, a, r2 = stats.slope(xs, [f.get(k) for f in flats]) + if b is not None: + out[k] = {"per_sim": round(b, 3), "at_zero": round(a, 3), "r2": round(r2, 3)} + return out + + +def summarize_scale(steps, target_specs, stopped=None): + tg = [targets.parse_target(t) for t in target_specs] + mx = {} + for t in tg: + mx[t["name"]] = targets.max_sustainable(steps, t) + for st in steps: + st["targets"] = {t["name"]: targets.evaluate(t, st["sims"])["pass"] for t in tg} + st["fairness"] = fairness(st["sims"]) + return {"max_n": mx, "stopped": stopped, "standing_cost": standing_cost(steps), + "fairness_min_jain": min([s["fairness"]["jain"] for s in steps if s["fairness"]["jain"] is not None] or [None]) + if any(s["fairness"]["jain"] is not None for s in steps) else None} + + +def pace_summary(records, t_first, t_last, t_start, t_end): + """GlassBench's own tick log ([(at, counter, fps, ticks)] every ~5 s, from before the window to after it): + app_tick_fps is the mean of the records written inside the window (what the app produced, before the compositor); + app_ticks brackets the frames seen: the counter's distance from the last record at or before the first frame to + the first record at or after the last one (an upper bound for the ticks between the frames: about 5 s of slack + on each side). Either is None when the log does not reach.""" + inside = [r[2] for r in records if t_start <= r[0] <= t_end + 5 and r[2] > 0] + before = [r for r in records if r[0] <= t_first] + after = [r for r in records if r[0] >= t_last] + ticks = (after[0][1] - before[-1][1]) % strip.WRAP if before and after else None + return {"app_tick_fps": stats.r1(stats.mean(inside)), "app_ticks": ticks} + + +def agent_metrics(actions, raw): + """The agent mix's actions: launch call and launch-to-first-frame, screenshot duration (ms, p50/p95).""" + out = {} + launches = [a for a in actions if a["op"] == "launch" and a.get("rc") == 0] + shots = [a for a in actions if a["op"] == "screenshot" and a.get("rc") == 0] + ttff = [] + for a in launches: + nxt = next((r[0] for r in raw if r[0] >= a["t0"] and r[1] >= 0), None) + if nxt is not None: + ttff.append((nxt - a["t0"]) * 1000.0) + for name, vals in (("launch_to_frame_ms", ttff), ("screenshot_ms", [(a["t1"] - a["t0"]) * 1000.0 for a in shots]), + ("launch_call_ms", [(a["t1"] - a["t0"]) * 1000.0 for a in launches])): + out[name + "_p50"] = stats.r1(stats.percentile(vals, 0.5)) + out[name + "_p95"] = stats.r1(stats.percentile(vals, 0.95)) + out["actions"] = len(actions) + out["action_failures"] = sum(1 for a in actions if a.get("rc") not in (0, None)) + return out + + +def motion_metrics(times, nominal_fps=60.0, still_ms=250.0): + """frame metrics over the stretches of motion only (the agent mix idles between actions): fps = frames per + second inside stretches; the intervals are the ones inside stretches; the gap that ends a stretch is not one.""" + st = stats.stretches(times, still_ms) + ivs, frames, span = [], 0, 0.0 + for t0, t1, n in st: + span += t1 - t0 + frames += n - 1 + inside = [t for t in times] + for a, b in zip(inside, inside[1:]): + if (b - a) * 1000.0 <= still_ms: + ivs.append((b - a) * 1000.0) + thr = 2000.0 / nominal_fps + return {"stretches": len(st), "fps": stats.r1(frames / span if span > 0 else 0.0), "p50_ms": stats.r1(stats.percentile(ivs, 0.5)), + "p95_ms": stats.r1(stats.percentile(ivs, 0.95)), "p99_ms": stats.r1(stats.percentile(ivs, 0.99)), + "max_ms": stats.r1(max(ivs) if ivs else None), "over2x": sum(1 for x in ivs if x > thr), + "longest_gap_s": stats.r1(max(ivs) / 1000.0, 3) if ivs else None} diff --git a/src/bench/macbench/chart.py b/src/bench/macbench/chart.py new file mode 100644 index 0000000..b8e1b93 --- /dev/null +++ b/src/bench/macbench/chart.py @@ -0,0 +1,122 @@ +"""A self-contained HTML/SVG scaling chart from a scale result: no scripts, no fonts, no external files.""" +import html + +COLORS = ["#2a6fdb", "#d9534f", "#2e9e5b", "#e08a1e", "#8e5ac8", "#17a2b8", "#a0522d", "#6c757d"] +W, H, L, R, T, B = 460, 260, 52, 14, 28, 40 + + +def _nice(hi): + if hi <= 0: + return 1.0 + import math + e = 10 ** math.floor(math.log10(hi)) + for m in (1, 2, 2.5, 5, 10): + if hi <= m * e: + return m * e + return 10 * e + + +def panel(title, xs, series, ylabel, hlines=(), ymax=None): + """One SVG chart. series: [(name, [(x, y)], style)] style 'line' or 'dots'. hlines: [(y, label)].""" + pts = [p for _, ps, _ in series for p in ps if p[1] is not None] + hi = ymax or _nice(max([p[1] for p in pts] + [h[0] for h in hlines] + [0]) * 1.05 or 1) + x0, x1 = min(xs), max(xs) + span = (x1 - x0) or 1 + + def px(x): + return L + (x - x0) / span * (W - L - R) if x1 != x0 else (L + W - R) / 2 + + def py(y): + return H - B - (y / hi) * (H - T - B) + o = ['' % (W, H, W, H), + '' % (W, H), + '%s' % (L, html.escape(title))] + for i in range(5): + y = hi * i / 4 + o.append('' % (L, W - R, py(y), py(y))) + o.append('%g' % (L - 4, py(y) + 4, round(y, 2))) + for x in xs: + o.append('%g' % (px(x), H - B + 14, x)) + o.append('simulators' % ((L + W - R) // 2, H - 6)) + o.append('%s' % ((T + H - B) // 2 + 20, (T + H - B) // 2 + 20, html.escape(ylabel))) + for y, label in hlines: + o.append('' % (L, W - R, py(y), py(y))) + o.append('%s' % (W - R, py(y) - 3, html.escape(label))) + for i, (name, ps, style) in enumerate(series): + c = COLORS[i % len(COLORS)] + ps = [p for p in ps if p[1] is not None] + if style == "line" and len(ps) > 1: + o.append('' % (c, " ".join("%.1f,%.1f" % (px(x), py(y)) for x, y in ps))) + for x, y in ps: + o.append('%s N=%g: %g' % ( + px(x), py(y), 3 if style == "line" else 2.5, c, "1" if style == "line" else "0.55", html.escape(name), x, y)) + lx = L + 6 + for i, (name, ps, style) in enumerate(series): + o.append('%s' % (lx, T - 12 + 0, COLORS[i % len(COLORS)], lx + 12, T - 4, html.escape(name))) + lx += 12 + 6 * len(name) + 14 + o.append("") + return "\n".join(o) + + +def _per_sim(steps, key, role=("busy", "agent")): + pts = [] + for st in steps: + for s in st["sims"]: + if s.get("role") in role and s.get(key) is not None: + pts.append((st["n"], s[key])) + return pts + + +def _agg(steps, fn, key, role=("busy", "agent")): + out = [] + for st in steps: + v = [s[key] for s in st["sims"] if s.get("role") in role and s.get(key) is not None] + out.append((st["n"], fn(v) if v else None)) + return out + + +def _sys(steps, *path): + out = [] + for st in steps: + v = st["system"] + for p in path: + v = v.get(p) if isinstance(v, dict) else None + out.append((st["n"], v)) + return out + + +def scale_chart(doc): + d = doc["data"] + steps = sorted(d["steps"], key=lambda s: s["n"]) + xs = [s["n"] for s in steps] + nominal = d.get("nominal_fps", 60) + panels = [ + panel("distinct frames/s per simulator", xs, [("each sim", _per_sim(steps, "fps"), "dots"), + ("min", _agg(steps, min, "fps"), "line"), ("mean", _agg(steps, lambda v: sum(v) / len(v), "fps"), "line")], + "frames/s", hlines=[(nominal, "nominal %g" % nominal)]), + panel("frame interval p95 / p99 (ms)", xs, [("p95 each", _per_sim(steps, "p95_ms"), "dots"), + ("worst p95", _agg(steps, max, "p95_ms"), "line"), ("worst p99", _agg(steps, max, "p99_ms"), "line")], + "ms", hlines=[(25, "25 ms")]), + panel("input proxy p95 (ms)", xs, [("each sim", [(st["n"], s["input_p95_ms"]) for st in steps for s in st["sims"] if s.get("input_p95_ms") is not None], "dots"), + ("worst", [(st["n"], max([s["input_p95_ms"] for s in st["sims"] if s.get("input_p95_ms") is not None] or [None])) for st in steps], "line")], + "ms", hlines=[(500, "500 ms")]), + panel("QEMU CPU (cores) and GPU busy (%)", xs, [("QEMU cores", _sys(steps, "qemu_cpu_cores"), "line"), + ("GPU busy %", [(x, None if y is None else y / 10.0) for x, y in _sys(steps, "gpu_busy_pct")], "line")], + "cores | % / 10"), + panel("guest CPU of the simulators (%)", xs, [("simulators", _sys(steps, "guest", "sims_cpu_pct"), "line"), + ("everything else", _sys(steps, "guest", "other_cpu_pct"), "line")], "% of one core"), + panel("guest memory (MB)", xs, [("used", _sys(steps, "guest", "mem_used_mb"), "line"), + ("compressed", _sys(steps, "guest", "compressed_mb"), "line"), ("simulators RSS", _sys(steps, "guest", "sims_rss_mb"), "line")], "MB"), + panel("GPU ms per frame", xs, [("GPU ms/frame", _sys(steps, "gpu_ms_per_frame"), "line")], "ms"), + panel("fairness (Jain index x 100)", xs, [("Jain x100", [(st["n"], None if st.get("fairness", {}).get("jain") is None else st["fairness"]["jain"] * 100) for st in steps], "line"), + ("min/max x100", [(st["n"], None if st.get("fairness", {}).get("min_max") is None else st["fairness"]["min_max"] * 100) for st in steps], "line")], + "index x100", ymax=100), + ] + mx = d.get("summary", {}).get("max_n", {}) + rows = "".join("%s%s%s" % (html.escape(k), v.get("max_n"), v.get("first_fail")) for k, v in mx.items()) + return ('macbox bench %s' + '

macbox bench: %s, %sx, %s

' + '

run %s, build %s

' + '%s
targetmax sustainable Nfirst failing N
%s
' + % (html.escape(doc["id"]), html.escape(str(d.get("mix"))), d.get("scale_x"), html.escape(str(d.get("device", ""))), + html.escape(doc["id"]), html.escape(str(doc["env"].get("build_id"))), rows, "\n".join(panels))) diff --git a/src/bench/macbench/cli.py b/src/bench/macbench/cli.py new file mode 100644 index 0000000..3835029 --- /dev/null +++ b/src/bench/macbench/cli.py @@ -0,0 +1,402 @@ +"""`macbox bench `: argument parsing and the wiring of a run.""" +import argparse +import json +import os +import shlex +import shutil +import subprocess +import sys +import tempfile +import time + +from . import abrule, ab, chart, compare, core, fakes, hostprobe, others, scale, schema, targets +from .runctx import DeadlineError, Run +from .transport import GuestError, LocalGuest, SshGuest + +BENCH_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + + +def add_common(p): + p.add_argument("--out", default=os.environ.get("BENCH_OUT", "bench-results"), help="directory for the result file") + p.add_argument("--deadline", type=int, default=5400, help="hard deadline in seconds: cleanup runs and the tool exits (default 5400)") + p.add_argument("--tag", action="append", default=[], help="key=value recorded in the result (for example mesa=26.2.4)") + p.add_argument("--build-id", default=os.environ.get("BENCH_BUILD_ID"), help="the VM's build id (macbox bench fills it in)") + p.add_argument("--min-guest-free-gib", type=float, default=1.5, help="refuse to boot more simulators below this guest free memory") + p.add_argument("--min-host-free-gib", type=float, default=6.0, help="refuse to boot more simulators below this host available memory") + p.add_argument("--disk-path", default=os.environ.get("BENCH_DISK_PATH") or os.environ.get("BENCH_HOME"), + help="a path on the filesystem that holds the VM's storage and that your storage guard watches (default $BENCH_DISK_PATH, else the macbox home); a reading that fails refuses to run") + p.add_argument("--disk-start-gib", type=float, default=18.0, help="start a run only with this much free disk (GiB)") + p.add_argument("--disk-boot-gib", type=float, default=14.0, help="boot another simulator only with this much free disk (GiB)") + p.add_argument("--disk-abort-gib", type=float, default=11.0, help="below this the run is aborted and cleaned up while it waits (GiB)") + p.add_argument("--max-compressor-mb", type=float, default=2048.0, help="refuse to boot (abort while waiting) when the guest's compressor holds more than this") + p.add_argument("--watch-every", type=float, default=10.0, help=argparse.SUPPRESS) + p.add_argument("--watch-guest-every", type=float, default=30.0, help=argparse.SUPPRESS) + p.add_argument("--set-name", default=os.environ.get("BENCH_SET_NAME", "bench"), help="the bench's own device set in the guest (kept between runs: the seed lives in it)") + p.add_argument("--allow-foreign", action="store_true", help="go on although simulators outside the bench are booted (recorded)") + p.add_argument("--allow-restart", action="store_true", help="let a NEW device type restart CoreSimulatorService (only when nothing is booted)") + p.add_argument("--scale-x", dest="scale_x", type=int, choices=(1, 2, 3), default=1, help="device pixel scale: 1 = the macbox 1x type, 3 = the stock iPhone (default 1)") + p.add_argument("--secs", type=float, default=40, help="length of a measurement window, seconds") + p.add_argument("--nominal-fps", type=float, default=60.0) + p.add_argument("--probe-method", default="auto", choices=("auto", "notify", "appearance", "openurl", "cmd", "none"), help="input injection (auto: notify, else appearance, checked at the first N; openurl raises an alert on iOS 26 that stays over the app)") + p.add_argument("--probe-cmd", default=None, help="for --probe-method cmd: a shell command with {udid}; it must flip GlassBench's marker (a tap)") + p.add_argument("--probe-every", type=float, default=2.5, help="seconds between input probes of one simulator") + p.add_argument("--probe-timeout", type=float, default=2.0, help="an input probe not answered in this time is a miss (counted at this value)") + p.add_argument("--agent-period", type=float, default=12.0) + p.add_argument("--settle-max", type=float, default=300, help="a settle (the guest quiet for two samples) that reaches this many seconds FAILS the run") + p.add_argument("--lead", type=float, default=None, help=argparse.SUPPRESS) + p.add_argument("--settle-step", type=float, default=10.0, help=argparse.SUPPRESS) + p.add_argument("--settle-bg", type=float, default=40, help="the guest counts as settled below this background CPU %% (of one core)") + p.add_argument("--stock-devices", action="store_true", help="create FRESH STOCK devices instead of cloning the production seed: about 1.8 GB of guest disk each, not what an agent gets; for a device type Simlock has no source for or a scenario that needs a stock device (recorded in the result)") + p.add_argument("--rebuild-seed", action="store_true", help="delete the bench's seed and make a new one (after `macbox bench tools` rebuilt the app, or when it cannot be reused)") + p.add_argument("--source-set", default=os.environ.get("BENCH_SIMLOCK_SOURCES"), help="Simlock's source device set in the guest, read only (default ~/.simlock/devices/ios-sources)") + p.add_argument("--simslim", default=os.environ.get("BENCH_SIMSLIM"), help="simslim in the guest (default ~/go/bin/simslim)") + p.add_argument("--slim-profile", default=os.environ.get("BENCH_SLIM_PROFILE"), help="simslim profile in the guest (default: the only .json in ~/.config/simslim)") + p.add_argument("--seed-settle-max", type=float, default=600, help="the seed must settle within this many seconds or it is deleted and the run fails") + p.add_argument("--quiesce", action="store_true", help="disable mediaanalysisd in each simulator (recorded; production does not)") + p.add_argument("--strict-crashes", action="store_true", help="any new crash report on the Mac fails the run (default: only reports of the bench's programs and the simulator's graphics path; the others are listed)") + p.add_argument("--crash-names", default=os.environ.get("BENCH_CRASH_NAMES", ""), help="more process names whose crash report fails the run, comma-separated") + p.add_argument("--fail-on-swap", action="store_true", help="a guest swap-out in a window fails the memory gate (default: reported, host swap fails)") + p.add_argument("--golden-file", default=os.path.join(BENCH_DIR, "baselines", "golden.json")) + + +def parser(): + ap = argparse.ArgumentParser(prog="macbox bench", description="scaling, regression and A/B suite for the macbox simulator stack (src/bench/README.md)") + sp = ap.add_subparsers(dest="cmd", required=True) + s = sp.add_parser("scale", help="N = 1, 2, 3... simulators, per-simulator numbers, the maximum sustainable N per quality target") + add_common(s) + s.add_argument("--n", dest="ns", default="1,2,3,4,5,6", help="the N list, e.g. 1,2,3,4,6,8") + s.add_argument("--mix", choices=("all-busy", "one-busy", "agent", "idle"), default="all-busy") + s.add_argument("--targets", action="append", default=None, help="'name: fps>=55, p95<=25' (repeatable; default: smooth60, smooth30, responsive)") + s.add_argument("--floor", default="fps>=20", help="stop the sweep when a simulator breaches this (default fps>=20)") + s.add_argument("--shots", type=int, default=2, help="screenshots per simulator per step for the tear gate (0: none)") + s.add_argument("--no-observer-check", dest="observer_check", action="store_false", help="skip the strip-read cost window at the first N") + s.add_argument("--no-golden", dest="golden_phase", action="store_false", help="skip the static reference screen at the first N") + for name, hlp in (("seed", "make the production seed (or report the one that is there)"), ("idle", "the VM with no simulators: the standing cost"), ("lifecycle", "create, boot, install, launch, shut down, resume, delete; leak per cycle; boot storm"), + ("soak", "N busy simulators for minutes: memory growth, frame rate drift"), ("build", "build timing: the bench tools, optionally an xcodebuild"), + ("panel", "the Device panel's stream (optional, needs Simlock/T3 Code)"), ("golden", "record the static screen's pixel hash for this setup"), + ("ab", "A/B one knob, A B B A inside one window, ADOPT or CLOSE")): + p = sp.add_parser(name, help=hlp) + add_common(p) + if name == "idle": + p.add_argument("--bar-qemu-cores", type=float, default=0.15, help="PASS bar for QEMU's CPU in cores") + if name == "lifecycle": + p.add_argument("--cycles", type=int, default=5, help="create/boot/.../delete cycles (the leak gates need at least 4)") + p.add_argument("--storm", type=int, default=4, help="boot this many at once (0: skip)") + p.add_argument("--storm-bar-s", type=float, default=300) + p.add_argument("--leak-rss-mb", type=float, default=20, help="FAIL above this steady growth of QEMU's memory (RSS + swap) per cycle (MB)") + p.add_argument("--leak-guest-mb", type=float, default=30, help="FAIL above this guest memory growth per cycle (MB)") + if name == "soak": + p.add_argument("--soak-n", type=int, default=3) + p.add_argument("--soak-minutes", type=float, default=15) + p.add_argument("--soak-every", type=float, default=30) + p.add_argument("--soak-rss-mb-min", type=float, default=5, help="FAIL above this steady growth of QEMU's memory (RSS + swap) (MB/min)") + p.add_argument("--soak-guest-mb-min", type=float, default=10, help="FAIL above this guest memory growth (MB/min)") + if name == "build": + p.add_argument("--xcodebuild", default="", help="arguments for xcodebuild (a project or workspace and scheme) to time cold and warm") + if name == "panel": + p.add_argument("--panel-cmd", default=None, help="command printing {\"frames\": [[t, counter], ...]}") + p.add_argument("--panel-fps", type=float, default=55.0) + p.add_argument("--panel-p95", type=float, default=25.0) + if name == "ab": + p.add_argument("--knob", help="live:KEY=VALUE (a line in the VM's live switch file), or use --set-cmd/--unset-cmd") + p.add_argument("--set-cmd") + p.add_argument("--unset-cmd") + p.add_argument("--n", type=int, default=3, help="simulators") + p.add_argument("--mix", choices=("all-busy", "one-busy", "agent"), default="all-busy") + p.add_argument("--metric", choices=sorted(ab.METRICS), default="total_fps") + p.add_argument("--tails", default=None, help="comma list of metrics that may not worsen (default per metric)") + p.add_argument("--threshold", type=float, default=abrule.DEFAULT_THRESHOLD, help="effect needed to ADOPT, fixed BEFORE the run (default 0.04 = the run-to-run noise)") + p.add_argument("--tail-tol", type=float, default=abrule.DEFAULT_THRESHOLD) + p.add_argument("--arm-settle", type=float, default=10) + c = sp.add_parser("compare", help="two result files") + c.add_argument("a") + c.add_argument("b") + c.add_argument("--noise", type=float, default=compare.NOISE) + h = sp.add_parser("chart", help="an HTML/SVG scaling chart from a scale result") + h.add_argument("result") + h.add_argument("-o", "--output") + r = sp.add_parser("report", help="the PASS/FAIL lines of a result") + r.add_argument("result") + t = sp.add_parser("tools", help="copy the bench to the Mac and build simfps and GlassBench there") + t.add_argument("--no-build", action="store_true") + sp.add_parser("check", help="what this host and the VM offer the bench") + cl = sp.add_parser("cleanup", help="shut down and delete what an earlier run left in the bench's device set (seeds stay unless --seeds)") + cl.add_argument("--seeds", action="store_true", help="delete the seeds too (the next run makes a new one)") + cl.add_argument("--set-name", default=os.environ.get("BENCH_SET_NAME", "bench")) + return ap + + +def log(msg): + print("%s %s" % (time.strftime("%H:%M:%S"), msg), flush=True) + + +def qemu_pid_from_docker(): + cont = os.environ.get("MACBOX_CONTAINER", "macbox") + try: + out = subprocess.run([os.environ.get("MACBOX_DOCKER", "docker"), "top", cont, "-eo", "pid,args"], capture_output=True, text=True, timeout=15).stdout + except (OSError, subprocess.TimeoutExpired): + return None + for line in out.splitlines(): + if "qemu-system-x86_64" in line: + return int(line.split()[0]) + return None + + +class Golden: + def __init__(self, path): + self.path = path + try: + with open(path) as f: + self.d = json.load(f) + except (OSError, ValueError): + self.d = {} + + def key(self, ctx, w, h): + g = ctx.doc["env"].get("guest", {}) + return "%s|runtime %s|%s|%dx%d" % (g.get("macos"), g.get("runtime_used"), g.get("device_type"), w, h) + + def record(self, key, h): + self.d[key] = h + os.makedirs(os.path.dirname(self.path), exist_ok=True) + with open(self.path, "w") as f: + json.dump(self.d, f, indent=1, sort_keys=True) + f.write("\n") + + +def build_ctx(a, kind): + dry = os.environ.get("DRY") == "1" + run = Run(a.deadline, log, margin=float(os.environ.get("BENCH_DEADLINE_MARGIN", "60"))) + alias = os.environ.get("MACBOX_SSH_HOST", "") + clean = schema.scrubber([alias] if alias and alias != "macbox" else []) # not user names or the default alias: ordinary words + stamp = schema.stamp() + set_name = a.set_name # ONE set for every run: the seed made in it is kept and reused + faults = os.environ.get("BENCH_STUB_FAULT", "") + if dry: + tmp = os.environ.get("BENCH_DRY_KEEP") or tempfile.mkdtemp(prefix="macbench-dry-") + os.makedirs(tmp, exist_ok=True) + if not os.environ.get("BENCH_DRY_KEEP"): + run.teardown.insert(0, ("dry state directory", lambda: shutil.rmtree(tmp, ignore_errors=True))) + a.lead, a.settle_step, a.watch_every, a.watch_guest_every = 1.0, 0.3, 0.5, 1.0 + host = fakes.FakeHost(faults) + coh = fakes.FakeCoherent(host, faults) + guest = LocalGuest(run, set_name, os.path.join(tmp, "guest"), BENCH_DIR, faults) + guest.fault_ssh_timeout = int(os.environ.get("BENCH_STUB_SSH_TIMEOUT", "0")) or None + a.build_id = a.build_id or "dry-run" + live = os.path.join(tmp, "live.conf") + with open(live, "w") as f: + f.write("surface_coherent=1\n") + a.disk_path = tmp + else: + pid = int(os.environ.get("BENCH_QEMU_PID") or qemu_pid_from_docker() or 0) + if not pid: + raise SystemExit("macbox bench: no running QEMU (macbox up first)") + runp = os.environ.get("BENCH_RUN") or "" + live = os.path.join(runp, "live.conf") if runp and os.path.exists(os.path.join(runp, "live.conf")) else None + host = hostprobe.Host(pid, live_conf=live) + from .gates import Coherent + census = hostprobe.qemu_env_value(pid, "REIMS_VGPU_CENSUS") # the path QEMU was given, if it exists here too + if not (census and os.path.isdir(census)): + census = os.path.join(runp, "coherent") if runp else None + coh = Coherent(census, os.environ.get("BENCH_QEMU_LOG"), + os.path.join(os.environ.get("BENCH_HOME", ""), "home/.cache/m2v-gpu/hang") if os.environ.get("BENCH_HOME") else None) + guest = SshGuest(run, os.environ.get("BENCH_SSH") or os.environ.get("MACBOX_SSH_HOST") or "macbox", set_name) + os.makedirs(a.out, exist_ok=True) + params = {k: v for k, v in vars(a).items() if k not in ("cmd", "out", "golden_file", "disk_path", "tag") and not callable(v)} + doc = schema.new_result(kind if kind in schema.KINDS else "scale", params, {"host": {}, "guest": {}}, stamp) + if kind == "scale": + doc["data"] = {"mix": a.mix, "scale_x": a.scale_x, "steps": [], "summary": {"max_n": {}}} + path = os.path.join(a.out, "%s-%s%s.json" % (kind, stamp, "-dry" if dry else "")) + ctx = core.Ctx(a, run, guest, host, coh, doc, path, clean, log) + ctx.live_conf = live + gold = Golden(a.golden_file) + ctx.golden_store = gold + ctx.golden = lambda w, h: (gold.d.get(gold.key(ctx, w, h)), gold.key(ctx, w, h)) + ctx.golden_key_for = lambda w, h: gold.key(ctx, w, h) + ctx.record_golden = gold.record + if dry: + ctx.dry_dir = tmp + return ctx + + +def execute(a, kind, body): + ctx = build_ctx(a, kind) + run = ctx.run + run.install() + run.on_abort = lambda reason: ctx.section("abort", "FAIL", "ABORT: %s" % reason) + run.final = lambda: finalize(ctx) + core.monitor(ctx) + if ctx.host.dry: + log("DRY=1: stub guest, fake host; nothing real is touched") + rc = 0 + try: + try: + body(ctx) + finally: + if not run.aborted: # a monitor that ended the run owns the result: no one else writes into it + core.run_gates(ctx) # every scenario: Coherent, kernel GPU lines and crash reports since the run began + except core.Stop as e: + if not run.aborted: + ctx.section("stopped", "FAIL", "STOPPED: %s" % e) + rc = 2 + except (GuestError, DeadlineError) as e: + if not run.aborted: + ctx.section("error", "FAIL", "ERROR: %s: %s" % (type(e).__name__, e)) + rc = 2 + finally: + if not run.aborted: + run.cleanup() + run.join_abort() # when a monitor aborted the run it writes the result and owns the exit code + rc = max(rc, finalize(ctx)) + fails = [s for s in ctx.doc["sections"] if s["status"] == "FAIL"] + if fails: + return max(rc, 1) + return rc + + +CLEANUP_FAILED_RC = 20 + + +def finalize(ctx): + """After the cleanup: its section (PASS only when verified), the idle guard's, the result file. Also what the abort + path of the run's monitor calls. Returns CLEANUP_FAILED_RC when the cleanup could not delete everything.""" + run, info = ctx.run, ctx.cleanup_info + rc = 0 + if ctx.taken or info or run.problems: + left_ok = bool(info) and not info.get("failed") and info.get("booted", 0) == 0 and info.get("left", 0) == 0 + sec = "PASS" if left_ok and not run.problems else "FAIL" + if sec == "FAIL": + rc = CLEANUP_FAILED_RC + ctx.section("cleanup", sec, "CLEANUP %s: bench simulators left booted %s, devices left %s, knobs restored %s%s" % ( + sec, info.get("booted", "n/a"), info.get("left", "n/a"), "yes" if not run.problems else "NO", + ("; " + "; ".join(run.problems)) if run.problems else "")) + if ctx.idle_guard: + absent = "absent" in ctx.idle_guard + ctx.doc["env"].setdefault("guest", {})["idle_guard"] = "absent" if absent else "kept" + ctx.section("idle_guard", "INFO", ("IDLE GUARD ABSENT: sim-idle-guard is not installed on the Mac, nothing keeps the simulators from an idle shutdown; " + "the run counts its booted simulators before every window") if absent else "IDLE GUARD kept every bench simulator for the run") + problems = ctx.save() + print("result: %s%s" % (os.path.basename(ctx.out_path), "" if not problems else " (NOT publishable as is: %s)" % "; ".join(problems[:3]))) + return rc + + +def main(argv=None): + a = parser().parse_args(argv) + c = a.cmd + if c in ("compare", "report", "chart"): + return offline(a) + if c == "tools": + return tools(a) + if c == "check": + return check(a) + if c == "cleanup": + return cleanup(a) + if c == "scale": + a.ns = [int(x) for x in a.ns.split(",")] + a.targets = a.targets or targets.DEFAULT_TARGETS + for t in a.targets + ["floor: " + a.floor]: + targets.parse_target(t) + return execute(a, "scale", scale.run_scale) + body = {"seed": others.run_seed, "idle": others.run_idle, "lifecycle": others.run_lifecycle, "soak": others.run_soak, "build": others.run_build, + "panel": others.run_panel, "golden": others.run_golden, "ab": ab.run_ab}[c] + for k, v in (("ns", [1]), ("mix", "all-busy"), ("targets", targets.DEFAULT_TARGETS), ("floor", "fps>=20"), ("shots", 2), + ("observer_check", False), ("golden_phase", False)): + if not hasattr(a, k): + setattr(a, k, v) + return execute(a, {"golden": "gates"}.get(c, c), body) + + +def cleanup(a): + """macbox bench cleanup: shut down and delete what an earlier run left in the bench's set; exit 0 only when verified.""" + run = Run(900, log) + faults = os.environ.get("BENCH_STUB_FAULT", "") + if os.environ.get("DRY") == "1": + keep = os.environ.get("BENCH_DRY_KEEP") + if not keep: + raise SystemExit("DRY cleanup needs BENCH_DRY_KEEP (the stub guest's state directory)") + guest = LocalGuest(run, a.set_name, os.path.join(keep, "guest"), BENCH_DIR, faults) + else: + guest = SshGuest(run, os.environ.get("BENCH_SSH") or os.environ.get("MACBOX_SSH_HOST") or "macbox", a.set_name) + try: + guest.call(["init-set"], 60) + info = guest.call(["teardown"] + (["--seeds"] if a.seeds else []), 300) + except (GuestError, DeadlineError) as e: + print("CLEANUP FAIL: the guest could not be reached or refused: %s" % e) + return CLEANUP_FAILED_RC + ok = not (info.get("failed") or info.get("left") or info.get("booted")) + print("CLEANUP %s: devices left %s, booted %s, seeds kept %s%s" % ("PASS" if ok else "FAIL", info.get("left"), info.get("booted"), info.get("seeds"), + ("; could not: " + "; ".join(info["failed"])) if info.get("failed") else "")) + return 0 if ok else CLEANUP_FAILED_RC + + +def offline(a): + if a.cmd == "compare": + text, worse = compare.compare(schema.load(a.a), schema.load(a.b), a.noise) + print(text) + return 1 if worse else 0 + doc = schema.load(a.result) + if a.cmd == "report": + print("%s %s %s (build %s)" % (doc["kind"], doc["id"], doc["created"], doc["env"].get("build_id"))) + for s in doc["sections"]: + print("%-4s %s" % (s["status"], s["line"])) + return 1 if any(s["status"] == "FAIL" for s in doc["sections"]) else 0 + if doc["kind"] != "scale": + print("chart: only scale results have a scaling chart", file=sys.stderr) + return 2 + out = a.output or os.path.splitext(a.result)[0] + ".html" + with open(out, "w") as f: + f.write(chart.scale_chart(doc)) + print(out) + return 0 + + +def tools(a): + ssh = os.environ.get("BENCH_SSH") or os.environ.get("MACBOX_SSH_HOST") or "macbox" + if os.environ.get("DRY") == "1": + print("DRY=1: would copy %s to the guest's ~/.macbox/bench and run build.sh" % BENCH_DIR) + return 0 + tar = subprocess.Popen(["tar", "-C", BENCH_DIR, "--exclude=__pycache__", "--exclude=tests", "--exclude=baselines", "-cf", "-", "."], stdout=subprocess.PIPE) + r = subprocess.run(["ssh", ssh, "mkdir -p ~/.macbox/bench && tar -C ~/.macbox/bench -xf -"], stdin=tar.stdout) + tar.wait() + if r.returncode: + return 2 + if a.no_build: + return 0 + return subprocess.run(["ssh", ssh, "cd ~/.macbox/bench && ./build.sh"]).returncode + + +def check(a): + pid = int(os.environ.get("BENCH_QEMU_PID") or qemu_pid_from_docker() or 0) + ok = True + + def line(good, text): + nonlocal ok + ok &= good + print("%s %s" % ("ok " if good else "MISS", text)) + line(bool(pid), "QEMU running (pid known)") + if pid: + hp = hostprobe.Host(pid, live_conf=os.path.join(os.environ.get("BENCH_RUN", ""), "live.conf")) + s = hp.snap() + line(bool(s["threads"]), "QEMU thread CPU readable (%d threads)" % len(s["threads"])) + line(s["gpu"] is not None, "GPU render time readable from DRM fdinfo (i915 drm-engine-render)") + line(s["mem"].get("cg_current") is not None, "the VM's cgroup memory readable") + n, note = hostprobe.kernel_gpu_lines(time.time() - 60) + line(n is not None, "kernel journal readable for GPU error lines" + ("" if n is not None else " (%s)" % note)) + line(hp.mem_avail_gb() is not None, "host available memory readable") + ssh = os.environ.get("BENCH_SSH") or os.environ.get("MACBOX_SSH_HOST") or "macbox" + try: + src = os.environ.get("BENCH_SIMLOCK_SOURCES") + out = subprocess.run(["ssh", "-o", "ConnectTimeout=8", "-o", "BatchMode=yes", ssh, "cd ~/.macbox/bench && %sxcrun python3 guest/benchguest.py info" % ( + "BENCH_SIMLOCK_SOURCES=%s " % shlex.quote(src) if src else "")], + capture_output=True, text=True, timeout=60).stdout + d = json.loads(out.strip().splitlines()[-1]) + line(True, "guest answers: macOS %s, %s" % (d.get("macos"), d.get("xcode"))) + line(d.get("has_simfps"), "simfps built in the guest (macbox bench tools)") + line(d.get("has_app"), "GlassBench.app built in the guest (macbox bench tools)") + types = d.get("simlock_types") or [] + line(bool(types), "Simlock's source devices found (production simulators: %s)%s" % (", ".join(types) or "none", + "" if types else "; without them only --stock-devices works")) + line(d.get("has_simslim") and d.get("has_slim_profile"), "simslim and its profile in the guest (the seed is made Slim with them)") + line(d.get("has_idle_guard"), "sim-idle-guard in the guest (keeps the bench's simulators from its 15-minute shutdown)") + except (OSError, ValueError, IndexError, subprocess.TimeoutExpired): + line(False, "the guest's bench helper (macbox bench tools)") + return 0 if ok else 1 diff --git a/src/bench/macbench/compare.py b/src/bench/macbench/compare.py new file mode 100644 index 0000000..0234970 --- /dev/null +++ b/src/bench/macbench/compare.py @@ -0,0 +1,107 @@ +"""compare two result files: what moved between a baseline and a candidate, beyond the noise.""" +from . import stats + +NOISE = 0.04 # run-to-run noise on this stack + +# (label, getter on a step, higher is better) +STEP_ROWS = [ + ("min distinct fps", lambda st: _min([s.get("fps") for s in st["sims"] if s.get("role") in ("busy", "agent")]), True), + ("worst p95 ms", lambda st: _max([s.get("p95_ms") for s in st["sims"] if s.get("role") in ("busy", "agent")]), False), + ("worst p99 ms", lambda st: _max([s.get("p99_ms") for s in st["sims"] if s.get("role") in ("busy", "agent")]), False), + ("longest gap s", lambda st: _max([s.get("longest_gap_s") for s in st["sims"]]), False), + ("input p95 ms", lambda st: _max([s.get("input_p95_ms") for s in st["sims"]]), False), + ("Jain index", lambda st: (st.get("fairness") or {}).get("jain"), True), + ("QEMU cores", lambda st: st["system"].get("qemu_cpu_cores"), False), + ("GPU busy %", lambda st: st["system"].get("gpu_busy_pct"), False), + ("GPU ms/frame", lambda st: st["system"].get("gpu_ms_per_frame"), False), + ("guest mem used MB", lambda st: (st["system"].get("guest") or {}).get("mem_used_mb"), False), +] + + +def _vals(xs): + return [x for x in xs if x is not None] + + +def _min(xs): + v = _vals(xs) + return min(v) if v else None + + +def _max(xs): + v = _vals(xs) + return max(v) if v else None + + +def verdict(a, b, higher_better, noise=NOISE): + d = stats.rel_diff(a, b) + if d is None: + return "n/a", None + if abs(d) <= noise: + return "same", d + better = d > 0 if higher_better else d < 0 + return ("better" if better else "WORSE"), d + + +def compare_scale(a, b, noise=NOISE): + """Lines and a worse-count for two scale results. Rows are per N present in both.""" + lines, worse = [], 0 + da, db = a["data"], b["data"] + lines.append("A %s B %s mix %s/%s scale %sx/%sx noise bar %.0f%%" % ( + a["id"], b["id"], da.get("mix"), db.get("mix"), da.get("scale_x"), db.get("scale_x"), noise * 100)) + if (da.get("mix"), da.get("scale_x")) != (db.get("mix"), db.get("scale_x")): + lines.append(" warning: different mix or scale: the numbers are not comparable") + src = [(r.get("env", {}).get("guest") or {}).get("device_source") for r in (a, b)] + if src[0] != src[1]: + lines.append(" warning: the simulators were not made the same way (A: %s, B: %s): the numbers are not comparable" % tuple(src)) + for name in sorted(set(da["summary"]["max_n"]) | set(db["summary"]["max_n"])): + ma = (da["summary"]["max_n"].get(name) or {}).get("max_n") + mb = (db["summary"]["max_n"].get(name) or {}).get("max_n") + tag = "same" if ma == mb else ("better" if (mb or 0) > (ma or 0) else "WORSE") + worse += tag == "WORSE" + lines.append(" max sustainable N %-12s A %s B %s %s" % (name, ma, mb, tag)) + sa = {s["n"]: s for s in da["steps"]} + sb = {s["n"]: s for s in db["steps"]} + for n in sorted(set(sa) & set(sb)): + lines.append(" N=%d" % n) + for label, get, hb in STEP_ROWS: + try: + x, y = get(sa[n]), get(sb[n]) + except (KeyError, TypeError): + continue + v, d = verdict(x, y, hb, noise) + if v == "n/a": + continue + worse += v == "WORSE" + lines.append(" %-18s %10s -> %-10s %+6.1f%% %s" % (label, _f(x), _f(y), d * 100, v)) + return lines, worse + + +def _f(x): + return "n/a" if x is None else ("%.2f" % x if abs(x) < 100 else "%.0f" % x) + + +def compare_sections(a, b): + sa = {s["name"]: s["status"] for s in a["sections"]} + out = [] + for s in b["sections"]: + was = sa.get(s["name"]) + mark = "" if was == s["status"] else " (was %s)" % was + out.append(" %s %s%s" % (s["status"], s["line"], mark)) + return out + + +def compare(a, b, noise=NOISE): + """(text, regressions): regressions counts rows that got worse beyond the noise bar and sections that + went from PASS to FAIL.""" + if a.get("kind") != b.get("kind"): + return "different kinds: %s vs %s" % (a.get("kind"), b.get("kind")), 1 + lines, worse = [], 0 + if a["kind"] == "scale": + l, worse = compare_scale(a, b, noise) + lines += l + lines.append("sections of B:") + lines += compare_sections(a, b) + sa = {s["name"]: s["status"] for s in a["sections"]} + worse += sum(1 for s in b["sections"] if sa.get(s["name"]) == "PASS" and s["status"] == "FAIL") + lines.append("%d regression(s) beyond the %.0f%% bar" % (worse, noise * 100)) + return "\n".join(lines), worse diff --git a/src/bench/macbench/core.py b/src/bench/macbench/core.py new file mode 100644 index 0000000..9b10c30 --- /dev/null +++ b/src/bench/macbench/core.py @@ -0,0 +1,519 @@ +"""The parts every scenario shares: the context, the guard, simulator preparation, one measurement window and the +step record built from it.""" +import time + +from . import analysis, gates, hostprobe, schema, stats, targets + + +class Stop(Exception): + """A reason to end the sweep early; the result so far is kept.""" + + +class Ctx: + def __init__(self, args, run, guest, host, coherent, doc, out_path, clean, log=print): + self.a, self.run, self.guest, self.host, self.coh = args, run, guest, host, coherent + self.doc, self.out_path, self.clean, self.log = doc, out_path, clean, log + self.offset = 0.0 + self.sims = [] # [{"udid", "idx"}] booted and prepared, in order + self.type_id = self.runtime = None + self.probe_method = None + self.cleanup_info = {} + self.live_conf = None + self.est_sim_mb = 1100.0 # a simulator's guest memory, until measured + self.marks = {} + self.golden = None # fn(w, h) -> (hash or None, key) + self.pixel_gate = None + self.stock = False # --stock-devices: fresh stock devices instead of clones of the seed + self.seed = None # the guest's answer to `seed` + self.prefix = "bench" + self.swap0 = None # the guest's swap-out counter when the run began + self.in_window = False # a measurement window is running: the monitors keep off the guest + self.taken = False # the bench's set is owned by this run (lock taken, cleanup registered) + self.idle_guard = set() # what sim-idle-guard keep answered for the bench's simulators: kept / absent + + def save(self): + return schema.write(self.doc, self.out_path, self.clean) + + def section(self, name, status, line, detail=None): + schema.add_section(self.doc, name, status, line, detail) + self.log("%s %s" % (status, line)) + + +def capture_env(ctx, info): + host, sw, qenv = ctx.host.env() + g = {k: info.get(k) for k in ("macos", "xcode", "arch", "cpus", "mem_mb")} + rts = info.get("runtimes") or [] + g["runtimes"] = ["%s %s" % (r.get("name"), r.get("build")) for r in rts] + ctx.doc["env"].setdefault("guest", {}).update(g) # keeps what setup_guest recorded (runtime, device type, seed) + ctx.doc["env"].update({"build_id": ctx.a.build_id or "unknown", "host": host, + "switches": sw, "qemu_env": qenv, "tags": dict(kv.split("=", 1) for kv in (ctx.a.tag or []) if "=" in kv), + "dry": bool(ctx.host.dry)}) + + +def disk_reason(ctx, floor_gib, what, label=""): + """None when the filesystem that holds the VM's storage has `floor_gib` free; else why not. A reading that fails refuses + (a guard that cannot see the disk is not a guard): that is what let the 2026-11-16 outage through.""" + df = ctx.host.disk_free_gb(ctx.a.disk_path or "/") + if df is None: + return "free disk space of the VM's storage (--disk-path) cannot be read: refusing to %s %s" % (what, label) + if df < floor_gib: + return "free disk %.1f GiB is under the %g GiB floor to %s %s" % (df, floor_gib, what, label) + return None + + +def guard(ctx, k_new, label): + """Refuses to boot k_new more simulators when the host or the guest would run short: returns a reason or None. Checked + before every boot (and by the monitor while waiting). The sweep stops there; it never swaps the VM to death. + Host: free disk (>= --disk-boot-gib to boot another), available memory. Guest: available memory after the new + simulators, memory pressure, swap-outs since the run began (must not grow), the compressor's size, simulators that + are not the bench's.""" + a = ctx.a + hm = ctx.host.mem_avail_gb() + if hm is not None and hm - k_new * 0.5 < a.min_host_free_gib: + return "host free memory %.1f GiB below the %.1f GiB floor %s" % (hm, a.min_host_free_gib, label) + why = disk_reason(ctx, a.disk_boot_gib, "boot another simulator", label) + if why: + return why + m = ctx.guest.call(["memcheck"], 60) + avail = m["mem"].get("avail_mb", m["mem"]["free_mb"]) # free + speculative + inactive + purgeable: what a boot can take + free_gib = (avail - k_new * ctx.est_sim_mb) / 1024.0 + if free_gib < a.min_guest_free_gib: + return "guest available memory would be %.1f GiB (%.1f now, %d x ~%d MB to boot), below the %.1f GiB floor %s" % ( + free_gib, avail / 1024.0, k_new, ctx.est_sim_mb, a.min_guest_free_gib, label) + if (m.get("pressure") or 1) >= 2: + return "guest memory pressure level %s %s" % (m.get("pressure"), label) + mem_why = guest_memory_reason(ctx, m["mem"]) + if mem_why: + return "%s %s" % (mem_why, label) + if m.get("foreign_booted", 0) > 0 and not a.allow_foreign: + return "%d simulator(s) outside the bench are booted (the pool or someone's lease): not a clean measurement; --allow-foreign to go on" % m["foreign_booted"] + return None + + +def guest_memory_reason(ctx, mem): + """Swap-outs since the run began (the guest started paging memory out: the simulators no longer fit) and the compressor's size.""" + base = ctx.swap0 + if base is not None and mem.get("swapouts", 0) > base: + return "guest swap-outs grew from %d to %d since the run began" % (base, mem["swapouts"]) + if mem.get("compressed_mb", 0) > ctx.a.max_compressor_mb: + return "guest compressor holds %.0f MB (limit %g MB)" % (mem["compressed_mb"], ctx.a.max_compressor_mb) + return None + + +def start_reason(ctx): + """Before the first simulator of a run: the host disk has --disk-start-gib free.""" + return disk_reason(ctx, ctx.a.disk_start_gib, "start a run that boots simulators", "") + + +def monitor(ctx): + ctx.run.watch("guards", monitor_check(ctx), ctx.a.watch_every) + + +def monitor_check(ctx): + """While the run waits (a boot, a settle, a window): the host's disk, QEMU and, outside measurement windows (their + own calls would be observer load), the Mac. Any breach aborts the run NOW: the monitor thread writes the partial + result, runs the one cleanup and exits (8 disk, 9 VM, 7 someone else's simulator, 10 guest memory).""" + a, st = ctx.a, {"dead": 0, "foreign": 0, "last_guest": 0.0} + + def check(): + why = disk_reason(ctx, a.disk_abort_gib, "go on", "") + if why: + return "%s; deleting the bench's simulators" % why, 8 + if not ctx.host.same_qemu(): + return "the VM's QEMU process is gone or was restarted", 9 + now = time.monotonic() + if ctx.in_window or not ctx.taken or now - st["last_guest"] < a.watch_guest_every: + return None + st["last_guest"] = now + if not ctx.guest.alive(): + st["dead"] += 1 + if st["dead"] >= 2: + return "the VM stopped answering over ssh (two checks in a row)", 9 + return None + st["dead"] = 0 + m = ctx.guest.quick(["memcheck"], 60) + if m.get("foreign_booted", 0) > 0 and not a.allow_foreign: + st["foreign"] += 1 # the bench's own boots and shutdowns race the count: two polls in a row + if st["foreign"] >= 2: + return "a simulator that is not the bench's is booted (someone needs the Mac): yielding", 7 + else: + st["foreign"] = 0 + why = guest_memory_reason(ctx, m["mem"]) + if why: + return why, 10 + return None + return check + + +def roles_for(mix, n): + if mix == "all-busy": + return ["busy"] * n + if mix == "one-busy": + return ["busy"] + ["static"] * (n - 1) + if mix == "agent": + return ["agent"] * n + if mix == "idle": + return ["idle"] * n + raise ValueError("unknown mix %r" % mix) + + +def teardown(ctx): + """The cleanup of the bench's simulators. Raises, naming what is left, when anything could not be deleted: the + cleanup section must never say 'done' on a cleanup that was not verified.""" + info = ctx.guest.call(["teardown"], 300) + ctx.cleanup_info.update(info) + if info.get("failed") or info.get("left") or info.get("booted"): + raise RuntimeError("could not clean up the bench's device set: %s; %d device(s) left, %d booted: run `macbox bench cleanup` " + "(the next run refuses to start until it is clean)" % ("; ".join(info.get("failed") or ["nothing reported"])[:300], info.get("left", 0), info.get("booted", 0))) + + +def keep_minutes(ctx): + """How long the Mac's idle guard must leave the bench's simulators alone: the run's deadline plus 5 minutes, at most 240.""" + return max(5, min(240, int(ctx.a.deadline / 60.0) + 5)) + + +def setup_guest(ctx, sims=True): + """Device set, device type, tools check. Returns the guest info. With sims=False the run boots no simulator of its + own (idle, build): the set is not taken and nothing is registered for cleanup.""" + a = ctx.a + info = ctx.guest.call(["info"], 120) + if not info.get("has_simfps") or not info.get("has_app"): + raise Stop("the bench tools are not built in the guest: macbox bench tools") + ctx.guest.call(["init-set"], 60) + if not sims: + _run_mark(ctx) + return info + why = start_reason(ctx) + if why: + raise Stop(why) + acq = ctx.guest.call(["acquire", "--run", ctx.doc["id"]], 60) + if not acq.get("ok"): + # refuse BEFORE the cleanup is registered: the leftovers are not this run's to delete (macbox bench cleanup does) + raise Stop("an earlier run left the bench's device set unclean (%s): run `macbox bench cleanup`, then start again" % "; ".join(acq.get("leftovers", [])[:4])) + ctx.run.teardown.append(("bench simulators", lambda: teardown(ctx))) + ctx.taken = True + t = ctx.guest.call(["ensure-type", "--scale", str(a.scale_x)] + (["--allow-restart"] if a.allow_restart else []), 180) + if t.get("needs_restart"): + raise Stop("device type %s was just created and CoreSimulatorService must restart to load it, which would shut " + "down every booted simulator: stop them, then run again with --allow-restart (or `macbox pool create`)" % t["type"]) + ctx.type_id, ctx.runtime = t["type"], t["runtime"] + base = "stock %s" % t["stock_name"] if a.scale_x == 3 else "macbox %dx of %s" % (a.scale_x, t["stock_name"]) + g = ctx.doc["env"]["guest"] + g["runtime_used"] = t.get("runtime_version") + g["device_type"] = base + (" (fresh stock devices)" if a.stock_devices else " (production seed)") + g["device_source"] = "stock devices" if a.stock_devices else "clones of a seed made from a Simlock source device" + ctx.stock = bool(a.stock_devices) + ctx.prefix = "bench-%s" % ctx.doc["id"] + _run_mark(ctx) + if not ctx.stock: + make_seed(ctx) + return info + + +def make_seed(ctx): + """The production seed: a settled Slim clone of Simlock's own source device of this type, kept in the bench's set + (reused when it is there). Every bench simulator is a clone of it. Fails closed: no source, no Slim, no settle, no run.""" + a = ctx.a + reason = guard(ctx, 1, "before the seed") + if reason: + raise Stop(reason) + args = ["seed", "--type", ctx.type_id, "--keep-min", keep_minutes(ctx), "--settle-max", a.seed_settle_max, "--step", a.settle_step] + for flag, val in (("--source-set", a.source_set), ("--simslim", a.simslim), ("--slim-profile", a.slim_profile)): + if val: + args += [flag, val] + if a.rebuild_seed: + args.append("--rebuild") + r = ctx.guest.call(args, a.seed_settle_max + 2400) + if not r.get("seed"): + raise Stop("%s. A bench simulator must be a production simulator (a clone of a Simlock source device with the Slim disables), " + "so there is nothing to run on. For a device type Simlock has no source for, or a scenario that needs a stock device, " + "use --stock-devices: it creates fresh stock devices (about 1.8 GB of guest disk each, the host-disk guard checks " + "before each) and the result says so" % r.get("no_source", "no seed")) + ctx.seed = r + ctx.runtime = r.get("runtime") or ctx.runtime + g = ctx.doc["env"]["guest"] + g["runtime_used"] = r.get("runtime_version") or g.get("runtime_used") + g["seed"] = {k: r.get(k) for k in ("source_type", "reused", "made_s", "services_off", "settle_s", "settle_bg_pct", "icons")} + ctx.section("seed", "INFO", "SEED %s: %s services off (Slim), %s" % ( + "reused" if r.get("reused") else "made from a Simlock source device in %s s (simslim %s s, settled in %s s at %s%% background)" % ( + r.get("made_s"), r.get("slimmed_s"), r.get("settle_s"), r.get("settle_bg_pct")), + r.get("services_off"), "kept shut down between runs" if not r.get("reused") else "settled when it was made")) + + +def _run_mark(ctx): + ctx.swap0 = ctx.guest.call(["memcheck"], 60)["mem"].get("swapouts", 0) # swap-outs may not grow from here on + ctx.offset = ctx.guest.clock_offset() + ctx.run_mark = {"coh": ctx.coh.mark(), "t0": time.time(), "ips": set(ctx.guest.call(["ips"], 60)["ips"])} + + +def grow(ctx, n, roles, install=True, parallel=False): + """Make and boot simulators up to n (the ones already running stay), launch each with its role. Returns boot timings. + Production path: clones of the seed (no disk cost, the seed's Slim disables copied to each). --stock-devices: fresh + stock devices, one at a time with the guard (host disk, guest memory) in front of each. parallel=True boots all the + new ones at once (the lifecycle's boot storm); otherwise they boot one after the other with the guard between.""" + have = len(ctx.sims) + new = n - have + boot = {} + if new > 0: + keep = keep_minutes(ctx) + done, usable = [], [] + t0 = time.time() + if parallel and not ctx.stock: + r = ctx.guest.call(["clone", "--seed", ctx.seed["seed"], "--n", new, "--start", have, "--prefix", ctx.prefix, "--keep-min", keep], 240 + 60 * new) + fresh = r["sims"] + b = ctx.guest.call(["boot", "--udids", ",".join(x["udid"] for x in fresh)] + ["--parallel"], 900) + groups = [(fresh, b)] + else: + groups = [] + for k in range(new): + if k: + reason = guard(ctx, 1, "before booting simulator %d" % (have + k + 1)) + if reason: + raise Stop(reason) + if ctx.stock: + r = ctx.guest.call(["create", "--n", 1, "--type", ctx.type_id, "--runtime", ctx.runtime, "--start", have + k, "--keep-min", keep], 240) + else: + r = ctx.guest.call(["clone", "--seed", ctx.seed["seed"], "--n", 1, "--start", have + k, "--prefix", ctx.prefix, "--keep-min", keep], 240) + groups.append((r["sims"], ctx.guest.call(["boot", "--udids", r["sims"][0]["udid"]], 900))) + for fresh, b in groups: + ctx.idle_guard.update(x.get("idle_guard") for x in fresh) + bad = [s for s in b["sims"] if not s.get("ok")] + if bad: + for s_, bs in zip(fresh, b["sims"]): + ctx.sims.append({"idx": s_["idx"], "udid": s_["udid"]}) # known to the cleanup and the count even when it failed + raise Stop("a simulator did not boot: %s" % bad[0].get("err")) + for s_, bs in zip(fresh, b["sims"]): + ctx.sims.append({"idx": s_["idx"], "udid": s_["udid"]}) + usable.append(bs["usable_s"]) + boot = {"new": new, "all_usable_s": round(time.time() - t0, 2) if not parallel else groups[0][1]["all_s"], "each_usable_s": usable} + # roles may change with N (one-busy: only the first animates); (re)launch every simulator whose role changed or is new + for i, s in enumerate(ctx.sims): + want = roles[i] + if s.get("role") != want: + ctx.guest.call(["prepare", "--udids", s["udid"], "--roles", want] + (["--install"] if ctx.stock and not s.get("installed") else ["--check-app"]) + + (["--quiesce"] if ctx.a.quiesce else []), 300) + s["role"], s["installed"] = want, True + return boot + + +def settle(ctx, at="before the window", max_s=None, bg=None): + """Waits until the guest is quiet (background CPU under --settle-bg for two samples in a row, up to --settle-max). + A settle that reaches its cap FAILS the run: a window now would measure whatever keeps the guest busy, not the + simulators. The top processes by CPU at that moment are recorded in the failure.""" + a = ctx.a + max_s = a.settle_max if max_s is None else max_s + bg = a.settle_bg if bg is None else bg + r = ctx.guest.call(["settle", "--max-s", max_s, "--bg-pct", bg, "--step", a.settle_step, "--consecutive", 2], max_s + 120 + 4 * a.settle_step) + if not r.get("settled"): + top = ", ".join("%s %g%%" % (t["cmd"], t["cpu_pct"]) for t in r.get("top", [])[:6]) or "nothing" + line = "NOT SETTLED %s: the guest's background CPU stayed at %.0f%% of a core (bar %g%%) for %g s; top processes by CPU: %s" % ( + at, r.get("bg_pct", -1), bg, max_s, top) + ctx.section("settle", "FAIL", line, {"at": at, "series": r.get("series"), "top": r.get("top")}) + raise Stop(line) + return r + + +def precheck(ctx, what): + """Before a measurement window: count the bench's OWN booted simulators and refuse any mismatch with the expected + number (a simulator the Mac's idle guard or someone shut down would be measured as if it ran); yield when a simulator + that is not the bench's is booted. The measured count is what the result records.""" + a = ctx.a + udids = [s["udid"] for s in ctx.sims] + r = ctx.guest.call(["precheck", "--udids", ",".join(udids), "--keep-min", keep_minutes(ctx)], 90) + ctx.idle_guard.update(r.get("keep", {})) + if r.get("foreign_booted") and not a.allow_foreign: + line = "YIELDING %s: %d simulator(s) outside the bench are booted (someone needs the Mac); --allow-foreign measures anyway" % (what, r["foreign_booted"]) + ctx.section("foreign", "FAIL", line) + raise Stop(line) + if r["booted"] != len(udids) or r.get("missing") or r.get("extra"): + line = "ABORT %s: %d of the bench's simulators are booted, expected %d (%d expected one(s) not booted, %d unexpected): one was shut down under the run (the Mac's idle guard?)" % ( + what, r["booted"], len(udids), len(r.get("missing", [])), len(r.get("extra", []))) + ctx.section("booted_check", "FAIL", line, {"booted": r["booted"], "expected": len(udids)}) + raise Stop(line) + return r + + +def window(ctx, secs, mix, no_strip=False): + """One measurement window over every simulator of ctx.sims: the guest measures, the host samples the same window. + Returns (guest result, host system dict, window start/end in guest time).""" + a = ctx.a + pre = precheck(ctx, "before the %g s window" % secs) + udids = ",".join(s["udid"] for s in ctx.sims) + roles = ",".join(s["role"] for s in ctx.sims) + lead = a.lead if a.lead is not None else 8 + 2 * len(ctx.sims) + t_go = time.time() + ctx.offset + lead + ctx.run.need(secs + lead + 60, "a %d s window" % secs) + args = ["measure", "--udids", udids, "--roles", roles, "--mix", mix, "--secs", secs, "--start-at", "%.3f" % t_go, + "--probe-method", ctx.probe_method or "none", "--probe-every", a.probe_every, "--agent-period", a.agent_period, "--probe-timeout", a.probe_timeout] + if a.probe_cmd: + args += ["--probe-cmd", a.probe_cmd] + if no_strip: # the unviewed screen: no strip, no marker, no injection, so simfps locks no surface + args[args.index("--probe-method") + 1] = "none" + args.append("--no-strip") + ctx.in_window = True + try: + pend = ctx.guest.start(args, secs + lead + 240) + host_go = t_go - ctx.offset + time.sleep(max(0, host_go - time.time())) + s0 = ctx.host.snap() + time.sleep(max(0, secs - (time.time() - host_go))) + s1 = ctx.host.snap() + g = pend.wait() + finally: + ctx.in_window = False + g["precheck"] = pre + return g, s0, s1, t_go + + +def system_dict(ctx, g, s0, s1, frames_total): + dt = max(s1["mono"] - s0["mono"], 0.001) + th = hostprobe.thread_delta(s0["threads"], s1["threads"], dt, getattr(ctx.host, "hz", 100)) + gpu_ns = None if s0["gpu"] is None or s1["gpu"] is None else s1["gpu"] - s0["gpu"] + gu = g["guest"] + return { + "qemu_cpu_cores": th["cores"], "qemu_user_cores": th["user"], "qemu_guest_cores": th["guest"], "qemu_sys_cores": th["sys"], + "qemu_top_threads": th["top"], + "gpu_busy_pct": None if gpu_ns is None else round(100.0 * gpu_ns / 1e9 / dt, 1), + "gpu_ms_per_frame": None if gpu_ns is None or not frames_total else round(gpu_ns / 1e6 / frames_total, 2), + "window_s": round(dt, 2), + "guest": {"mem_used_mb": gu["mem"]["used_mb"], "mem_free_mb": gu["mem"]["free_mb"], "mem_avail_mb": gu["mem"].get("avail_mb"), "observer_cpu_pct": gu.get("observer_cpu_pct"), "compressed_mb": gu["mem"]["compressed_mb"], + "swapins_d": gu["swapins_d"], "swapouts_d": gu["swapouts_d"], "pageouts_d": gu["pageouts_d"], + "pressure_max": gu["pressure_max"], "total_cpu_pct": gu["total_cpu_pct"], "sims_cpu_pct": gu["sims_cpu_pct"], + "other_cpu_pct": gu["other_cpu_pct"], "sims_rss_mb": gu["sims_rss_mb"]}, + "host": hostprobe.mem_delta(s0["mem"], s1["mem"], dt), + } + + +def build_step(ctx, n, mix, g, s0, s1, t_go, boot=None): + """The step record: per simulator metrics, system numbers, stability.""" + a = ctx.a + secs = g["secs"] + t_end = t_go + a.secs + sims, frames_total = [], 0 + for i, s in enumerate(g["sims"]): + role = s["role"] + raw = [r for r in analysis.parse_raw(s["raw"]) if r[0] >= t_go - 0.001 and r[0] <= t_end + 0.5] or analysis.parse_raw(s["raw"])[:1] + inj = [(x[0], x[1]) for x in s["injections"] if x[2] == 0 or True] + if role == "agent": + sm = {"idx": i, "role": role} + a_ = analysis.strip.analyze_counters([(t, c) for t, c, _ in raw]) + sm.update(analysis.motion_metrics([t for t, _ in a_["unique"]], a.nominal_fps)) + sm["strip"] = {k: a_[k] for k in ("repeats", "lost", "backwards", "jumps", "unreadable", "span_ticks")} + ok, why = analysis.strip.strip_gate(a_) + sm["strip_gate"] = {"ok": ok, "why": why} + sm.update(analysis.agent_metrics(s["actions"], raw)) + else: + ticks = None + if role == "busy" and s["pace"] and raw: + ps = analysis.pace_summary(s["pace"], raw[0][0], raw[-1][0], t_go, t_end) + ticks = ps["app_ticks"] + sm = analysis.sim_metrics(i, role, raw, inj, a.nominal_fps, a.probe_timeout, ticks) + if role == "busy" and s["pace"] and raw: + sm["app_tick_fps"] = ps["app_tick_fps"] + if role in ("busy", "agent") and sm.get("frames"): + frames_total += sm["frames"] + if s.get("simfps"): + sm["surface_fps"] = s["simfps"].get("fps") + if s.get("simfps_error"): + sm["observer_error"] = s["simfps_error"] + p = s.get("proc") or {} + sm["guest_cpu_pct"], sm["guest_rss_mb"] = p.get("cpu_pct"), p.get("rss_mb") + if sm.get("longest_gap_s") is not None and role == "static": + sm["longest_gap_s"] = None + sims.append(sm) + system = system_dict(ctx, g, s0, s1, frames_total) + died = [s["idx"] for s in sims if g["sims"][s["idx"]].get("state_after") != "Booted"] + kern, knote = ctx.host.kernel_gpu(s0["wall"]) + foreign = max((g.get("foreign_booted") or {}).values() or [0]) + judged, listed = gates.crash_split(g["new_ips"], crash_names(a), getattr(a, "strict_crashes", False)) + stab = {"new_ips": len(judged), "ips": judged[:5], "other_ips": listed[:10], "sims_died": died, "kernel_gpu_lines": kern, "kernel_note": knote, + "qemu_same": ctx.host.same_qemu(), "observer_errors": [s["idx"] for s in sims if s.get("observer_error")], + "foreign_booted": foreign} + stab["ok"] = not (stab["new_ips"] or died or kern or not stab["qemu_same"] or (foreign and not a.allow_foreign)) + pre = g.get("precheck") or {} + # "booted" is what the guest counted in the bench's set right before the window, never the expected number + return {"n": n, "mix": mix, "booted": pre.get("booted"), "expected_booted": len(ctx.sims), "sims": sims, "system": system, + "stability": stab, "boot": boot or {}, "counter": counter_gate(g["sims"], mix)} + + +def counter_gate(gsims, mix): + """A frame counter that returns nothing or an error for a simulator fails the window: every simulator of a window with + a counter must have an answer from simfps, and an animating one must have read counters from its surfaces.""" + if mix == "idle": + return gates.gate("SKIP", "no counter in the idle mix") + bad = [] + for i, s in enumerate(gsims): + if s.get("simfps_error") or not s.get("simfps"): + bad.append("sim %d: %s" % (i, s.get("simfps_error") or "simfps returned nothing")) + elif s.get("role") in ("busy", "agent"): + rd = [r for r in analysis.parse_raw(s.get("raw") or "") if 0 <= r[1] < 65535] + if len(rd) < 2: + bad.append("sim %d: no frame counter read from its surfaces (%d reads)" % (i, len(rd))) + if bad: + return gates.gate("FAIL", "; ".join(bad[:4])) + return gates.gate("PASS", "simfps answered for %d simulator(s), counters read" % len(gsims)) + + +def step_gates(ctx, step, shots=None): + gs = {"counter": step["counter"], "strip": gates.strip_gate_of(step["sims"]), "kernel_gpu": gates.kernel_gate(step["stability"]["kernel_gpu_lines"], step["stability"]["kernel_note"])} + gs.update(ctx.coh.check(ctx.marks["coh"])) + if shots: + gs.update(shots) + swapped = step["system"]["guest"]["swapouts_d"] or 0 + hs = step["system"]["host"] + cg = hs.get("cgroup_swap_delta_mb") # the VM's own cgroup: other projects' swapping on the host is not this run's + host_swap = cg is not None and cg > 50 + guest_swap = swapped > 0 and ctx.a.fail_on_swap + gs["memory"] = gates.gate("FAIL" if guest_swap or host_swap else ("PASS" if cg is not None else "SKIP"), + "guest swapouts +%s, pressure max %s; host cgroup swap %s MB, pswpout %s pages" % ( + swapped, step["system"]["guest"]["pressure_max"], hs.get("cgroup_swap_delta_mb"), hs.get("pswpout_pages"))) + step["gates"] = gs + step["gates_ok"] = gates.worst(gs) + step["stable"] = step["stability"]["ok"] + return gs + + +def crash_names(a): + """Extra process names whose crash report fails a run (--crash-names a,b).""" + return [n for n in (getattr(a, "crash_names", None) or "").split(",") if n.strip()] + + +def run_gates(ctx): + """The gates that hold for a whole run, whatever the scenario: Coherent and QEMU's log since the run began, kernel + GPU error lines since then, and crash reports that appeared on the guest. A scenario that stopped before its setup + has nothing to judge. Never raises: a gate that cannot be read is SKIP.""" + mk = getattr(ctx, "run_mark", None) + if not mk: + return + gs = {} + try: + for k, v in ctx.coh.check(mk["coh"]).items(): + gs["run_" + k] = v + except Exception as e: + gs["run_coherent"] = gates.gate("SKIP", "unreadable: %s" % type(e).__name__) + try: + kern, note = ctx.host.kernel_gpu(mk["t0"]) + gs["run_kernel_gpu"] = gates.kernel_gate(kern, note) + except Exception as e: + gs["run_kernel_gpu"] = gates.gate("SKIP", "unreadable: %s" % type(e).__name__) + try: + now = set(ctx.guest.call(["ips"], 60)["ips"]) + gs["run_crash_reports"] = gates.crash_gate(now - mk["ips"], crash_names(ctx.a), getattr(ctx.a, "strict_crashes", False)) + except Exception as e: + gs["run_crash_reports"] = gates.gate("SKIP", "guest unreachable: %s" % type(e).__name__) + ctx.doc.setdefault("gates", {}).update(gs) + for name, v in gs.items(): + ctx.section("gate:" + name, v["status"], "GATE %s %s: %s" % (v["status"], name, v["detail"])) + + +def take_shots(ctx, count=2, judge_static=False): + """Screenshots of every simulator, through the guest, judged by the screenshot gates. Static screens are judged + for pixels only when `judge_static`: after a window with input probes their marker state is not the golden's + (the pixel gate runs on a freshly launched static app, see scale.golden_phase).""" + udids = ",".join(s["udid"] for s in ctx.sims) + r = ctx.guest.call(["shots", "--udids", udids, "--count", count], 60 + 30 * len(ctx.sims)) + by = [] + for s, one in zip(ctx.sims, r["sims"]): + role = s["role"] if (s["role"] != "static" or judge_static) else "static-unjudged" + by.append({"idx": s["idx"], "role": role, "shots": one["shots"]}) + return gates.shots_gates(by, ctx.golden), by diff --git a/src/bench/macbench/fakes.py b/src/bench/macbench/fakes.py new file mode 100644 index 0000000..1a29149 --- /dev/null +++ b/src/bench/macbench/fakes.py @@ -0,0 +1,84 @@ +"""Stand-ins for DRY=1 and the tests: a fake Linux host (QEMU CPU, GPU, memory) and fake Coherent gates. The fake +numbers follow the number of booted simulators so every code path that reads them runs; they mean nothing.""" +import os +import time + + +class FakeHost: + dry = True + + def __init__(self, faults=""): + self.n = 0 + self.faults = faults.split(",") + self.t_last = time.monotonic() + self.cpu_ticks = {"vcpu": 0, "sc": 0} + self.gpu = 0 + self.pid = 4242 + self.kernel_lines = 0 + + def qemu_pid(self): + return self.pid + + def snap(self): + now = time.monotonic() + dt, self.t_last = now - self.t_last, now + self.cpu_ticks["vcpu"] += int(dt * 100 * (0.05 + 0.8 * self.n)) + self.cpu_ticks["sc"] += int(dt * 100 * (0.01 + 0.05 * self.n)) + self.gpu += int(dt * 1e9 * min(0.95, 0.15 * self.n)) + v, s = self.cpu_ticks["vcpu"], self.cpu_ticks["sc"] + thr = {"1": ("CPU 0/KVM", v - v // 5, v // 10, v // 2), "2": ("metal2vulkan-sc", s, s // 4, 0)} + return {"mono": now, "wall": time.time(), "pid": self.pid, "threads": thr, "gpu": self.gpu, + "mem": {"mem_avail_kb": int((40 - 0.5 * self.n) * 1048576), "qemu_rss_kb": int((9 + 0.4 * self.n) * 1048576), + "pswpin": 0, "pswpout": 0, "cg_swap": 0, "psi_some_us": 0}} + + def mem_avail_gb(self): + v = 40 - 0.5 * self.n + mf = [f for f in self.faults if f.startswith("hostmem@")] + if mf and self.n >= int(mf[0].split("@")[1]): + return 0.5 + return v + + def _fault_n(self, name): + for f in self.faults: + if f == name or f.startswith(name + "@"): + return int(f.split("@")[1]) if "@" in f else 0 + return None + + def disk_free_gb(self, path): + """Faults: diskstart (16 GiB from the start: under the 18 to start), disklow@N (12.5 from N simulators: under the 14 to + boot another), diskabort@N (9: under the 11 to go on), diskunreadable.""" + if self._fault_n("diskunreadable") is not None: + return None + for name, val in (("diskabort", 9.0), ("disklow", 12.5)): + k = self._fault_n(name) + if k is not None and self.n >= k: + return val + if self._fault_n("diskstart") is not None: + return 16.0 + return 500.0 + + def kernel_gpu(self, since): + if any(f.startswith("kernelgpu@") and self.n >= int(f.split("@")[1]) for f in self.faults): + return 2, "fence expired (fake)" + return 0, "" + + def same_qemu(self): + k = self._fault_n("qemugone") # the VM restarted: another QEMU process + return not (k is not None and self.n >= k) + + def env(self): + return {"kernel": "fake", "mesa": "fake", "cpu_model": "fake", "cpus": 20}, {"surface_coherent": "1"}, {"REIMS_VGPU_COHERENT": "1"} + + +class FakeCoherent: + def __init__(self, host, faults=""): + self.host, self.faults = host, faults.split(",") + + def mark(self): + return {"n": 0} + + def check(self, mark): + bad = any(f.startswith("coherent@") and self.host.n >= int(f.split("@")[1]) for f in self.faults) + ok = lambda d: {"status": "PASS", "detail": d} + return {"coherent_census": {"status": "FAIL", "detail": "overwrite_risk 1 (fake)"} if bad else ok("fake"), + "coherent_process": ok("fake"), "qemu_log": ok("fake"), "gpu_hang": ok("fake")} diff --git a/src/bench/macbench/gates.py b/src/bench/macbench/gates.py new file mode 100644 index 0000000..244c170 --- /dev/null +++ b/src/bench/macbench/gates.py @@ -0,0 +1,189 @@ +"""Correctness gates. Each returns {"status": PASS|FAIL|SKIP, "detail": str}. An unreadable gate is SKIP, shown as +such, never PASS. Coherent and kernel gates are DELTAS over a window (a mark taken before, a check after), never +absolute counts or a fixed number of surfaces.""" +import json +import os +import re + +PROCESS_GATES = ("late_copies_in_busy", "late_beside_unplanned", "plan_left_planned") + + +def gate(status, detail=""): + return {"status": status, "detail": detail} + + +# Crash reports that fail a run: the bench's own programs and the processes the measured frames pass through. +# macOS files reports for other daemons of a simulator all day (widget extensions, searchd, containermanagerd ...); +# those are listed with the gate, not judged, unless --strict-crashes says so. +CRASH_NAMES = ("GlassBench", "GenieBench", "simfps", "SpringBoard", "backboardd", "SimRenderServer", "SimMetalHost", + "WindowServer", "com.apple.CoreSimulator.CoreSimulatorService", "CoreSimulatorBridge", "launchd_sim", + "Simulator", "simctl", "Kernel", "JetsamEvent") +_CRASH_STAMP = re.compile(r"[-_.]\d{4}-\d{2}-\d{2}[-_]\d{6}.*$") + + +def crash_proc(report): + """The process a crash report's file name names: 'searchd-2026-01-02-030405.ips' -> 'searchd'.""" + n = _CRASH_STAMP.sub("", report) + return re.sub(r"\.(ips|crash)$", "", n) + + +def crash_split(reports, extra=(), strict=False): + """(judged, listed): new crash reports that fail a gate, and those that are only shown.""" + names = {n.lower() for n in CRASH_NAMES} | {n.strip().lower() for n in extra if n.strip()} + judged, listed = [], [] + for r in sorted(reports): + (judged if strict or crash_proc(r).lower() in names else listed).append(r) + return judged, listed + + +def crash_gate(reports, extra=(), strict=False): + judged, listed = crash_split(reports, extra, strict) + also = ("; not judged: %d other report(s): %s" % (len(listed), ", ".join(listed[:5]))) if listed else "" + if judged: + return gate("FAIL", "%d new crash report(s): %s%s" % (len(judged), ", ".join(judged[:3]), also)) + return gate("PASS", "no new crash report of the bench or the simulator's graphics path" + also) + + +def worst(gates): + """FAIL if any FAIL; else SKIP-only/mixed is still ok: a run PASSes when nothing FAILed (SKIPs stay visible).""" + return not any(g["status"] == "FAIL" for g in gates.values()) + + +class Coherent: + """Reads Coherent's census (coherent.jsonl, a report a minute per surface), QEMU's log and the GPU hang flag.""" + + def __init__(self, census_dir, qemu_log, hang_file): + self.census = os.path.join(census_dir, "coherent.jsonl") if census_dir else None + self.qemu_log, self.hang_file = qemu_log, hang_file + + def _lines(self): + out = [] + if self.census and os.path.exists(self.census): + with open(self.census, errors="replace") as f: + for l in f: + try: + out.append(json.loads(l)) + except ValueError: + pass + return out + + def _log_from(self, off): + if not (self.qemu_log and os.path.exists(self.qemu_log)): + return None + with open(self.qemu_log, "rb") as f: + f.seek(off) + return f.read().decode(errors="replace") + + def mark(self): + lines = self._lines() + proc = {k: 0 for k in PROCESS_GATES} + for d in lines: + if "process" in d: + for k in PROCESS_GATES: + proc[k] = max(proc[k], d.get(k, 0) or 0) + size = os.path.getsize(self.qemu_log) if self.qemu_log and os.path.exists(self.qemu_log) else None + return {"n": len(lines), "proc": proc, "log_off": size, "hang": bool(self.hang_file and os.path.exists(self.hang_file))} + + def check(self, mark): + """Gates since the mark: overwrite_risk, sync_errors (new surface reports), process counters that rose, + command buffer failures and GPU waits in QEMU's log, a new hang flag.""" + res = {} + lines = self._lines() + new = lines[mark["n"]:] + surf = [d for d in new if "process" not in d] + if self.census is None or not os.path.exists(self.census): + res["coherent_census"] = gate("SKIP", "no census file (Coherent off?)") + elif not surf: + res["coherent_census"] = gate("SKIP", "no census report since the mark (they come once a minute)") + else: + risk = max((d.get("overwrite_risk", 0) or 0) for d in surf) + errs = max((d.get("sync_errors", 0) or 0) for d in surf) + res["coherent_census"] = gate("FAIL" if risk or errs else "PASS", + "%d report(s): overwrite_risk max %d, sync_errors max %d" % (len(surf), risk, errs)) + proc = {k: mark["proc"][k] for k in PROCESS_GATES} + for d in new: + if "process" in d: + for k in PROCESS_GATES: + proc[k] = max(proc[k], d.get(k, 0) or 0) + rose = {k: proc[k] for k in PROCESS_GATES if proc[k] > mark["proc"][k]} + if self.census and os.path.exists(self.census): + res["coherent_process"] = gate("FAIL" if rose else "PASS", ("rose: %s" % rose) if rose else "late_copies_in_busy, late_beside_unplanned, plan_left_planned flat") + else: + res["coherent_process"] = gate("SKIP", "no census file") + txt = self._log_from(mark["log_off"]) if mark["log_off"] is not None else None + if txt is None: + res["qemu_log"] = gate("SKIP", "QEMU log not readable") + else: + cb = txt.count("command buffer failed") + waits = len(re.findall(r"waited \d+s for stamp", txt)) + res["qemu_log"] = gate("FAIL" if cb else "PASS", "command buffer failed %d, stamp waits %d" % (cb, waits)) + hang = bool(self.hang_file and os.path.exists(self.hang_file)) + if self.hang_file is None: + res["gpu_hang"] = gate("SKIP", "no hang flag path") + else: + res["gpu_hang"] = gate("FAIL" if hang and not mark["hang"] else "PASS", "hang flag " + ("set" if hang else "clear")) + return res + + +def kernel_gate(count, note): + if count is None: + return gate("SKIP", note) + return gate("FAIL" if count else "PASS", "%d GPU error line(s)%s" % (count, (": " + note) if note else "")) + + +def strip_gate_of(sims): + bad = [s for s in sims if s.get("strip_gate") and not s["strip_gate"]["ok"]] + judged = [s for s in sims if s.get("strip_gate")] + if not judged: + return gate("SKIP", "no simulator drew the strip") + if bad: + return gate("FAIL", "; ".join("sim %s: %s" % (s["idx"], s["strip_gate"]["why"]) for s in bad[:3])) + return gate("PASS", "%d simulator(s): no frame out of order" % len(judged)) + + +def shots_gates(shots_by_sim, golden=None): + """Tear, strip-readable and pixel-exact gates from the guest's `shots` output. + shots_by_sim: [{"idx", "role", "shots": [{ok, hash, tear, tear_applicable, counter}]}]""" + res = {} + tear_bad, tear_judged = [], 0 + for s in shots_by_sim: + if s["role"] != "busy": + continue + for sh in s["shots"]: + if sh.get("ok") and sh.get("tear_applicable"): + tear_judged += 1 + if sh["tear"]: + tear_bad.append(s["idx"]) + if not tear_judged: + res["tears"] = gate("SKIP", "no screenshot showed the scrolling cards") + else: + res["tears"] = gate("FAIL" if tear_bad else "PASS", ("torn: sim %s" % sorted(set(tear_bad))) if tear_bad else "%d screenshot(s) clean" % tear_judged) + statics = [sh for s in shots_by_sim if s["role"] == "static" for sh in s["shots"] if sh.get("ok")] + hashes = [sh["hash"] for sh in statics] + golden_hash, golden_key = golden(statics[0]["w"], statics[0]["h"]) if (golden and statics) else (None, None) + if not hashes: + res["pixels"] = gate("SKIP", "no static reference screen in this step") + elif len(set(hashes)) > 1: + res["pixels"] = gate("FAIL", "%d different pictures of the same static screen" % len(set(hashes))) + elif golden_hash is None: + res["pixels"] = gate("SKIP", "screens agree with each other; no golden stored for %s (bench golden record)" % (golden_key or "this setup")) + else: + res["pixels"] = gate("PASS" if hashes[0] == golden_hash else "FAIL", + "static screen %s the golden %s" % ("matches" if hashes[0] == golden_hash else "DIFFERS from", golden_key)) + return res + + +def leak_gate(name, xs, ys, bar_per_unit, unit): + """FAIL when the least-squares slope of ys over xs exceeds the bar (e.g. MB per cycle) AND the growth is steady: + at least 4 samples on a line (R2 >= 0.8). A slope above the bar from fewer samples or a scatter is one step, not a + leak: SKIP, and it says to run longer.""" + from . import stats + b, _, r2 = stats.slope(xs, ys) + if b is None: + return gate("SKIP", "%s: not enough samples" % name) + detail = "%s: %+.2f %s (bar %g, R2 %.2f, %d samples)" % (name, b, unit, bar_per_unit, r2, len(xs)) + if b <= bar_per_unit: + return gate("PASS", detail) + if len(xs) < 4 or r2 < 0.8: + return gate("SKIP", detail + ": above the bar but not a steady growth; run more cycles or longer to judge") + return gate("FAIL", detail) diff --git a/src/bench/macbench/hostprobe.py b/src/bench/macbench/hostprobe.py new file mode 100644 index 0000000..77f9faa --- /dev/null +++ b/src/bench/macbench/hostprobe.py @@ -0,0 +1,306 @@ +"""What the Linux host says about the VM: QEMU's threads (user / guest / sys CPU), the GPU's render time from +DRM fdinfo, the VM's cgroup (memory, swap, pressure), host memory and swap, and kernel GPU error lines. +Passive reads of /proc and /sys only; no tracer, no GPU work. `root` lets the tests point at a fake tree.""" +import os +import re +import subprocess + +GPU_ERRORS = re.compile( + r"fence expir|gpu hang|context reset|i915.*\bhang\b" + r"|xe \S+: .*(timedout job|wedged|engine reset|gt\d+: (trying reset|reset (queued|started|failed))" + r"|\*error\* (tile\d+: )?gt\d|fault response|faulted address|banned|coredump has been created)", re.I) + + +def _read(root, path): + try: + with open(root + path) as f: + return f.read() + except OSError: + return None + + +def parse_task_stat(text): + """/proc//task//stat -> (comm, utime, stime, guest_time) in clock ticks. comm may hold spaces and ')'.""" + i, j = text.index("("), text.rindex(")") + comm = text[i + 1:j] + f = text[j + 2:].split() # f[0] is field 3 (state) + return comm, int(f[11]), int(f[12]), int(f[40]) if len(f) > 40 else 0 + + +def thread_class(comm): + c = re.sub(r"\d+", "N", comm) + return "vCPU" if c.startswith("CPU N/KVM") else c + + +def thread_snapshot(pid, root=""): + out = {} + base = "%s/proc/%d/task" % (root, pid) + try: + tids = os.listdir(base) + except OSError: + return out + for tid in tids: + t = _read(root, "/proc/%d/task/%s/stat" % (pid, tid)) + if t: + try: + out[tid] = parse_task_stat(t) + except (ValueError, IndexError): + pass + return out + + +def thread_delta(a, b, secs, hz=100, top=6): + """CPU of the process between two snapshots, in cores: user (host-side), guest (inside the VM), sys, total, and the + top thread classes. guest_time is part of utime in Linux, so user = utime - guest.""" + cls, tot_u = {}, [0, 0, 0] + for tid, (comm, ut, st, gt) in b.items(): + if tid not in a: + continue + du, ds, dg = ut - a[tid][1], st - a[tid][2], gt - a[tid][3] + tot_u[0] += du - dg + tot_u[1] += dg + tot_u[2] += ds + k = thread_class(comm) + e = cls.setdefault(k, [0, 0.0]) + e[0] += 1 + e[1] += (du + ds) / hz / secs + f = lambda x: round(x / hz / secs, 3) + rows = sorted(cls.items(), key=lambda kv: -kv[1][1])[:top] + return {"user": f(tot_u[0]), "guest": f(tot_u[1]), "sys": f(tot_u[2]), "cores": f(sum(tot_u)), + "top": [{"name": k, "threads": n, "cores": round(c, 3)} for k, (n, c) in rows]} + + +def parse_fdinfo(text): + """-> (client id, render ns or None). i915 states `drm-engine-render: ns`; xe states cycles (not converted: + None, so a GPU number is never made up).""" + cid = re.search(r"^drm-client-id:\s*(\S+)", text, re.M) + r = re.search(r"^drm-engine-render:\s*(\d+)", text, re.M) + if not cid: + return None, None + return cid.group(1), int(r.group(1)) if r else None + + +def gpu_render_ns(pid, root=""): + """Sum of the render-engine busy ns over the process's distinct DRM clients; None when none is readable.""" + seen, got = {}, False + base = "%s/proc/%d/fdinfo" % (root, pid) + try: + fds = os.listdir(base) + except OSError: + return None + for fd in fds: + t = _read(root, "/proc/%d/fdinfo/%s" % (pid, fd)) + if t and "drm-client-id" in t: + cid, ns = parse_fdinfo(t) + if ns is not None: + seen[cid] = ns + got = True + return sum(seen.values()) if got else None + + +def parse_kv(text): + d = {} + for line in (text or "").splitlines(): + p = line.split() + if len(p) == 2 and p[1].lstrip("-").isdigit(): + d[p[0]] = int(p[1]) + return d + + +def parse_meminfo(text): + d = {} + for line in (text or "").splitlines(): + p = line.replace(":", " ").split() + if len(p) >= 2 and p[1].isdigit(): + d[p[0]] = int(p[1]) + return d + + +def parse_psi(text): + """/proc/pressure/memory or memory.pressure -> {"some": {avg10,total}, "full": {...}}""" + d = {} + for line in (text or "").splitlines(): + p = line.split() + if p and p[0] in ("some", "full"): + d[p[0]] = {k: float(v) for k, v in (x.split("=") for x in p[1:])} + return d + + +def cgroup_dir(pid, root=""): + t = _read(root, "/proc/%d/cgroup" % pid) + for line in (t or "").splitlines(): + if line.startswith("0::"): + return "/sys/fs/cgroup" + line[3:] + return None + + +def mem_snapshot(pid, root=""): + """Host memory and swap counters, and the VM's cgroup's: every number is a counter or a level, deltas are taken by the caller.""" + mi = parse_meminfo(_read(root, "/proc/meminfo")) + vs = parse_kv(_read(root, "/proc/vmstat")) + cg = cgroup_dir(pid, root) + s = {"mem_total_kb": mi.get("MemTotal"), "mem_avail_kb": mi.get("MemAvailable"), "swap_total_kb": mi.get("SwapTotal"), + "swap_free_kb": mi.get("SwapFree"), "pswpin": vs.get("pswpin"), "pswpout": vs.get("pswpout"), + "psi_some_us": parse_psi(_read(root, "/proc/pressure/memory")).get("some", {}).get("total")} + if cg: + for name, key in (("memory.current", "cg_current"), ("memory.swap.current", "cg_swap")): + t = _read(root, cg + "/" + name) + if t and t.strip().isdigit(): + s[key] = int(t.strip()) + ev = parse_kv(_read(root, cg + "/memory.events")) + s["cg_high"], s["cg_max"], s["cg_oom"] = ev.get("high"), ev.get("max"), ev.get("oom_kill") + s["cg_psi_some_us"] = parse_psi(_read(root, cg + "/memory.pressure")).get("some", {}).get("total") + st = _read(root, "/proc/%d/status" % pid) or "" + m = re.search(r"VmRSS:\s+(\d+)", st) + s["qemu_rss_kb"] = int(m.group(1)) if m else None + sw = re.search(r"VmSwap:\s+(\d+)", st) + s["qemu_swap_kb"] = int(sw.group(1)) if sw else None + # what QEMU holds, wherever the host keeps it: RSS alone also rises when the host brings swapped pages back, which + # is not a leak (a VM with 8 GB in swap gained 480 MB of RSS in one create/boot/delete cycle and then stayed flat; + # swap-in is the likely cause, it was not counted in that run) + s["qemu_mem_kb"] = None if not m else int(m.group(1)) + (int(sw.group(1)) if sw else 0) + return s + + +def mem_delta(a, b, secs): + d = lambda k: None if a.get(k) is None or b.get(k) is None else b[k] - a[k] + out = {"mem_avail_gb": None if b.get("mem_avail_kb") is None else round(b["mem_avail_kb"] / 1048576.0, 2), + "qemu_rss_mb": None if b.get("qemu_rss_kb") is None else round(b["qemu_rss_kb"] / 1024.0, 1), + "pswpin_pages": d("pswpin"), "pswpout_pages": d("pswpout"), + "cgroup_swap_mb": None if b.get("cg_swap") is None else round(b["cg_swap"] / 1048576.0, 1), + "cgroup_swap_delta_mb": None if d("cg_swap") is None else round(d("cg_swap") / 1048576.0, 2), + "cgroup_mem_gb": None if b.get("cg_current") is None else round(b["cg_current"] / 1073741824.0, 2), + "cgroup_events": {k: d(k) for k in ("cg_high", "cg_max", "cg_oom") if d(k) is not None}, + "psi_mem_some_pct": None if d("psi_some_us") is None or secs <= 0 else round(100.0 * d("psi_some_us") / 1e6 / secs, 2), + "cgroup_psi_some_pct": None if d("cg_psi_some_us") is None or secs <= 0 else round(100.0 * d("cg_psi_some_us") / 1e6 / secs, 2)} + return out + + +def kernel_gpu_lines(since_epoch): + """Kernel GPU error lines since a time: (count or None, note). None = the journal can't be read (not a PASS).""" + try: + p = subprocess.run(["journalctl", "-k", "--since", "@%d" % int(since_epoch), "--no-pager", "-q"], + capture_output=True, text=True, timeout=30) + except (OSError, subprocess.TimeoutExpired) as e: + return None, "journalctl unavailable: %s" % e + if p.returncode != 0: + return None, "journalctl: " + (p.stderr.strip()[:100] or "failed") + lines = [l for l in p.stdout.splitlines() if GPU_ERRORS.search(l)] + return len(lines), ("; ".join(l[-80:] for l in lines[-2:]) if lines else "") + + +def read_live_conf(path): + """live.conf as {key: value} (read only).""" + out = {} + try: + with open(path) as f: + for line in f: + line = line.strip() + if line and not line.startswith("#") and "=" in line: + k, v = line.split("=", 1) + out[k.strip()] = v.strip() + except OSError: + pass + return out + + +def qemu_env_switches(pid, root=""): + """REIMS_VGPU_* names and values of the QEMU process (a directory value is shown as ).""" + t = _read(root, "/proc/%d/environ" % pid) or "" + out = {} + for kv in t.split("\0"): + if kv.startswith("REIMS_VGPU_") and "=" in kv: + k, v = kv.split("=", 1) + out[k] = "" if v.startswith("/") else v + return out + + +def qemu_env_value(pid, key, root=""): + """The raw value of one environment variable of the QEMU process (None when unreadable or unset).""" + t = _read(root, "/proc/%d/environ" % pid) or "" + for kv in t.split("\0"): + if kv.startswith(key + "="): + return kv.split("=", 1)[1] + return None + + +def mesa_is_custom(pid, root=""): + """True when QEMU maps an ANV driver from outside the system library directories (its own Mesa build).""" + t = _read(root, "/proc/%d/maps" % pid) or "" + paths = {l.split()[-1] for l in t.splitlines() if "libvulkan_intel" in l and l.split()[-1].startswith("/")} + return bool(paths) and not all(p.startswith(("/usr/lib", "/lib")) for p in paths) + + +def host_versions(pid=None): + kernel = os.uname().release + mesa = None + for argv in (["dpkg-query", "-W", "-f=${Version}", "mesa-vulkan-drivers"], ["rpm", "-q", "--qf", "%{VERSION}", "mesa-vulkan-drivers"]): + try: + p = subprocess.run(argv, capture_output=True, text=True, timeout=10) + if p.returncode == 0 and p.stdout.strip(): + mesa = p.stdout.strip() + break + except (OSError, subprocess.TimeoutExpired): + pass + cpu = None + try: + with open("/proc/cpuinfo") as f: + m = re.search(r"model name\s*:\s*(.+)", f.read()) + cpu = m.group(1).strip() if m else None + except OSError: + pass + note = "unknown" + if mesa: + note = mesa + " (system package)" + if pid and mesa_is_custom(pid): + note = "custom build in QEMU (system package is %s): pass --tag mesa=" % (mesa or "n/a") + return {"kernel": kernel, "mesa": note, "cpu_model": cpu, "cpus": os.cpu_count()} + + +class Host: + """The real host side for one QEMU process. `root` is "" in production (tests point it at a fake tree).""" + dry = False + + def __init__(self, pid, root="", live_conf=None, hz=None): + self.pid, self.root, self.live_conf = pid, root, live_conf + self.hz = hz or os.sysconf("SC_CLK_TCK") + self.n = 0 + self.born = self._starttime() # the QEMU process as this run first saw it: a restart is a different process + + def qemu_pid(self): + return self.pid + + def snap(self): + import time + return {"mono": time.monotonic(), "wall": time.time(), "pid": self.pid, "threads": thread_snapshot(self.pid, self.root), + "gpu": gpu_render_ns(self.pid, self.root), "mem": mem_snapshot(self.pid, self.root)} + + def mem_avail_gb(self): + v = mem_snapshot(self.pid, self.root).get("mem_avail_kb") + return None if v is None else v / 1048576.0 + + def disk_free_gb(self, path): + try: + st = os.statvfs(path) + return st.f_bavail * st.f_frsize / 1073741824.0 + except OSError: + return None + + def kernel_gpu(self, since): + return kernel_gpu_lines(since) + + def _starttime(self): + t = _read(self.root, "/proc/%d/stat" % self.pid) + try: + return t[t.rindex(")") + 2:].split()[19] if t else None # field 22: start time in ticks since boot + except (ValueError, IndexError): + return None + + def same_qemu(self): + """The QEMU process is still the one the run started with (same pid AND same start time: a restarted VM may reuse the pid).""" + now = self._starttime() + return now is not None and now == self.born + + def env(self): + sw = read_live_conf(self.live_conf) if self.live_conf else {} + return host_versions(self.pid), sw, qemu_env_switches(self.pid, self.root) diff --git a/src/bench/macbench/others.py b/src/bench/macbench/others.py new file mode 100644 index 0000000..d82a0c0 --- /dev/null +++ b/src/bench/macbench/others.py @@ -0,0 +1,226 @@ +"""The other scenarios: idle VM, lifecycle (+ leak per cycle, boot storm), soak, build timing, the Device panel plug, +the golden recorder.""" +import json +import os +import subprocess +import time + +from . import analysis, core, gates, scale, stats, strip +from .core import Stop + + +def _fmt(x, unit=""): + return "n/a" if x is None else "%g%s" % (x, unit) + + +def run_seed(ctx): + """macbox bench seed [--rebuild-seed]: make (or check and report) the production seed and nothing else.""" + info = core.setup_guest(ctx) + core.capture_env(ctx, info) + ctx.doc["data"] = {"seed": ctx.doc["env"]["guest"].get("seed")} + ctx.section("seed_ready", "PASS", "SEED READY: %s" % [s for s in ctx.doc["sections"] if s["name"] == "seed"][-1]["line"] if not ctx.stock else "SEED SKIP: --stock-devices makes no seed") + ctx.save() + + +def run_idle(ctx): + a = ctx.a + info = core.setup_guest(ctx, sims=False) + core.capture_env(ctx, info) + ctx.marks = {"coh": ctx.coh.mark(), "t0": time.time()} + m = ctx.guest.call(["memcheck"], 60) + if (m["booted"] or m.get("foreign_booted")) and not a.allow_foreign: + raise Stop("simulators are booted (%d of the bench, %d outside it): an idle VM has none; --allow-foreign to measure anyway" % (m["booted"], m.get("foreign_booted", 0))) + core.settle(ctx, "before the idle window") + g, s0, s1, t_go = core.window(ctx, a.secs, "idle") + sysd = core.system_dict(ctx, g, s0, s1, 0) + kern, note = ctx.host.kernel_gpu(s0["wall"]) + gs = dict(ctx.coh.check(ctx.marks["coh"])) + gs["kernel_gpu"] = gates.kernel_gate(kern, note) + ctx.doc["data"] = {"window_s": a.secs, "system": sysd} + ctx.doc["gates"] = gs + ok = sysd["qemu_cpu_cores"] <= a.bar_qemu_cores + ctx.section("idle", "PASS" if ok and gates.worst(gs) else "FAIL", + "IDLE %s: QEMU %.3f cores (bar <= %g; user %.3f guest %.3f sys %.3f), GPU busy %s%%, guest CPU %s%%, guest used %s MB, host cgroup swap %s MB; gates %s" % ( + "PASS" if ok and gates.worst(gs) else "FAIL", sysd["qemu_cpu_cores"], a.bar_qemu_cores, sysd["qemu_user_cores"], sysd["qemu_guest_cores"], + sysd["qemu_sys_cores"], sysd["gpu_busy_pct"], sysd["guest"]["total_cpu_pct"], sysd["guest"]["mem_used_mb"], + sysd["host"].get("cgroup_swap_mb"), "ok" if gates.worst(gs) else [k for k, v in gs.items() if v["status"] == "FAIL"])) + ctx.save() + + +def _median(v): + return stats.r1(stats.percentile(v, 0.5), 2) + + +def run_lifecycle(ctx): + a = ctx.a + info = core.setup_guest(ctx) + core.capture_env(ctx, info) + ctx.marks = {"coh": ctx.coh.mark(), "t0": time.time()} + cyc, xs, rss, used, comp = [], [], [], [], [] + base = ctx.host.snap()["mem"].get("qemu_mem_kb") + mem0 = ctx.guest.call(["memcheck"], 60)["mem"] + data = ctx.doc["data"] = {"cycles": cyc, "storm": None} + for i in range(a.cycles): + why = core.guard(ctx, 1, "before lifecycle cycle %d" % (i + 1)) + if why: + ctx.section("stopped", "FAIL", "STOPPED: %s" % why) + break + r = ctx.guest.call(["cycle", "--keep-min", core.keep_minutes(ctx)] + (["--type", ctx.type_id, "--runtime", ctx.runtime] if ctx.stock else ["--seed", ctx.seed["seed"]]), 900) + cyc.append(r["timings"]) + m = ctx.guest.call(["memcheck"], 60) + xs.append(i + 1) + used.append(m["mem"]["used_mb"] - mem0["used_mb"]) + comp.append(m["mem"]["compressed_mb"]) + sn = ctx.host.snap()["mem"].get("qemu_mem_kb") + rss.append(None if sn is None or base is None else (sn - base) / 1024.0) + ctx.save() + keys = sorted({k for c in cyc for k in c}) + med = {k: _median([c[k] for c in cyc if c.get(k) is not None]) for k in keys} + data["median_s"] = med + ok = all(v is not None for v in med.values()) and bool(cyc) + names = (("create_s", "create"), ("clone_s", "clone of the seed" if not ctx.stock else "clone"), ("cold_boot_s", "cold boot"), ("install_s", "install"), + ("cold_launch_call_s", "launch call"), ("cold_first_frame_s", "first app frame"), ("shutdown_s", "shutdown"), ("resume_boot_s", "resume boot"), + ("resume_first_frame_s", "first frame after resume"), ("clone_boot_s", "clone boot"), ("delete_s", "delete")) + ctx.section("lifecycle", "PASS" if ok else "FAIL", "LIFECYCLE %s (%d cycles, medians, s; %s): %s" % ( + "PASS" if ok else "FAIL", len(cyc), "fresh stock devices" if ctx.stock else "clones of the seed", ", ".join("%s %s" % (label, med[k]) for k, label in names if k in med))) + leaks = {"qemu_mem": gates.leak_gate("QEMU memory (RSS + swap)", xs, rss, a.leak_rss_mb, "MB/cycle"), + "guest_used": gates.leak_gate("guest used memory", xs, used, a.leak_guest_mb, "MB/cycle")} + data["leak"] = {"qemu_mem_mb": rss, "guest_used_mb": used} + for k, g in leaks.items(): + ctx.doc["gates"]["leak_" + k] = g + ctx.section("leak:" + k, g["status"], "LEAK %s: %s" % (g["status"], g["detail"])) + if a.storm: + n = a.storm + reason = core.guard(ctx, n, "before the boot storm of %d" % n) + if reason: + ctx.section("storm", "SKIP", "BOOT STORM SKIP: " + reason) + else: + t0 = time.time() + boot = core.grow(ctx, n, ["busy"] * n, parallel=True) + usable = time.time() - t0 + data["storm"] = {"n": n, "boot_all_s": boot.get("all_usable_s"), "each_s": boot.get("each_usable_s"), "boot_and_launch_s": round(usable, 1)} + ok = usable <= a.storm_bar_s + ctx.section("storm", "PASS" if ok else "FAIL", "BOOT STORM %s: %d simulators booted at once: all booted in %s s (slowest %s s), booted and app launched in %.0f s (bar <= %g)" % ( + "PASS" if ok else "FAIL", n, boot.get("all_usable_s"), max(boot.get("each_usable_s") or [0]), usable, a.storm_bar_s)) + ctx.doc["gates"].update(ctx.coh.check(ctx.marks["coh"])) + ctx.save() + + +def run_soak(ctx): + a = ctx.a + info = core.setup_guest(ctx) + core.capture_env(ctx, info) + ctx.marks = {"coh": ctx.coh.mark(), "t0": time.time()} + n = a.soak_n + reason = core.guard(ctx, n, "before the soak") + if reason: + raise Stop(reason) + ctx.host.n = n + core.grow(ctx, n, ["busy"] * n) + core.settle(ctx, "before the soak") + res = {} + for label in ("start", "end"): + if label == "end": + t_end = time.time() + a.soak_minutes * 60 + xs, rss, used = [], [], [] + while time.time() < t_end and not ctx.run.expired.is_set(): + ctx.run.need(60, "a soak sample") + time.sleep(min(a.soak_every, max(0, t_end - time.time()))) + m = ctx.guest.call(["memcheck"], 60) + sn = ctx.host.snap()["mem"].get("qemu_mem_kb") + xs.append((time.time() - ctx.marks["t0"]) / 60.0) + used.append(m["mem"]["used_mb"]) + rss.append(None if sn is None else sn / 1024.0) + ctx.doc["data"]["soak"] = {"minutes": xs, "qemu_mem_mb": rss, "guest_used_mb": used} + g, s0, s1, t_go = core.window(ctx, a.secs, "all-busy") + res[label] = core.build_step(ctx, n, "all-busy", g, s0, s1, t_go) + ctx.doc["data"]["steps"] = [res["start"], res["end"]] + for k, gname, ys, bar in (("QEMU memory (RSS + swap)", "leak_qemu_mem", rss, a.soak_rss_mb_min), ("guest used memory", "leak_guest_used", used, a.soak_guest_mb_min)): + gt = gates.leak_gate(k, xs, ys, bar, "MB/min") + ctx.doc["gates"][gname] = gt + ctx.section(gname, gt["status"], "SOAK %s: %s" % (gt["status"], gt["detail"])) + f0 = min([s["fps"] for s in res["start"]["sims"] if s.get("fps")] or [0]) + f1 = min([s["fps"] for s in res["end"]["sims"] if s.get("fps")] or [0]) + ok = f0 > 0 and f1 >= 0.95 * f0 + ctx.section("soak_fps", "PASS" if ok else "FAIL", "SOAK %s: slowest simulator %.1f fps at the start, %.1f after %g min (bar: not below 95%%)" % ("PASS" if ok else "FAIL", f0, f1, a.soak_minutes)) + ctx.doc["gates"].update(ctx.coh.check(ctx.marks["coh"])) + ctx.doc["gates"]["strip"] = gates.strip_gate_of(res["end"]["sims"]) + cg = [res[k]["counter"] for k in ("start", "end")] + ctx.doc["gates"]["counter"] = next((g for g in cg if g["status"] == "FAIL"), cg[-1]) + for name in ("counter", "strip"): # a failed gate is a FAIL section: it sets the exit code + g = ctx.doc["gates"][name] + ctx.section("gate:" + name, g["status"], "GATE %s %s: %s" % (g["status"], name, g["detail"])) + ctx.save() + + +def run_build(ctx): + a = ctx.a + info = core.setup_guest(ctx, sims=False) + core.capture_env(ctx, info) + r = ctx.guest.call(["build"] + (["--xcodebuild", a.xcodebuild] if a.xcodebuild else []), 3900) + ctx.doc["data"] = r + ok = r.get("tools_cold_rc") == 0 and r.get("tools_warm_rc") == 0 + ctx.section("build", "PASS" if ok else "FAIL", "BUILD %s: bench tools (swiftc, GlassBench + simfps + GenieBench) cold %s s, warm %s s" % ("PASS" if ok else "FAIL", r.get("tools_cold_s"), r.get("tools_warm_s"))) + if a.xcodebuild: + ok = r.get("xcodebuild_cold_rc") == 0 and r.get("xcodebuild_warm_rc") == 0 + ctx.section("xcodebuild", "PASS" if ok else "FAIL", "XCODEBUILD %s: cold %s s, warm %s s" % ("PASS" if ok else "FAIL", r.get("xcodebuild_cold_s"), r.get("xcodebuild_warm_s"))) + else: + ctx.section("xcodebuild", "SKIP", "XCODEBUILD SKIP: GlassBench is not an Xcode project; pass --xcodebuild '' to time yours (cold: new DerivedData, warm: again)") + ctx.save() + + +def run_panel(ctx): + """The Device panel's stream, measured outside this tool: --panel-cmd runs a command that prints one JSON object, + {"frames": [[t_seconds, counter], ...]}: each frame the viewer received and the counter GlassBench's strip showed in it.""" + a = ctx.a + ctx.doc["env"].update({"build_id": a.build_id or "unknown"}) + cmd = a.panel_cmd or os.environ.get("MACBOX_BENCH_PANEL_CMD") + if not cmd: + ctx.section("panel", "SKIP", "PANEL SKIP: no --panel-cmd (needs Simlock and T3 Code's Device panel; see README, 'The panel section')") + ctx.save() + return + p = ctx.run.popen(["/bin/sh", "-c", cmd], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, + env=dict(os.environ, BENCH_PANEL_SECS=str(a.secs))) + try: + out, err = p.communicate(timeout=a.secs + 300) + except subprocess.TimeoutExpired: + ctx.run.kill_children() + ctx.section("panel", "FAIL", "PANEL FAIL: the command did not finish") + ctx.save() + return + try: + frames = json.loads(out.strip().splitlines()[-1])["frames"] + except (ValueError, KeyError, IndexError): + ctx.section("panel", "FAIL", "PANEL FAIL: no JSON with frames on the command's last line (%s)" % err.strip()[-100:]) + ctx.save() + return + an = strip.analyze_counters([(t, c) for t, c in frames]) + m = stats.frame_stats([t for t, _ in an["unique"]], a.nominal_fps) + ok_s, why = strip.strip_gate(an) + from . import targets + tg = targets.parse_target("panel: fps>=%g, p95<=%g" % (a.panel_fps, a.panel_p95)) + ev = targets.evaluate(tg, [m]) + ctx.doc["data"] = {"metrics": m, "strip": {k: an[k] for k in ("repeats", "lost", "backwards", "jumps", "unreadable")}} + ctx.doc["gates"]["strip"] = gates.gate("PASS" if ok_s else "FAIL", why) + ok = bool(ev["pass"]) and ok_s + ctx.section("panel", "PASS" if ok else "FAIL", "PANEL %s: %s distinct/s, interval p50 %s p95 %s p99 %s max %s ms, %d over 2x nominal, longest gap %s s (bars: >= %g/s, p95 <= %g ms); strip %s" % ( + "PASS" if ok else "FAIL", m["fps"], m["p50_ms"], m["p95_ms"], m["p99_ms"], m["max_ms"], m["over2x"], m["longest_gap_s"], a.panel_fps, a.panel_p95, why)) + ctx.save() + + +def run_golden(ctx): + """Records the pixel hash of GlassBench -static for this setup (macOS runtime, device type, scale, size).""" + a = ctx.a + info = core.setup_guest(ctx) + core.capture_env(ctx, info) + core.grow(ctx, 1, ["static"]) + core.settle(ctx, "before the golden screen") + scale.golden_phase(ctx) + seen = getattr(ctx, "golden_seen", None) + if not seen: + ctx.section("golden", "FAIL", "GOLDEN FAIL: the static screen gave different pictures, or none: nothing recorded") + else: + key = ctx.golden_key_for(seen["w"], seen["h"]) + ctx.record_golden(key, seen["hash"]) + ctx.section("golden", "PASS", "GOLDEN recorded for %s: %s" % (key, seen["hash"][:16])) + ctx.save() diff --git a/src/bench/macbench/png.py b/src/bench/macbench/png.py new file mode 100644 index 0000000..fd332c9 --- /dev/null +++ b/src/bench/macbench/png.py @@ -0,0 +1,110 @@ +"""A small PNG reader and writer (8-bit gray, gray+alpha, RGB, RGBA; not interlaced), standard library only. +simctl's screenshots are of that kind. No Pillow.""" +import hashlib +import struct +import zlib + + +class Image: + def __init__(self, width, height, channels, rows): + self.w, self.h, self.ch, self.rows = width, height, channels, rows # rows: list of bytes + + def px(self, x, y): + """(r, g, b) of a pixel.""" + r = self.rows[y] + i = x * self.ch + if self.ch >= 3: + return r[i], r[i + 1], r[i + 2] + return r[i], r[i], r[i] + + def luma(self, x, y): + r, g, b = self.px(x, y) + return (r + g + b) // 3 + + def pixel_hash(self): + """sha256 over the RGB values of every pixel (alpha ignored), with the size mixed in: two screenshots + have the same hash only when every pixel is the same.""" + h = hashlib.sha256() + h.update(struct.pack(">II", self.w, self.h)) + for y in range(self.h): + r = self.rows[y] + if self.ch == 3: + h.update(r) + elif self.ch == 4: + h.update(bytes(v for i, v in enumerate(r) if i % 4 != 3)) + elif self.ch == 2: + h.update(bytes(r[i] for i in range(0, len(r), 2))) + else: + h.update(r) + return h.hexdigest() + + +def _paeth(a, b, c): + p = a + b - c + pa, pb, pc = abs(p - a), abs(p - b), abs(p - c) + return a if pa <= pb and pa <= pc else (b if pb <= pc else c) + + +def read_png(data): + if data[:8] != b"\x89PNG\r\n\x1a\n": + raise ValueError("not a PNG") + pos, idat, w = 8, [], None + while pos + 8 <= len(data): + n, typ = struct.unpack(">I4s", data[pos:pos + 8]) + body = data[pos + 8:pos + 8 + n] + pos += 12 + n + if typ == b"IHDR": + w, h, depth, ctype, _, _, interlace = struct.unpack(">IIBBBBB", body) + if depth != 8 or interlace != 0 or ctype not in (0, 2, 4, 6): + raise ValueError("unsupported PNG (need 8-bit, not interlaced, gray/RGB/RGBA)") + ch = {0: 1, 2: 3, 4: 2, 6: 4}[ctype] + elif typ == b"IDAT": + idat.append(body) + elif typ == b"IEND": + break + if w is None: + raise ValueError("no IHDR") + raw = zlib.decompress(b"".join(idat)) + stride = w * ch + rows, prev = [], bytearray(stride) + p = 0 + for _ in range(h): + ft = raw[p] + line = bytearray(raw[p + 1:p + 1 + stride]) + p += 1 + stride + if ft == 1: + for i in range(ch, stride): + line[i] = (line[i] + line[i - ch]) & 255 + elif ft == 2: + for i in range(stride): + line[i] = (line[i] + prev[i]) & 255 + elif ft == 3: + for i in range(stride): + left = line[i - ch] if i >= ch else 0 + line[i] = (line[i] + ((left + prev[i]) >> 1)) & 255 + elif ft == 4: + for i in range(stride): + left = line[i - ch] if i >= ch else 0 + ul = prev[i - ch] if i >= ch else 0 + line[i] = (line[i] + _paeth(left, prev[i], ul)) & 255 + elif ft != 0: + raise ValueError("bad PNG filter %d" % ft) + rows.append(bytes(line)) + prev = line + return Image(w, h, ch, rows) + + +def write_png(img_or_rows, width=None, height=None, channels=3): + """PNG bytes from an Image, or from rows (a list of bytes, each width*channels long). For tests and fixtures.""" + if isinstance(img_or_rows, Image): + rows, width, height, channels = img_or_rows.rows, img_or_rows.w, img_or_rows.h, img_or_rows.ch + else: + rows = img_or_rows + ctype = {1: 0, 2: 4, 3: 2, 4: 6}[channels] + + def chunk(t, b): + c = struct.pack(">I", len(b)) + t + b + return c + struct.pack(">I", zlib.crc32(t + b) & 0xFFFFFFFF) + raw = b"".join(b"\x00" + bytes(r) for r in rows) + return (b"\x89PNG\r\n\x1a\n" + chunk(b"IHDR", struct.pack(">IIBBBBB", width, height, 8, ctype, 0, 0, 0)) + + chunk(b"IDAT", zlib.compress(raw, 6)) + chunk(b"IEND", b"")) diff --git a/src/bench/macbench/runctx.py b/src/bench/macbench/runctx.py new file mode 100644 index 0000000..da175ae --- /dev/null +++ b/src/bench/macbench/runctx.py @@ -0,0 +1,232 @@ +"""One run's safety: a hard deadline, explicit child pids, knobs that are put back, and ONE cleanup that runs on +every way out (normal end, error, SIGTERM/SIGINT, the deadline). Nothing waits on `wait` without a pid.""" +import atexit +import os +import signal +import subprocess +import sys +import threading +import time + + +class DeadlineError(Exception): + pass + + +class Run: + def __init__(self, deadline_s, log=print, margin=60.0): + self.t0 = time.monotonic() + self.deadline = self.t0 + deadline_s + self.margin = margin # the watchdog acts this long before the deadline + self.log = log + self.children = [] # subprocess.Popen objects this run started + self.undo = [] # (label, fn) restoring a knob, run last-in first-out + self.teardown = [] # (label, fn): shut down what the run started (simulators) + self.on_deadline = None # fn: write the partial result + self.cleaned = False + self.in_cleanup = False + self._lock = threading.Lock() + self.expired = threading.Event() + self.problems = [] # things cleanup found it could not restore + self.aborted = None # (reason, exit code) once a monitor ended the run + self.on_abort = None # fn(reason): records the abort in the result + self.final = None # fn(): after the cleanup, writes the result's closing sections (the cleanup's verdict) + self._done = threading.Event() + + # --- time + def remaining(self): + return self.deadline - time.monotonic() + + def need(self, secs, what): + """Refuse a step that would run past the deadline (the cleanup keeps its own 90 s).""" + if self.remaining() < secs + self.margin + 30: + raise DeadlineError("no time for %s (%d s needed, %d s left before the deadline)" % (what, secs, max(0, self.remaining() - self.margin - 30))) + + # --- children + def popen(self, argv, **kw): + p = subprocess.Popen(argv, start_new_session=True, **kw) + with self._lock: + self.children.append(p) + return p + + def kill_children(self): + for p in list(self.children): + if p.poll() is None: + try: + os.killpg(p.pid, signal.SIGTERM) + except OSError: + pass + t_end = time.monotonic() + 3 + for p in list(self.children): + try: + p.wait(max(0.1, t_end - time.monotonic())) + except subprocess.TimeoutExpired: + try: + os.killpg(p.pid, signal.SIGKILL) + except OSError: + pass + self.children.clear() + + # --- the one trap + def install(self): + atexit.register(self.cleanup) + + def on_signal(signum, frame): + self.log("signal %d: stopping" % signum) + self.expired.set() + sys.exit(128 + signum) # runs atexit -> cleanup + signal.signal(signal.SIGTERM, on_signal) + signal.signal(signal.SIGINT, on_signal) + t = threading.Thread(target=self._watchdog, daemon=True) + t.start() + + def _watchdog(self): + while time.monotonic() < self.deadline - self.margin: + time.sleep(1) + self.log("DEADLINE: %d s left, stopping" % self.margin) + self.expired.set() + if self.on_deadline: + try: + self.on_deadline() + except Exception as e: # the partial result is best effort + self.log("partial result failed: %s" % e) + self.cleanup() + if self.final: # the cleanup's verdict goes into the result of a run that ran out of time too + try: + self.final() + except Exception as e: + self.log("closing the result failed: %s" % e) + os._exit(124) + + # --- monitors: what must stop the run while a long call is waiting + def watch(self, name, fn, every): + """Calls fn() every `every` seconds on a thread until the run is over; fn returns None (all well) or + (reason, exit code): the run is then aborted: partial result, cleanup, exit.""" + def loop(): + while not self.expired.wait(every): + try: + r = fn() + except Exception as e: # a check that cannot run is not a verdict + self.log("monitor %s: %s: %s" % (name, type(e).__name__, e)) + continue + if r: + self.abort(*r) + return + t = threading.Thread(target=loop, daemon=True, name="watch-" + name) + t.start() + return t + + def abort(self, reason, code): + """End the run NOW from a monitor thread: record why, write the partial result, run the one cleanup, write the + closing sections, exit with `code`. The main thread, which may be blocked in a call, is killed with its children.""" + with self._lock: + if self.aborted or self.cleaned: + return + self.aborted = (reason, code) + self.log("ABORT: %s" % reason) + self.expired.set() + for hook, arg in ((self.on_abort, (reason,)), (self.on_deadline, (reason,))): + if hook: + try: + hook(*arg) + except Exception as e: + self.log("abort hook failed: %s" % e) + self.cleanup() + if self.final: + try: + self.final() + except Exception as e: + self.log("closing the result failed: %s" % e) + os._exit(code) + + def join_abort(self): + """Called by the main thread after its own cleanup: when a monitor aborted the run, it owns the exit.""" + if self.aborted: + time.sleep(600) + + def cleanup(self): + with self._lock: + if self.cleaned: + already = True + else: + already = False + self.cleaned = True + if already: + self._done.wait(900) # another thread is cleaning up: do not exit before it is done + return + self.expired.set() + self.in_cleanup = True + self.kill_children() + for label, fn in reversed(self.teardown): + try: + fn() + self.log("cleanup: %s" % label) + except Exception as e: + self.problems.append("%s: %s" % (label, e)) + self.log("cleanup FAILED: %s: %s" % (label, e)) + for label, fn in reversed(self.undo): + try: + fn() + self.log("restored: %s" % label) + except Exception as e: + self.problems.append("%s: %s" % (label, e)) + self.log("restore FAILED: %s: %s" % (label, e)) + self._done.set() + + +class LiveKnob: + """A key=value line in the VM's live switch file: A = the file as it was, B = with the line set. The file's + original text is kept and put back whole on any exit.""" + + def __init__(self, path, key, value, run): + self.path, self.key, self.value, self.run = path, key, value, run + with open(path) as f: + self.original = f.read() + run.undo.append(("live switches file", self.restore)) + + def _write(self, text): + tmp = self.path + ".bench.tmp" + with open(tmp, "w") as f: + f.write(text) + os.replace(tmp, self.path) + + def set(self): + lines = [l for l in self.original.splitlines() if not l.strip().startswith(self.key + "=")] + self._write("\n".join(lines + ["%s=%s" % (self.key, self.value)]) + "\n") + + def unset(self): + self._write(self.original) + + def restore(self): + self._write(self.original) + + def describe(self): + return "live:%s=%s" % (self.key, self.value) + + +class CmdKnob: + """Shell commands for the knob (set / unset) run on the host. The unset command also runs on cleanup.""" + + def __init__(self, set_cmd, unset_cmd, run, runner): + self.set_cmd, self.unset_cmd, self.run, self.runner = set_cmd, unset_cmd, run, runner + run.undo.append(("knob unset command", self.restore)) + + def set(self): + self.runner(self.set_cmd) + + def unset(self): + self.runner(self.unset_cmd) + + def restore(self): + self.runner(self.unset_cmd) + + def describe(self): + return "cmd" + + +def shell_runner(timeout=60): + def run_cmd(cmd): + p = subprocess.run(["/bin/sh", "-c", cmd], capture_output=True, text=True, timeout=timeout) + if p.returncode != 0: + raise RuntimeError("knob command failed (rc %d): %s" % (p.returncode, p.stderr.strip()[:120])) + return run_cmd diff --git a/src/bench/macbench/scale.py b/src/bench/macbench/scale.py new file mode 100644 index 0000000..17f84a1 --- /dev/null +++ b/src/bench/macbench/scale.py @@ -0,0 +1,229 @@ +"""The scaling sweep: N = 1, 2, 3, ... simulators, per-simulator numbers at each N, the maximum sustainable N per +quality target.""" +import time + +from . import analysis, core, gates, stats, targets +from .core import Stop +from .runctx import DeadlineError +from .transport import GuestError + +STEP_GATES = ("counter", "strip", "kernel_gpu", "coherent_census", "coherent_process", "qemu_log", "gpu_hang", "tears", "pixels", "memory") + + +# A torn screenshot fails the step and the run (exit code, the gate's line), but the sweep goes on: the pictures of the +# next N are not made worse by it, and a sweep that ends at the first torn picture measures nothing above it. +SWEEP_GOES_ON = ("tears",) + + +def calibrate(ctx, sim): + """Pick the input injection that is answered (or 'none'): 3 tries of each method on one simulator.""" + a = ctx.a + if a.probe_method != "auto": + ctx.probe_method = a.probe_method + return {"chosen": a.probe_method, "calibrated": False} + # notify first: a Darwin notification puts nothing on the screen but the marker's flip. appearance redraws the whole + # screen (0.5 s to answer, a quarter of the distinct frames lost in a window that uses it): the fallback only. + # Never openurl: iOS 26 answers `simctl openurl` with an "Open in GlassBench?" alert that nobody taps. It stays over + # the app, through a relaunch too, and its glass cost a third of the frame rate (121 -> 80-83 screen frames/s). + seen, chosen = {}, "none" + for m in ("notify", "appearance"): + r = ctx.guest.call(["calibrate", "--udid", sim["udid"], "--methods", m, "--time-scale", 0.3 if ctx.host.dry else 1.0] + (["--probe-cmd", a.probe_cmd] if a.probe_cmd else []), 200) + seen.update(r["methods"]) + if r["methods"].get(m, {}).get("answered", 0) >= 2: + chosen = m + break + ctx.probe_method = chosen + return {"chosen": chosen, "calibrated": True, "methods": seen} + + +def golden_phase(ctx, record=False): + """Pixel-exact reference: the first simulator shows GlassBench -static; its screenshots must agree and match + the stored golden. Returns the pixels gate. Costs a few seconds.""" + s0 = ctx.sims[0] + keep = s0["role"] + ctx.guest.call(["prepare", "--udids", s0["udid"], "--roles", "static"] + (["--quiesce"] if ctx.a.quiesce else []), 120) + s0["role"] = "static" + time.sleep(0.3 if ctx.host.dry else 3) + r = ctx.guest.call(["shots", "--udids", s0["udid"], "--count", 3], 120) + by = [{"idx": 0, "role": "static", "shots": r["sims"][0]["shots"]}] + g = gates.shots_gates(by, ctx.golden)["pixels"] + ok = [x for x in by[0]["shots"] if x.get("ok")] + ctx.golden_seen = {"hash": ok[0]["hash"], "w": ok[0]["w"], "h": ok[0]["h"]} if ok and len({x["hash"] for x in ok}) == 1 else None + s0["role"] = None # forces grow() to launch it with its real role + return g + + +def run_scale(ctx): + a = ctx.a + doc, mix = ctx.doc, a.mix + tg_specs = a.targets + floor = targets.parse_target("floor: " + a.floor) + info = core.setup_guest(ctx) + core.capture_env(ctx, info) + data = doc["data"] = {"mix": mix, "scale_x": a.scale_x, "device": doc["env"]["guest"].get("device_type"), "ns": a.ns, + "nominal_fps": a.nominal_fps, "window_s": a.secs, "targets": tg_specs, "floor": a.floor, "steps": [], + "summary": {}, "calibration": None, "observer": None} + ctx.marks = {"coh": ctx.coh.mark(), "t0": time.time()} + ctx.run.on_deadline = lambda reason="deadline": (_finish(ctx, reason), None)[1] + stopped, first, kind = None, True, None + try: + for n in a.ns: + roles = core.roles_for(mix, n) + reason = core.guard(ctx, n - len(ctx.sims), "before booting simulator %d" % n) + if reason: + stopped, kind = reason, "guard" + break + ctx.host.n = n + boot = core.grow(ctx, n, roles) + if first: + if mix in ("all-busy", "one-busy"): + data["calibration"] = calibrate(ctx, ctx.sims[0]) + core.settle(ctx, "before the input calibration", bg=90) + if a.golden_phase: + ctx.pixel_gate = golden_phase(ctx) + core.grow(ctx, n, roles) + st = core.settle(ctx, "at N=%d" % n) + g, s0, s1, t_go = core.window(ctx, a.secs, mix) + step = core.build_step(ctx, n, mix, g, s0, s1, t_go, boot) + step["settle"] = {k: st.get(k) for k in ("settled", "bg_pct", "waited_s")} + shots, by = core.take_shots(ctx, a.shots) if a.shots and mix != "idle" else ({}, []) + if first and ctx.pixel_gate: + shots["pixels"] = ctx.pixel_gate # judged on the freshly launched static app, not after input probes + core.step_gates(ctx, step, shots) + step["window"] = {"start_offset_s": round(t_go - ctx.marks["t0"] - ctx.offset, 1)} + data["steps"].append(step) + if first and a.observer_check and mix in ("all-busy", "one-busy", "agent"): + data["observer"] = observer_check(ctx, step, mix) + _est(ctx, data["steps"]) + _finish(ctx, None, partial=True) + first = False + bad = [k for k, v in step["gates"].items() if v["status"] == "FAIL"] + if [k for k in bad if k not in SWEEP_GOES_ON]: + stopped, kind = "gate failed at N=%d: %s" % (n, ", ".join(bad)), "gate" + break + if not step["stable"]: + stopped, kind = "instability at N=%d: %s" % (n, _why_unstable(step)), "instability" + break + fl = targets.evaluate(floor, step["sims"]) + if fl["pass"] is False: + kind = "floor" + stopped = "floor %s breached at N=%d (sim %s: %s = %s)" % (a.floor, n, fl["failing"][0]["sim"], fl["failing"][0]["cond"], fl["failing"][0]["value"]) + break + except Stop as e: + stopped, kind = str(e), "error" + except (DeadlineError, GuestError) as e: + stopped, kind = "%s: %s" % (type(e).__name__, e), "error" + data["stopped_kind"] = kind + return _finish(ctx, stopped) + + +def _why_unstable(step): + s = step["stability"] + out = [] + if s["new_ips"]: + out.append("%d new crash report(s)" % s["new_ips"]) + if s["sims_died"]: + out.append("simulator(s) %s no longer booted" % s["sims_died"]) + if s.get("foreign_booted") and s["foreign_booted"]: + out.append("%d simulator(s) outside the bench were booted during the window" % s["foreign_booted"]) + if s["kernel_gpu_lines"]: + out.append("%d kernel GPU error line(s)" % s["kernel_gpu_lines"]) + if not s["qemu_same"]: + out.append("QEMU restarted") + return "; ".join(out) + + +def _est(ctx, steps): + """A simulator's guest memory from the steps so far (for the guard).""" + if len(steps) >= 2: + d = [(b["system"]["guest"]["mem_used_mb"] - a["system"]["guest"]["mem_used_mb"]) / max(1, b["n"] - a["n"]) for a, b in zip(steps, steps[1:])] + ctx.est_sim_mb = max(300.0, max(d)) + elif steps: + ctx.est_sim_mb = max(ctx.est_sim_mb, 1100.0) + + +def observer_check(ctx, step, mix): + """Decision: if reading the strip moves the surface frame rate by more than the noise (4%), every distinct-frame + number in this result carries that observer cost. A second window of the same N reads nothing from the screen + and injects nothing.""" + g, s0, s1, t_go = core.window(ctx, ctx.a.secs, mix, no_strip=True) + with_strip = [s["surface_fps"] for s in step["sims"] if s.get("role") in ("busy", "agent") and s.get("surface_fps")] + without = [s["simfps"]["fps"] for s in g["sims"] if s["role"] in ("busy", "agent") and s.get("simfps")] + missing = len([s for s in g["sims"] if s["role"] in ("busy", "agent")]) - len(without) # a counter that answered nothing is not a measurement + ws, wo = stats.mean(with_strip), stats.mean(without) + d = None if missing else stats.rel_diff(wo, ws) + return {"n": step["n"], "surface_fps_with_strip": stats.r1(ws), "surface_fps_without_strip": stats.r1(wo), + "delta": None if d is None else round(d, 4), "missing": missing, "ok": d is not None and abs(d) <= 0.04} + + +def _finish(ctx, stopped, partial=False): + """Summary, sections and the result file. Also the deadline's partial write (partial=True keeps the doc open).""" + a, doc = ctx.a, ctx.doc + data = doc["data"] + steps = data["steps"] + if not steps and not stopped: + return None + data["no_steps"] = not steps + data["summary"] = analysis.summarize_scale(steps, a.targets, stopped) if steps else {"max_n": {}, "stopped": stopped, "standing_cost": {}} + doc["data"]["summary"]["stopped"] = stopped + doc["gates"] = _gate_rollup(steps, ctx) + if not partial: + doc["sections"] = [s for s in doc["sections"] if s["name"] not in ("scale", "gates", "stability", "observer", "input", "targets", "cost")] + _sections(ctx, data, stopped) + ctx.save() + return data["summary"] + + +def _gate_rollup(steps, ctx): + roll = {} + for name in STEP_GATES: + rows = [(st["n"], st["gates"].get(name)) for st in steps if st.get("gates", {}).get(name)] + if not rows: + roll[name] = gates.gate("SKIP", "not run") + continue + bad = [(n, g) for n, g in rows if g["status"] == "FAIL"] + if bad: + roll[name] = gates.gate("FAIL", "N=%d: %s" % (bad[0][0], bad[0][1]["detail"])) + elif all(g["status"] == "SKIP" for _, g in rows): + roll[name] = gates.gate("SKIP", rows[-1][1]["detail"]) + else: + judged = [g for _, g in rows if g["status"] != "SKIP"] + roll[name] = gates.gate("PASS", "%d of %d step(s) judged; %s" % (len(judged), len(rows), judged[-1]["detail"])) + return roll + + +def _sections(ctx, data, stopped): + a = ctx.a + steps = data["steps"] + sm = data["summary"] + gate_fail = [k for k, v in ctx.doc["gates"].items() if v["status"] == "FAIL"] + unstable = any(not s["stable"] for s in steps) + mx = ", ".join("%s=%s" % (k, v["max_n"] if v["max_n"] is not None else "none") for k, v in sm["max_n"].items()) + status = "FAIL" if gate_fail or unstable or not steps or data.get("stopped_kind") == "error" else "PASS" + unsettled = [s["n"] for s in steps if (s.get("settle") or {}).get("settled") is False] + ctx.section("scale", status, "SCALE %s %s %dx %s: max sustainable N: %s | steps %s%s%s | min Jain %s" % ( + status, a.mix, a.scale_x, "(DRY)" if ctx.host.dry else "", mx or "none (no step ran)", [s["n"] for s in steps], + (" | stopped: %s" % stopped) if stopped else " | all requested N done", + (" | NOT SETTLED at N=%s (guest background CPU stayed above %g%%)" % (unsettled, a.settle_bg)) if unsettled else "", sm.get("fairness_min_jain"))) + for name, v in ctx.doc["gates"].items(): + ctx.section("gate:" + name, v["status"] if v["status"] != "PASS" else "PASS", "GATE %s %s: %s" % (v["status"], name, v["detail"])) + st = [s for s in steps] + ctx.section("stability", "FAIL" if unstable else "PASS", "STABILITY %s: %s" % ( + "FAIL" if unstable else "PASS", "; ".join("N=%d %s" % (s["n"], _why_unstable(s)) for s in st if not s["stable"]) or "no crash report, no dead simulator, no kernel GPU line, one QEMU")) + cal = data.get("calibration") + if cal: + ctx.section("input", "INFO", "INPUT PROXY method %s %s" % (cal["chosen"], "(calibrated: %s)" % {k: "%d/%d" % (v["answered"], v["n"]) for k, v in cal.get("methods", {}).items()} if cal.get("calibrated") else "(given)")) + ob = data.get("observer") + if ob: + # a cost of looking is a property of the measurement, not a failure of the system: INFO, never FAIL. A window + # whose counter did not answer (delta None) is the failure. + status = "FAIL" if ob["delta"] is None else "PASS" if ob["ok"] else "INFO" + ctx.section("observer", status, "OBSERVER %s: surface fps %s measured (strip and marker read, events injected), %s on an unviewed screen (%s): %s" % ( + status, ob["surface_fps_with_strip"], ob["surface_fps_without_strip"], + "n/a" if ob["delta"] is None else "%+.1f%%" % (ob["delta"] * 100), + "the unviewed window's counter did not answer" if ob["delta"] is None else "within the noise" if ob["ok"] + else "looking at the screen costs frames (a lock of every picture, the injected events): the distinct-fps numbers are lower bounds")) + sc = sm.get("standing_cost") or {} + if sc: + parts = ["%s %+.2f/sim" % (k, v["per_sim"]) for k, v in sc.items() if k in ("qemu_cpu_cores", "gpu_busy_pct", "guest_mem_used_mb", "guest_sims_cpu_pct", "guest_sims_rss_mb")] + ctx.section("cost", "INFO", "STANDING COST per simulator (slope over N): " + ", ".join(parts)) diff --git a/src/bench/macbench/schema.py b/src/bench/macbench/schema.py new file mode 100644 index 0000000..1432eda --- /dev/null +++ b/src/bench/macbench/schema.py @@ -0,0 +1,139 @@ +"""The result file: one JSON document per run, schema versioned. Nothing in it names a host, an account or a path.""" +import datetime +import json +import os +import re +import socket + +SCHEMA = 1 +KINDS = ("scale", "ab", "idle", "lifecycle", "build", "soak", "panel", "gates", "seed") +SECTION_STATUS = ("PASS", "FAIL", "SKIP", "INFO") + +_PATH = re.compile(r"(?:^|[\s\"'=:(,])(/(?:srv|home|Users|var|tmp|private|etc|opt|mnt|root|usr|run|Volumes|proc|sys)/[^\s\"',)]*)") +_IPV4 = re.compile(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b") + + +def now_utc(): + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def stamp(): + return datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d-%H%M%S") + + +def scrubber(extra_names=()): + """A function that cleans a string: home paths -> ~, the host name, ssh alias and other names -> , + IPv4 addresses -> . Paths not under a home directory are left for validate() to refuse.""" + names = {n for n in [socket.gethostname(), socket.gethostname().split(".")[0], *extra_names] if n and len(n) > 2} + home = os.path.expanduser("~") + + def clean(s): + if home and home != "/": + s = s.replace(home, "~") + s = re.sub(r"/Users/[^/\s\"']+", "~", s) + for n in sorted(names, key=len, reverse=True): + s = re.sub(r"(?", s) + return _IPV4.sub("", s) + return clean + + +def scrub(obj, clean): + if isinstance(obj, str): + return clean(obj) + if isinstance(obj, list): + return [scrub(x, clean) for x in obj] + if isinstance(obj, dict): + return {k: scrub(v, clean) for k, v in obj.items()} + return obj + + +def _strings(obj): + if isinstance(obj, str): + yield obj + elif isinstance(obj, list): + for x in obj: + yield from _strings(x) + elif isinstance(obj, dict): + for k, v in obj.items(): + yield str(k) + yield from _strings(v) + + +def privacy_problems(doc, extra_names=()): + """Strings in a result that would not be safe to publish: absolute paths, IPv4 addresses, host names.""" + names = {n for n in [socket.gethostname(), socket.gethostname().split(".")[0], *extra_names] if n and len(n) > 2} + bad = [] + for s in _strings(doc): + if _PATH.search(s): + bad.append("path: " + s[:80]) + if _IPV4.search(s): + bad.append("ip: " + s[:80]) + for n in names: + if re.search(r"(? None.""" + if not values: + return None + s = sorted(values) + return s[min(len(s) - 1, int((len(s) - 1) * p))] + + +def mean(values): + return sum(values) / len(values) if values else None + + +def median(values): + return percentile(values, 0.5) + + +def r1(x, digits=1): + return None if x is None else round(x, digits) + + +def intervals_ms(times_s): + """Gaps between consecutive times (seconds, ascending) in milliseconds.""" + return [(b - a) * 1000.0 for a, b in zip(times_s, times_s[1:])] + + +def frame_stats(times_s, nominal_fps=60.0): + """Per-simulator frame metrics from the times (seconds) of its distinct frames. + + fps (n - 1) / (last - first) + p50..max of the intervals between consecutive distinct frames, ms + over2x intervals longer than 2 / nominal_fps + longest_gap_s the longest interval in seconds + """ + n = len(times_s) + out = {"frames": n, "fps": 0.0, "p50_ms": None, "p95_ms": None, "p99_ms": None, "max_ms": None, + "over2x": 0, "longest_gap_s": None} + if n < 2: + return out + iv = intervals_ms(times_s) + span = times_s[-1] - times_s[0] + out["fps"] = r1((n - 1) / span if span > 0 else 0.0) + out["p50_ms"] = r1(percentile(iv, 0.5)) + out["p95_ms"] = r1(percentile(iv, 0.95)) + out["p99_ms"] = r1(percentile(iv, 0.99)) + out["max_ms"] = r1(max(iv)) + thr = 2000.0 / nominal_fps + out["over2x"] = sum(1 for x in iv if x > thr) + out["longest_gap_s"] = r1(max(iv) / 1000.0, 3) + return out + + +def stretches(times_s, still_ms=250.0): + """Split frame times into stretches of motion: a gap above still_ms ends a stretch (the screen stood + still or the app froze that long). Returns [(first, last, count)] for each stretch of 2+ frames.""" + out, cur = [], [] + for t in times_s: + if cur and (t - cur[-1]) * 1000.0 > still_ms: + if len(cur) > 1: + out.append((cur[0], cur[-1], len(cur))) + cur = [] + cur.append(t) + if len(cur) > 1: + out.append((cur[0], cur[-1], len(cur))) + return out + + +def jain(values): + """Jain's fairness index (sum x)^2 / (n * sum x^2): 1.0 when every value is equal, 1/n when one + simulator gets everything. None for no values; 1.0 for all zeros.""" + if not values: + return None + s2 = sum(v * v for v in values) + if s2 == 0: + return 1.0 + return sum(values) ** 2 / (len(values) * s2) + + +def min_max_ratio(values): + """min / max (1.0 = equal). None for no values, 1.0 for all zeros.""" + if not values: + return None + hi = max(values) + return 1.0 if hi == 0 else min(values) / hi + + +def slope(xs, ys): + """Least-squares line through (x, y): (slope, intercept, r2). Needs 2+ distinct x, else (None, None, None). + The per-simulator standing cost of a resource is the slope over N.""" + pts = [(x, y) for x, y in zip(xs, ys) if x is not None and y is not None] + n = len(pts) + if n < 2: + return None, None, None + mx = sum(p[0] for p in pts) / n + my = sum(p[1] for p in pts) / n + sxx = sum((p[0] - mx) ** 2 for p in pts) + if sxx == 0: + return None, None, None + sxy = sum((p[0] - mx) * (p[1] - my) for p in pts) + b = sxy / sxx + a = my - b * mx + syy = sum((p[1] - my) ** 2 for p in pts) + r2 = 1.0 if syy == 0 else (sxy * sxy) / (sxx * syy) + return b, a, r2 + + +def rel_diff(a, b): + """(b - a) / a as a fraction; None when a is 0 or missing.""" + if a is None or b is None or a == 0: + return None + return (b - a) / a + + +def finite(x): + return x is not None and isinstance(x, (int, float)) and math.isfinite(x) diff --git a/src/bench/macbench/strip.py b/src/bench/macbench/strip.py new file mode 100644 index 0000000..299d2de --- /dev/null +++ b/src/bench/macbench/strip.py @@ -0,0 +1,88 @@ +"""GlassBench's frame strip: a 16-bit counter drawn on every display-link tick as 18 squares of 20 pt from +(21, 120) pt of a 402 pt wide screen: white and black references, then bits 0..15 (white = 1).""" +from . import stats + +WRAP = 65536 +MAX_FWD = 1000 # a forward step of up to this many ticks is a stall (frames lost); larger is a jump + + +def read_counter_luma(luma_at, width_px): + """The counter from a luma function luma_at(x_px, y_px). Same geometry as simfps's readCounter. + Returns the counter, or -1 when the references are unreadable (no strip, or it is covered).""" + scale = width_px / 402.0 + + def block(i): + return luma_at(int((21 + 20 * i + 10) * scale), int(130 * scale)) + white, black = block(0), block(1) + if white - black <= 100: + return -1 + mid = (white + black) // 2 + n = 0 + for b in range(16): + if block(b + 2) > mid: + n |= 1 << b + return n + + +def read_counter_image(img): + return read_counter_luma(img.luma, img.w) + + +def analyze_counters(samples): + """Distinct app frames from the strip samples [(t_seconds, counter or -1)] of one simulator. + + A sample whose counter equals the previous distinct one is a repeat (the compositor made another + surface; normal). A counter that moved forward by d: d - 1 ticks never reached the screen (lost). + A counter that moved BACKWARD by up to 1000 (an older frame shown after a newer one) or jumped + forward by more than 1000 is `backwards` / `jumps`: never normal, and the strip gate fails on them. + Returns {"unique": [(t, counter)], "repeats", "lost", "backwards", "jumps", "unreadable", "span_ticks"} + where span_ticks is the counter distance covered, i.e. ticks the app made between first and last frame.""" + unique, repeats, lost, back, jumps, unreadable, span = [], 0, 0, 0, 0, 0, 0 + for t, c in samples: + if c < 0: + unreadable += 1 + continue + if unique: + d = (c - unique[-1][1]) % WRAP + if d == 0: + repeats += 1 + continue + if d <= MAX_FWD: + lost += d - 1 + span += d + elif d >= WRAP - MAX_FWD: + back += 1 # went back: do not advance the reference, the next sample is judged against it + continue + else: + jumps += 1 + span += d + unique.append((t, c)) + return {"unique": unique, "repeats": repeats, "lost": lost, "backwards": back, "jumps": jumps, + "unreadable": unreadable, "span_ticks": span} + + +def strip_gate(analysis, app_ticks=None, max_unreadable_frac=0.01): + """(ok, reason) of the strip gate. + FAIL: a frame out of order (backwards), a jump, a counter that advanced more than the app says it ticked + (reading garbage), or more than 1% of samples unreadable. Lost ticks are NOT a failure: they are the fps + metric (a simulator that drops frames is slow, not wrong). Skips are judged against the app's own tick + count (GlassBench's pace.json) when given.""" + a = analysis + total = len(a["unique"]) + a["repeats"] + a["unreadable"] + a["backwards"] + if a["backwards"]: + return False, "%d frame(s) shown out of order" % a["backwards"] + if a["jumps"]: + return False, "%d counter jump(s) over %d ticks" % (a["jumps"], MAX_FWD) + if app_ticks is not None and a["span_ticks"] > app_ticks * 1.02 + 2: + return False, "counter advanced %d ticks but the app ticked %d times" % (a["span_ticks"], app_ticks) + if total and a["unreadable"] / total > max_unreadable_frac: + return False, "%d of %d samples unreadable" % (a["unreadable"], total) + return True, "ok" + + +def distinct_metrics(samples, nominal_fps=60.0): + """frame_stats of the distinct frames, plus the strip bookkeeping, for one simulator.""" + a = analyze_counters(samples) + m = stats.frame_stats([t for t, _ in a["unique"]], nominal_fps) + m.update({k: a[k] for k in ("repeats", "lost", "backwards", "jumps", "unreadable", "span_ticks")}) + return m diff --git a/src/bench/macbench/targets.py b/src/bench/macbench/targets.py new file mode 100644 index 0000000..cf321fe --- /dev/null +++ b/src/bench/macbench/targets.py @@ -0,0 +1,81 @@ +"""Quality targets ("every sim >= 55 distinct/s and p95 <= 25 ms") and the maximum sustainable N for each. + +A target is `name: cond, cond, ...`; a cond is ` ` with op in >= <= > <. +Metrics (per simulator): fps (distinct app frames/s), p50, p95, p99, max (ms between distinct frames), over2x, +stall (longest gap between distinct frames, s), input_p50, input_p95 (ms, proxy; misses count at the timeout), +cpu (guest CPU % of one core), rss (MB). A metric a simulator has no value for (fps of an idle simulator, +the input proxy where no probe ran) is not judged for that simulator; a target with a condition that NO simulator of +an N has a value for is `None` (unknown), never a pass (unless another condition already failed it).""" +import re + +METRICS = {"fps": "fps", "p50": "p50_ms", "p95": "p95_ms", "p99": "p99_ms", "max": "max_ms", "over2x": "over2x", + "stall": "longest_gap_s", "input_p50": "input_p50_ms", "input_p95": "input_p95_ms", + "cpu": "guest_cpu_pct", "rss": "guest_rss_mb"} +OPS = {">=": lambda a, b: a >= b, "<=": lambda a, b: a <= b, ">": lambda a, b: a > b, "<": lambda a, b: a < b} + +DEFAULT_TARGETS = [ + "smooth60: fps>=55, p95<=25", + "smooth30: fps>=30, p95<=50", + # the proxy's floor is the injecting command itself: `simctl spawn notifyutil` answered in 280 ms (p50, one 3x + # simulator alone, the command took 355 ms). 500 ms leaves about 200 ms for the simulator to be late. + "responsive: input_p95<=500, stall<0.3", +] +_COND = re.compile(r"^\s*([a-z0-9_]+)\s*(>=|<=|>|<)\s*([0-9.]+)\s*$") + + +def parse_target(spec): + """'name: fps>=55, p95<=25' -> {"name", "conds": [(metric, op, value)]}""" + if ":" not in spec: + raise ValueError("target %r: expected name: cond, cond" % spec) + name, rest = spec.split(":", 1) + conds = [] + for c in rest.split(","): + m = _COND.match(c) + if not m or m.group(1) not in METRICS: + raise ValueError("target %r: bad condition %r (metrics: %s)" % (spec, c.strip(), ", ".join(METRICS))) + conds.append((m.group(1), m.group(2), float(m.group(3)))) + if not conds: + raise ValueError("target %r has no condition" % spec) + return {"name": name.strip(), "conds": conds} + + +def evaluate(target, sims): + """target against one N's per-simulator dicts: {"pass": True|False|None, "failing": [...], "judged": int}""" + failing, judged = [], 0 + unjudged = set(c[0] for c in target["conds"]) + for sim in sims: + for metric, op, val in target["conds"]: + v = sim.get(METRICS[metric]) + if v is None: + continue + judged += 1 + unjudged.discard(metric) + if not OPS[op](v, val): + failing.append({"sim": sim.get("idx"), "cond": "%s%s%g" % (metric, op, val), "value": v}) + if judged == 0 or (unjudged and not failing): + # a condition no simulator has a value for is unknown, and so is the target (never a pass by default) + return {"pass": None, "failing": failing, "judged": judged, "unjudged": sorted(unjudged)} + return {"pass": not failing, "failing": failing, "judged": judged} + + +def max_sustainable(steps, target): + """steps: [{"n", "sims": [...], "gates_ok": bool, "stable": bool}] in any order. The largest N such that this + N and every smaller measured N passed the target with its gates ok and no instability. The first N that does + not pass ends it (a pass above a fail is noise or luck, not sustainability). + Returns {"max_n": int|None, "first_fail": int|None, "unknown_at": int|None, "evaluated": {n: pass}}""" + best, first_fail, unknown, ev = None, None, None, {} + for st in sorted(steps, key=lambda s: s["n"]): + r = evaluate(target, st["sims"]) + ok = r["pass"] + if ok and not (st.get("gates_ok", True) and st.get("stable", True)): + ok = False + ev[st["n"]] = ok + if first_fail is not None or unknown is not None: + continue + if ok: + best = st["n"] + elif ok is None: + unknown = st["n"] + else: + first_fail = st["n"] + return {"max_n": best, "first_fail": first_fail, "unknown_at": unknown, "evaluated": ev} diff --git a/src/bench/macbench/tears.py b/src/bench/macbench/tears.py new file mode 100644 index 0000000..89a1a6c --- /dev/null +++ b/src/bench/macbench/tears.py @@ -0,0 +1,60 @@ +"""Tear detector for GlassBench screenshots (ported from the private tears.py; no Pillow). + +Each card is a vertical gradient, so the strip at the card's right edge (no text) changes by about one level +a row at 3x. A band of rows taken from another frame (the scroll moved meanwhile) shows as a step there, away +from card edges, and moves the cards' spacing away from 336 px (at 3x). Calibrated for GlassBench; it needs a +screenshot of the scrolling app (it says `applicable: false` when it finds no cards).""" + + +def _row_means(img, x0, x1, y0, y1): + rows = [] + n = max(1, x1 - x0) + for y in range(img.h): + acc = [0, 0, 0] + if y0 - 80 <= y <= y1 + 80: # rows outside the checked band are not needed + for x in range(x0, x1): + r, g, b = img.px(x, y) + acc[0] += r; acc[1] += g; acc[2] += b + rows.append((acc[0] / n, acc[1] / n, acc[2] / n)) + return rows + + +def detect_tears(img): + """{"tear": bool, "applicable": bool, "odd": [...], "steps": [(row, size)], "gaps": [heights]}""" + w, h = img.w, img.h + s = w / 1206.0 + x0, x1 = int(1125 * s), max(int(1125 * s) + 1, int(1150 * s)) + # The band starts below the navigation bar's glass, which ends at 420 px (3x): a gap that scrolls under the glass + # is cut there and looked like a short gap at a wrong distance (a "tear" in every third screenshot). + y0, y1 = int(440 * s), min(int(1700 * s), h - 1) + rows = _row_means(img, x0, x1, y0, y1) + gap = [min(c) > 225 for c in rows] # the white background between cards + margin = int(75 * s) + near = [False] * h + for y in range(h): + if gap[y]: + for k in range(max(0, y - margin), min(h, y + margin + 1)): + near[k] = True + thr = 6.0 if s > 0.9 else 6.0 / s * 0.5 + steps = [] + for y in range(y0 + 1, y1): + if near[y] or near[y - 1]: + continue + d = max(abs(a - b) for a, b in zip(rows[y], rows[y - 1])) + if d > thr: + steps.append((y, round(d, 1))) + gaps, y = [], y0 + while y < y1: + if gap[y]: + st = y + while y < y1 and gap[y]: + y += 1 + gaps.append((st, y - st)) + y += 1 + gaps = [g for g in gaps if g[0] > y0 + 1 and g[0] + g[1] < y1 - 1] # whole gaps only + spacing = [b[0] - a[0] for a, b in zip(gaps, gaps[1:])] + exp, tol = 336 * s, max(2, 4 * s) + odd = [x for x in spacing if abs(x - exp) > tol] + [g[1] for g in gaps if abs(g[1] - 58 * s) > tol] + # As in the private tool, the card spacing and gap heights decide; steps in the gradient are reported only. + return {"tear": bool(odd), "applicable": len(gaps) >= 2, "odd": odd, "steps": steps[:8], + "gaps": [g[1] for g in gaps]} diff --git a/src/bench/macbench/transport.py b/src/bench/macbench/transport.py new file mode 100644 index 0000000..eb70bab --- /dev/null +++ b/src/bench/macbench/transport.py @@ -0,0 +1,151 @@ +"""Reaching the guest: ssh to the Mac (or the same helper run locally on stubs for DRY=1).""" +import json +import os +import shlex +import subprocess +import sys +import time + +from .runctx import DeadlineError + +GUEST_DIR = "~/.macbox/bench" + + +class GuestError(Exception): + pass + + +class Pending: + """A helper call running in the background: .wait() gives its JSON. The process is one of the run's children.""" + + def __init__(self, guest, p, what, timeout): + self.guest, self.p, self.what, self.timeout, self.t0 = guest, p, what, timeout, time.monotonic() + + def wait(self): + left = max(1, self.timeout - (time.monotonic() - self.t0)) + try: + out, err = self.p.communicate(timeout=left) + except subprocess.TimeoutExpired: + self.guest.run.kill_children() + raise GuestError("%s: no answer in %ds" % (self.what, self.timeout)) + return self.guest.parse(self.what, self.p.returncode, out, err) + + +class SshGuest: + def __init__(self, run, ssh_host, set_path, stubs_dir=None): + self.run, self.ssh_host, self.set_path = run, ssh_host, set_path + self.calls = 0 + self.fault_ssh_timeout = None # DRY: fail the k-th call (tests) + + def argv(self, args): + src = os.environ.get("BENCH_SIMLOCK_SOURCES") # the guest expands a leading ~ itself + remote = "cd %s && BENCH_SET=\"$HOME/.macbox/bench/sets/%s\" %sxcrun python3 guest/benchguest.py %s" % ( + GUEST_DIR, self.set_path, "BENCH_SIMLOCK_SOURCES=%s " % shlex.quote(src) if src else "", + " ".join(shlex.quote(str(a)) for a in args)) + return ["ssh", "-o", "ConnectTimeout=10", "-o", "BatchMode=yes", "-o", "ServerAliveInterval=15", self.ssh_host, remote] + + def parse(self, what, rc, out, err): + lines = [l for l in (out or "").strip().splitlines() if l.startswith("{")] + if not lines: + raise GuestError("%s: rc %s: %s" % (what, rc, (err or out or "no output").strip()[-200:])) + d = json.loads(lines[-1]) + if isinstance(d, dict) and "error" in d: + raise GuestError("%s: %s" % (what, d["error"])) + return d + + def _check(self, what, timeout, counted=True): + if self.run.in_cleanup: + return min(timeout, 240) + if self.run.expired.is_set(): + raise DeadlineError("stopping: %s not started" % what) + if self.run.remaining() < 5: + raise DeadlineError("deadline: %s not started" % what) + if counted: + self.calls += 1 + if self.fault_ssh_timeout and self.calls == self.fault_ssh_timeout: + raise GuestError("%s: ssh timed out (injected)" % what) + return min(timeout, max(5, int(self.run.remaining()))) + + def start(self, args, timeout=300, counted=True): + what = args[0] + timeout = self._check(what, timeout, counted) + p = self.run.popen(self.argv(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + return Pending(self, p, what, timeout) + + def call(self, args, timeout=300, counted=True): + return self.start(args, timeout, counted).wait() + + def clock_offset(self): + """Guest clock minus host clock in seconds, from the quickest of 5 round trips.""" + best = None + for _ in range(5): + t1 = time.time() + try: + p = subprocess.run(self.argv_plain("python3 -c 'import time; print(time.time())'"), capture_output=True, text=True, timeout=20) + g = float(p.stdout.strip().splitlines()[-1]) + except (OSError, ValueError, IndexError, subprocess.TimeoutExpired): + continue + t2 = time.time() + if best is None or t2 - t1 < best[0]: + best = (t2 - t1, g - (t1 + t2) / 2) + return best[1] if best else 0.0 + + def argv_plain(self, cmd): + return ["ssh", "-o", "ConnectTimeout=10", "-o", "BatchMode=yes", self.ssh_host, cmd] + + def alive(self): + """The Mac answers over ssh (a monitor's question: it does not count as one of the run's calls).""" + try: + return subprocess.run(self.argv_plain("true"), capture_output=True, timeout=20).returncode == 0 + except (OSError, subprocess.TimeoutExpired): + return False + + def _env(self): + return None + + def quick(self, args, timeout=60): + """A helper call for the monitors: not counted, never injected with the DRY faults of the run's own calls, and a + timeout kills only itself (a hung monitor must not take the run's own call down with it).""" + try: + p = subprocess.run(self.argv(args), capture_output=True, text=True, timeout=timeout, env=self._env(), cwd=getattr(self, "bench_dir", None)) + except subprocess.TimeoutExpired: + raise GuestError("%s: no answer in %ds" % (args[0], timeout)) + return self.parse(args[0], p.returncode, p.stdout, p.stderr) + + def sh(self, cmd, timeout=120): + """A plain command in the guest: (rc, stdout).""" + if self.run.expired.is_set() and not self.run.in_cleanup: + raise DeadlineError("stopping") + p = subprocess.run(self.argv_plain(cmd), capture_output=True, text=True, timeout=min(timeout, max(5, int(self.run.remaining())))) + return p.returncode, p.stdout + + +class LocalGuest(SshGuest): + """DRY=1: the same helper, run here with its stub primitives. State in a directory, so calls share one fake guest.""" + + def __init__(self, run, set_path, state_dir, bench_dir, faults=""): + super().__init__(run, "stub", set_path) + self.state_dir, self.bench_dir, self.faults = state_dir, bench_dir, faults + self.set_dir = os.path.join(state_dir, "set") + + def argv(self, args): + return [sys.executable, os.path.join(self.bench_dir, "guest", "benchguest.py"), "--set", self.set_dir] + [str(a) for a in args] + + def start(self, args, timeout=300, counted=True): + what = args[0] + timeout = self._check(what, timeout, counted) + p = self.run.popen(self.argv(args), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, env=self._env(), cwd=self.bench_dir) + return Pending(self, p, what, timeout) + + def _env(self): + return dict(os.environ, BENCHGUEST_STUB="1", BENCHGUEST_STATE=self.state_dir, BENCH_STUB_FAULT=self.faults, + BENCHGUEST_CALLS=os.path.join(self.state_dir, "calls.log"), PYTHONPATH=self.bench_dir) + + def clock_offset(self): + return 0.0 + + def alive(self): + return True + + def sh(self, cmd, timeout=120): + return 0, "" diff --git a/src/bench/run b/src/bench/run new file mode 100755 index 0000000..4090e53 --- /dev/null +++ b/src/bench/run @@ -0,0 +1,11 @@ +#!/usr/bin/env python3 +"""macbox bench: the scaling, regression and A/B suite. `macbox bench `, or this file directly. +See README.md here, and `run --help`.""" +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +from macbench.cli import main # noqa: E402 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/bench/simfps.swift b/src/bench/simfps.swift index 58b1f7c..72e40cf 100644 --- a/src/bench/simfps.swift +++ b/src/bench/simfps.swift @@ -6,6 +6,14 @@ // // simfps [label] [--strip] prints one JSON line // +// Environment (all optional; used by `macbox bench`): +// SIMFPS_SET= a simulator device set of its own, instead of the default one +// SIMFPS_START_AT= attach first, then wait for this wall-clock second to start counting, so that +// several simfps (one per simulator) measure the same window +// SIMFPS_PROBE=, a point in points (402 pt wide screen): its luma is read from every new surface +// SIMFPS_STAMPS= one line per new surface: " " +// (counter -3: the same surface looked at again, its marker changed since the first look) +// // --strip also reads GlassBench's frame strip (a 16-bit counter the app draws every tick, see // GlassBench/main.swift) from each new surface: how many app frames reached the screen, how many were // lost on the way (gaps in the counter) and how many surface frames showed no new app frame (repeats). @@ -39,7 +47,7 @@ for p in ["/Library/Developer/PrivateFrameworks/CoreSimulator.framework/CoreSimu guard let ctxClass = NSClassFromString("SimServiceContext") as? NSObject.Type, let ctx = ctxClass.perform(NSSelectorFromString("sharedServiceContextForDeveloperDir:error:"), with: dev, with: nil)? .takeUnretainedValue() as? NSObject, - let set = ctx.perform(NSSelectorFromString("defaultDeviceSetWithError:"), with: nil)?.takeUnretainedValue() as? NSObject, + let set = (ProcessInfo.processInfo.environment["SIMFPS_SET"].map { ctx.perform(NSSelectorFromString("deviceSetWithPath:error:"), with: $0, with: nil) } ?? ctx.perform(NSSelectorFromString("defaultDeviceSetWithError:"), with: nil))?.takeUnretainedValue() as? NSObject, let byUDID = set.value(forKey: "devicesByUDID") as? [NSUUID: NSObject], let device = byUDID[NSUUID(uuidString: udid)!] else { die("simulator \(udid) not found") } @@ -96,34 +104,65 @@ var tb = mach_timebase_info_data_t(); mach_timebase_info(&tb) func nowNs() -> UInt64 { mach_absolute_time() * UInt64(tb.numer) / UInt64(tb.denom) } var stamps: [UInt64] = [] var strip: [(UInt64, Int)] = [] // (time, counter or -1 when unreadable) -/// GlassBench's counter from a BGRA surface: 18 blocks of 20 pt from (21, 120) pt, at the surface's scale. -func readCounter(_ s: IOSurface) -> Int { +let env = ProcessInfo.processInfo.environment +let rawPath = env["SIMFPS_STAMPS"] +let probe: (Double, Double)? = env["SIMFPS_PROBE"].flatMap { s -> (Double, Double)? in + let p = s.split(separator: ",").compactMap { Double($0) } + return p.count == 2 ? (p[0], p[1]) : nil +} +var raw: [(UInt64, Int, Int)] = [] // (time, counter or -1, probe luma or -1) of every new surface +/// One look at a BGRA surface under ONE lock: GlassBench's counter (18 blocks of 20 pt from (21, 120) pt, at the +/// surface's scale; -1 when unreadable or not asked for) and the luma (0-255) at the probe point (-1 without one). +/// Locks are not free for the simulator: two per surface halved its frame rate (55 against 102 screen frames/s, one +/// 3x simulator); one cost between nothing and 31% over seven runs (why it varies is not known). +func readSurface(_ s: IOSurface, strip: Bool, probe: (Double, Double)?) -> (Int, Int) { + guard strip || probe != nil else { return (-1, -1) } let w = IOSurfaceGetWidth(s), h = IOSurfaceGetHeight(s), bpr = IOSurfaceGetBytesPerRow(s) let scale = Double(w) / 402.0 - guard IOSurfaceLock(s, .readOnly, nil) == kIOReturnSuccess else { return -1 } + guard IOSurfaceLock(s, .readOnly, nil) == kIOReturnSuccess else { return (-1, -1) } defer { IOSurfaceUnlock(s, .readOnly, nil) } let base = IOSurfaceGetBaseAddress(s).assumingMemoryBound(to: UInt8.self) - func luma(_ i: Int) -> Int { - let x = Int((21 + 20 * Double(i) + 10) * scale), y = Int(130 * scale) - guard x < w, y < h else { return 0 } + func lumaAt(_ px: Double, _ py: Double) -> Int? { + let x = Int(px * scale), y = Int(py * scale) + guard x < w, y < h else { return nil } let p = base + y * bpr + x * 4 return (Int(p[0]) + Int(p[1]) + Int(p[2])) / 3 } - let white = luma(0), black = luma(1) - guard white - black > 100 else { return -1 } - let mid = (white + black) / 2 - var n = 0 - for b in 0..<16 where luma(b + 2) > mid { n |= 1 << b } - return n + var counter = -1 + if strip { + func block(_ i: Int) -> Int { lumaAt(21 + 20 * Double(i) + 10, 130) ?? 0 } + let white = block(0), black = block(1) + if white - black > 100 { + let mid = (white + black) / 2 + var n = 0 + for b in 0..<16 where block(b + 2) > mid { n |= 1 << b } + counter = n + } + } + return (counter, probe.flatMap { lumaAt($0.0, $0.1) } ?? -1) } stamps.reserveCapacity(Int(secs * 200)) var polls = 0 +// SIMFPS_START_AT: attached by now; count from that wall-clock second on. +if let at = env["SIMFPS_START_AT"].flatMap({ Double($0) }) { + while Date().timeIntervalSince1970 < at { usleep(1000) } +} +let wallStart = Date().timeIntervalSince1970 let start = nowNs(), end = start + UInt64(secs * 1e9) var finished = false // Poll on a thread of its own; the main thread keeps its run loop going for CoreSimulator's replies. let poller = Thread { var lastSeed = IOSurfaceGetSeed(surf0) var lastSurf = surf0 + // The first line of SIMFPS_STAMPS is the screen as it stands at the start (a static screen makes no new surface). + // A picture read at the moment it appears can still show the frame before it. On a moving screen the next + // picture corrects that; on a still one nothing follows, and a flipped marker was never seen (a "missed" probe). + // So a reader that does not read the strip (a still screen) looks at the same surface again 30, 150 and 600 ms + // after it appeared; a marker that changed meanwhile is a line with counter -3 (a second look, not a new picture). + // Never with --strip: at 30 pictures a second the first second look would be a second lock of most pictures. + let lookAt: [UInt64] = [30_000_000, 150_000_000, 600_000_000] + var lastNew = start, looks = 0, lastLuma = -1 + if rawPath != nil { let r = readSurface(surf0, strip: readStrip, probe: probe); raw.append((start, r.0, r.1)); lastLuma = r.1 } while true { let t = nowNs() if t >= end { break } @@ -131,7 +170,14 @@ let poller = Thread { let seed = IOSurfaceGetSeed(s) if s !== lastSurf || seed != lastSeed { stamps.append(t); lastSeed = seed; lastSurf = s - if readStrip { strip.append((t, readCounter(s))) } + let (counter, luma) = readSurface(s, strip: readStrip, probe: rawPath != nil ? probe : nil) + if readStrip { strip.append((t, counter)) } + if rawPath != nil { raw.append((t, counter, luma)) } + lastNew = t; looks = 0; lastLuma = luma + } else if rawPath != nil, !readStrip, let p = probe, looks < lookAt.count, t - lastNew >= lookAt[looks] { + looks += 1 + let l = readSurface(lastSurf, strip: false, probe: p).1 + if l >= 0 && (l > 127) != (lastLuma > 127) { raw.append((t, -3, l)); lastLuma = l } } polls += 1 usleep(500) @@ -187,5 +233,9 @@ if readStrip { "gt20ms": uiv.filter { $0 > 20 }.count, "gt34ms": uiv.filter { $0 > 34 }.count, ] as [String: Any] } +if let path = rawPath { + let lines = raw.map { String(format: "%.6f %ld %ld", wallStart + Double($0.0 - start) / 1e9, $0.1, $0.2) }.joined(separator: "\n") + try? (lines + "\n").write(toFile: path, atomically: true, encoding: .utf8) +} let data = try! JSONSerialization.data(withJSONObject: result, options: [.sortedKeys]) print(String(data: data, encoding: .utf8)!) diff --git a/src/bench/tests/__init__.py b/src/bench/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/bench/tests/helpers.py b/src/bench/tests/helpers.py new file mode 100644 index 0000000..8d86c3c --- /dev/null +++ b/src/bench/tests/helpers.py @@ -0,0 +1,66 @@ +"""Fixtures for the tests: synthetic screenshots and a synthetic scale result. No VM, no network.""" +from macbench import png + + +def card_image(width=603, tear_at=None, with_cards=True, scrolled=0, bar=False): + """A screenshot like GlassBench's at scale s = width/1206: white background, cards 336*s apart whose right edge + strip (x 1125..1150 * s) is a vertical gradient (1 level per row at 3x), gaps 58*s tall. tear_at=y shifts every + row below y by 40*s rows (a band from another frame). scrolled=N moves the content up by N*s rows; bar=True + draws the navigation bar's glass over everything above 420*s (grey, as the real one tints what is under it).""" + s = width / 1206.0 + h = int(2622 * s) + pitch, gap = 336 * s, 58 * s + rows = [] + for y in range(h): + yy = y + int(scrolled * s) + if tear_at is not None and y >= tear_at: + yy += int(40 * s) + k = (yy - int(300 * s)) % pitch if yy >= 300 * s else None + row = bytearray([255] * (width * 3)) + if with_cards and k is not None and gap / 2 <= k < pitch - gap / 2 and 300 * s < yy < 2300 * s: + level = int(40 + (k - gap / 2) / (pitch - gap) * 150) + for x in range(width): + if int(1125 * s) - 10 <= x < int(1150 * s) + 10: + row[x * 3:x * 3 + 3] = bytes([level, level // 2, 200 - level // 2]) + else: + row[x * 3:x * 3 + 3] = bytes([60, 60, 200]) + if bar and y < 420 * s: + row = bytearray([200] * (width * 3)) + rows.append(bytes(row)) + return png.Image(width, h, 3, rows) + + +def strip_image(counter, width=402 * 3): + """An image with GlassBench's strip showing `counter`.""" + s = width / 402.0 + h = int(900 * s) + rows = [bytearray([128] * (width * 3)) for _ in range(h)] + + def block(i, white): + x0, x1 = int((21 + 20 * i) * s), int((21 + 20 * i + 20) * s) + for y in range(int(120 * s), int(140 * s)): + for x in range(x0, x1): + rows[y][x * 3:x * 3 + 3] = bytes([255, 255, 255] if white else [0, 0, 0]) + block(0, True) + block(1, False) + for b in range(16): + block(b + 2, bool((counter >> b) & 1)) + return png.Image(width, h, 3, [bytes(r) for r in rows]) + + +def fake_scale_result(ns=(1, 2, 3), fps=(60, 58, 40), mix="all-busy"): + steps = [] + for n, f in zip(ns, fps): + sims = [{"idx": i, "role": "busy", "fps": float(f), "p50_ms": 16.7, "p95_ms": 20.0 + (60 - f), "p99_ms": 30.0, + "max_ms": 50.0, "over2x": 2, "longest_gap_s": 0.05, "input_p95_ms": 100.0, "guest_cpu_pct": 30.0, + "guest_rss_mb": 800.0} for i in range(n)] + steps.append({"n": n, "sims": sims, "gates_ok": True, "stable": True, + "system": {"qemu_cpu_cores": 4.0 + n, "gpu_busy_pct": 30.0 * n, "gpu_ms_per_frame": 6.8, + "guest": {"mem_used_mb": 8000.0 + 900 * n, "compressed_mb": 100.0, "sims_cpu_pct": 30.0 * n, + "other_cpu_pct": 20.0, "sims_rss_mb": 800.0 * n}}, + "stability": {"ok": True}, "gates": {}}) + from macbench import analysis + summary = analysis.summarize_scale(steps, ["smooth60: fps>=55, p95<=25", "smooth30: fps>=30, p95<=50"]) + return {"schema": 1, "tool": "macbox-bench", "kind": "scale", "id": "20260101-000000", "created": "2026-01-01T00:00:00Z", + "params": {}, "env": {"build_id": "test"}, "sections": [], "gates": {}, + "data": {"mix": mix, "scale_x": 1, "ns": list(ns), "nominal_fps": 60, "steps": steps, "summary": summary}} diff --git a/src/bench/tests/test_dry.py b/src/bench/tests/test_dry.py new file mode 100644 index 0000000..3b2120b --- /dev/null +++ b/src/bench/tests/test_dry.py @@ -0,0 +1,339 @@ +"""DRY=1 runs of the whole suite against the stub guest and a fake host: control flow, faults, safety, outputs. +Each case is a real `run` process (its own signal handlers, its own exit trap). They run in parallel.""" +import concurrent.futures +import json +import os +import signal +import subprocess +import sys +import tempfile +import time +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +RUN = os.path.join(HERE, "..", "run") +FAST = ["--secs", "3", "--probe-every", "0.7", "--probe-timeout", "1", "--settle-max", "2", "--no-observer-check"] + + +def run_case(args, faults="", extra_env=None, kill_after=None, keep=True, timeout=240, state=None): + """One real `run` process in DRY mode. `state` = a directory shared by several cases in a row (the stub guest, its + device set, the seed): the run after another one sees what the first left.""" + d = tempfile.mkdtemp(prefix="dry-case-") + env = dict(os.environ, DRY="1", BENCH_STUB_FAULT=faults, BENCH_DRY_KEEP=state or os.path.join(d, "state") if keep else "", + BENCH_DEADLINE_MARGIN="3", **(extra_env or {})) + if not keep: + env.pop("BENCH_DRY_KEEP") + out_arg = [] if args[0] == "cleanup" else ["--out", os.path.join(d, "out")] + p = subprocess.Popen([sys.executable, RUN] + args + out_arg, env=env, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, text=True) + if kill_after: + time.sleep(kill_after) + p.send_signal(signal.SIGTERM) + out, _ = p.communicate(timeout=timeout) + res = None + od = os.path.join(d, "out") + if os.path.isdir(od): + for f in sorted(os.listdir(od)): + if f.endswith(".json"): + with open(os.path.join(od, f)) as fh: + res = json.load(fh) + return {"rc": p.returncode, "out": out, "doc": res, "state": state or os.path.join(d, "state")} + + +def sims_booted(state): + try: + with open(os.path.join(state, "guest", "state.json")) as f: + return [u for u, v in json.load(f)["devs"].items() if v["state"] == "Booted"] + except OSError: + return None + + +def simctl_calls(state): + out = [] + with open(os.path.join(state, "guest", "calls.log")) as f: + for line in f: + out.append(json.loads(line)) + return [c for c in out if c[:2] == ["xcrun", "simctl"]] + + +def sections(doc, name): + return [s for s in doc["sections"] if s["name"] == name] + + +CASES = { + "clean": (["scale", "--n", "1,2,3"] + FAST, ""), + "memfloor": (["scale", "--n", "1,2,3,4,5"] + FAST, "memfloor@3"), + "refuse": (["scale", "--n", "1,2"] + FAST, "memfloor@0"), + "hostmem": (["scale", "--n", "1,2,3,4,5"] + FAST, "hostmem@3"), + "crash": (["scale", "--n", "1,2,3,4"] + FAST, "crash@2"), + "daemoncrash": (["scale", "--n", "1,2"] + FAST, "daemoncrash@2"), + "observer": (["scale", "--n", "1"] + [x for x in FAST if x != "--no-observer-check"], ""), + "backwards": (["scale", "--n", "1,2,3"] + FAST, "backwards@2"), + "kernelgpu": (["scale", "--n", "1,2,3"] + FAST, "kernelgpu@2"), + "coherent": (["scale", "--n", "1,2,3"] + FAST, "coherent@2"), + "slow": (["scale", "--n", "1,2,3,4"] + FAST, "slow@3"), + "hang": (["scale", "--n", "1,2,3"] + FAST, "hang_simfps"), + "nomarker": (["scale", "--n", "1,2"] + FAST, "nomarker"), + "nonotify": (["scale", "--n", "1"] + FAST, "nonotify"), + "tear": (["scale", "--n", "1,2,3"] + FAST, "tear@2"), + "onebusy": (["scale", "--n", "1,2,3", "--mix", "one-busy"] + FAST, ""), + "agent": (["scale", "--n", "1,2", "--mix", "agent", "--agent-period", "2"] + FAST, ""), + "idlemix": (["scale", "--n", "1,2", "--mix", "idle"] + FAST, ""), + "idle": (["idle", "--secs", "3", "--settle-max", "2"], ""), + "lifecycle": (["lifecycle", "--cycles", "2", "--storm", "2", "--secs", "3"], ""), + "build": (["build"], ""), + "panel_skip": (["panel"], ""), +} + + +class Dry(unittest.TestCase): + @classmethod + def setUpClass(cls): + with concurrent.futures.ThreadPoolExecutor(8) as ex: + futs = {k: ex.submit(run_case, a, f) for k, (a, f) in CASES.items()} + futs["ssh"] = ex.submit(run_case, ["scale", "--n", "1,2,3"] + FAST, "", {"BENCH_STUB_SSH_TIMEOUT": "30"}) + futs["term"] = ex.submit(run_case, ["scale", "--n", "1,2,3,4,5,6"] + FAST, "", None, 35) + futs["deadline"] = ex.submit(run_case, ["scale", "--n", "1,2,3,4,5,6", "--deadline", "40"] + FAST, "") + cls.r = {k: f.result() for k, f in futs.items()} + + def test_clean_sweep(self): + r = self.r["clean"] + self.assertEqual(r["rc"], 0, r["out"][-800:]) + d = r["doc"] + self.assertEqual([s["n"] for s in d["data"]["steps"]], [1, 2, 3]) + self.assertEqual(d["data"]["summary"]["max_n"]["smooth60"]["max_n"], 2) + self.assertTrue(d["publishable"], d.get("problems")) + self.assertEqual(sims_booted(r["state"]), []) + self.assertEqual(sections(d, "cleanup")[0]["status"], "PASS") + for s in d["data"]["steps"]: + self.assertEqual(len(s["sims"]), s["n"]) + for sim in s["sims"]: + self.assertIn("input_p95_ms", sim) + self.assertIn("guest_rss_mb", sim) + + def test_every_simctl_call_names_the_bench_set(self): + """Every simctl call names the bench's own set; the one exception is the clone OUT of Simlock's source set (a read of + the source, written into the bench's set), and nothing else may name Simlock's set.""" + for k in ("clean", "onebusy", "lifecycle", "agent"): + state = self.r[k]["state"] + bench, src = os.path.join(state, "guest", "set"), os.path.join(state, "guest", "simlock-sources") + calls = simctl_calls(state) + self.assertTrue(calls, k) + clones_from_source = 0 + for c in calls: + if c[2:4] == ["list", "devicetypes"] or c[2:4] == ["list", "runtimes"] or c[2:5] == ["list", "devices", "booted"]: + continue # global, read only + self.assertEqual(c[2], "--set", (k, c)) + if c[3] == src: + self.assertEqual((c[4], len(c), c[-1]), ("clone", 8, bench), (k, c)) # clone , nothing else + clones_from_source += 1 + else: + self.assertEqual(c[3], bench, (k, c)) + self.assertEqual(clones_from_source, 1, k) # ONE seed for the device type, whatever N is + with open(os.path.join(state, "guest", "state.json")) as f: + self.assertEqual(json.load(f).get("source_violations", []), [], k) + + def test_memory_floor_stops_the_sweep_and_keeps_the_data(self): + r = self.r["memfloor"] + d = r["doc"] + self.assertEqual([s["n"] for s in d["data"]["steps"]], [1, 2, 3]) # the guard looks before booting: it refuses the 4th + self.assertIn("guest available memory", d["data"]["summary"]["stopped"]) + self.assertEqual(sims_booted(r["state"]), []) + self.assertEqual(self.r["hostmem"]["doc"]["data"]["steps"][-1]["n"], 3) + self.assertIn("host free memory", self.r["hostmem"]["doc"]["data"]["summary"]["stopped"]) + + def test_a_sweep_that_ran_no_step_fails(self): + r = self.r["refuse"] # the guard refuses before the seed is even made: nothing ran + self.assertEqual(r["doc"]["data"]["steps"], []) + self.assertNotEqual(r["rc"], 0) + st = sections(r["doc"], "stopped")[0] + self.assertEqual(st["status"], "FAIL") + self.assertIn("guest available memory", st["line"]) + self.assertEqual(sims_booted(r["state"]), []) + + def test_steps_record_whether_the_guest_settled(self): + for st in self.r["clean"]["doc"]["data"]["steps"]: + self.assertIn("settled", st["settle"]) + self.assertIn("waited_s", st["settle"]) + + def test_run_wide_gates_catch_what_a_step_gate_does_not_name(self): + d = self.r["crash"]["doc"] + self.assertEqual(d["gates"]["run_crash_reports"]["status"], "FAIL") + self.assertEqual(sections(d, "gate:run_crash_reports")[0]["status"], "FAIL") + for k in ("clean", "lifecycle", "idle"): + g = self.r[k]["doc"]["gates"] + self.assertEqual(g["run_crash_reports"]["status"], "PASS", k) + self.assertIn("run_kernel_gpu", g) + + def test_observer_check_window_reads_nothing_from_the_screen(self): + r = self.r["observer"] + self.assertIsNotNone(r["doc"]["data"].get("observer"), r["out"][-600:]) + with open(os.path.join(r["state"], "guest", "calls.log")) as f: + fps = [c[2:] for c in map(json.loads, f) if c[0] == "simfps"] + self.assertIn(["strip", "probe"], fps) # the measured window: strip and marker, one look + self.assertEqual(fps[-1], ["nostrip", "noprobe"]) # the observer check's window: an unviewed screen + + def test_another_daemons_crash_report_is_listed_not_judged(self): + r = self.r["daemoncrash"] + d = r["doc"] + self.assertEqual(r["rc"], 0, r["out"][-600:]) + self.assertEqual([st["n"] for st in d["data"]["steps"]], [1, 2]) + self.assertTrue(all(st["stable"] for st in d["data"]["steps"])) + self.assertEqual(d["data"]["steps"][-1]["stability"]["other_ips"], ["searchd-2026-01-02-030405.ips"]) + g = d["gates"]["run_crash_reports"] + self.assertEqual(g["status"], "PASS") + self.assertIn("not judged: 1 other report(s): searchd-", g["detail"]) + + def test_crash_report_is_instability_and_ends_the_sweep(self): + d = self.r["crash"]["doc"] + self.assertEqual(d["data"]["steps"][-1]["n"], 2) + self.assertFalse(d["data"]["steps"][-1]["stable"]) + self.assertIn("instability", d["data"]["summary"]["stopped"]) + self.assertEqual(sections(d, "stability")[0]["status"], "FAIL") + self.assertNotEqual(self.r["crash"]["rc"], 0) + + def test_strip_out_of_order_fails_the_run_but_keeps_smaller_n(self): + r = self.r["backwards"] + d = r["doc"] + self.assertEqual(d["gates"]["strip"]["status"], "FAIL") + self.assertEqual(d["data"]["steps"][0]["gates"]["strip"]["status"], "PASS") # N=1 intact + self.assertEqual(d["data"]["steps"][-1]["n"], 2) + self.assertEqual(r["rc"], 1) + + def test_kernel_and_coherent_gates(self): + k = self.r["kernelgpu"]["doc"] + self.assertEqual(k["gates"]["kernel_gpu"]["status"], "FAIL") + self.assertFalse(k["data"]["steps"][-1]["stable"]) + c = self.r["coherent"]["doc"] + self.assertEqual(c["gates"]["coherent_census"]["status"], "FAIL") + self.assertEqual(c["data"]["steps"][-1]["n"], 2) + + def test_floor_breach_stops_early(self): + d = self.r["slow"]["doc"] + self.assertEqual(d["data"]["steps"][-1]["n"], 3) + self.assertIn("floor", d["data"]["summary"]["stopped"]) + self.assertEqual(d["data"]["summary"]["max_n"]["smooth60"]["max_n"], 2) + + def test_hung_observer_is_reported_not_swallowed(self): + d = self.r["hang"]["doc"] + self.assertTrue(d["data"]["steps"][0]["stability"]["observer_errors"]) + self.assertEqual(sims_booted(self.r["hang"]["state"]), []) + + def test_a_torn_screenshot_fails_the_run_but_the_sweep_goes_on(self): + r = self.r["tear"] + d = r["doc"] + self.assertEqual([st["n"] for st in d["data"]["steps"]], [1, 2, 3]) # N=3 is still measured + self.assertEqual([st["gates"]["tears"]["status"] for st in d["data"]["steps"]], ["PASS", "FAIL", "PASS"]) + self.assertNotEqual(r["rc"], 0) + self.assertIsNone(d["data"]["summary"]["stopped"]) + for name, m in d["data"]["summary"]["max_n"].items(): # the torn step is not sustained + self.assertNotIn(m["max_n"], (2, 3), name) + self.assertEqual(sims_booted(r["state"]), []) + + def test_auto_never_sends_openurl(self): + # iOS 26 answers `simctl openurl` with an alert that stays over the app and slows it for the whole run + r = self.r["clean"] + self.assertEqual(list(r["doc"]["data"]["calibration"]["methods"]), ["notify"]) # answered: appearance is not tried + self.assertEqual(r["doc"]["data"]["calibration"]["chosen"], "notify") + self.assertFalse([c for c in simctl_calls(r["state"]) if "openurl" in c]) + + def test_appearance_is_the_fallback_when_the_notification_is_not_answered(self): + cal = self.r["nonotify"]["doc"]["data"]["calibration"] + self.assertEqual(cal["chosen"], "appearance") + self.assertEqual(cal["methods"]["notify"]["answered"], 0) + + def test_unanswered_input_probe_is_not_dropped(self): + d = self.r["nomarker"]["doc"] + cal = d["data"]["calibration"] + self.assertEqual(cal["chosen"], "none") + self.assertEqual(d["data"]["steps"][0]["targets"]["responsive"], None) # unknown, never a pass + + def test_one_busy_mix(self): + d = self.r["onebusy"]["doc"] + st = d["data"]["steps"][-1] + self.assertEqual([s["role"] for s in st["sims"]], ["busy", "static", "static"]) + self.assertIn("input_p95_ms", st["sims"][1]) + self.assertNotIn("fps", st["sims"][1]) + self.assertIsNone(st["sims"][1].get("longest_gap_s")) + + def test_agent_mix(self): + st = self.r["agent"]["doc"]["data"]["steps"][-1] + self.assertEqual([s["role"] for s in st["sims"]], ["agent", "agent"]) + self.assertTrue(any(s.get("actions") for s in st["sims"]), st["sims"]) + + def test_idle_mix_has_no_frames_or_probes(self): + st = self.r["idlemix"]["doc"]["data"]["steps"][-1] + self.assertEqual({s["role"] for s in st["sims"]}, {"idle"}) + self.assertNotIn("input_p95_ms", st["sims"][0]) + + def test_idle_lifecycle_build_panel(self): + self.assertEqual(self.r["idle"]["rc"], 0, self.r["idle"]["out"][-500:]) + self.assertIn("IDLE", self.r["idle"]["doc"]["sections"][0]["line"]) + lc = self.r["lifecycle"]["doc"] + self.assertTrue(sections(lc, "lifecycle")) + self.assertTrue(sections(lc, "storm")) + self.assertIn("leak_qemu_mem", lc["gates"]) + self.assertEqual(sims_booted(self.r["lifecycle"]["state"]), []) + self.assertEqual(sections(self.r["build"]["doc"], "xcodebuild")[0]["status"], "SKIP") + self.assertEqual(sections(self.r["panel_skip"]["doc"], "panel")[0]["status"], "SKIP") + + def test_ssh_timeout_mid_run_keeps_results_and_cleans_up(self): + r = self.r["ssh"] + d = r["doc"] + self.assertTrue(d["data"]["steps"], r["out"][-600:]) + self.assertIn("ssh timed out", d["data"]["summary"]["stopped"]) + self.assertEqual(sims_booted(r["state"]), []) + + def test_sigterm_runs_the_one_trap(self): + r = self.r["term"] + self.assertEqual(r["rc"], 143, r["out"][-500:]) + self.assertEqual(sims_booted(r["state"]), []) + self.assertIn("cleanup: bench simulators", r["out"]) + self.assertTrue(r["doc"] is not None and r["doc"]["data"]["steps"]) + + def test_deadline_refuses_a_window_that_does_not_fit(self): + r = self.r["deadline"] + self.assertNotEqual(r["rc"], 0, r["out"][-500:]) # a window that does not fit before the deadline is refused + self.assertEqual(sims_booted(r["state"]), []) + self.assertIn("no time for", r["doc"]["data"]["summary"]["stopped"]) + self.assertEqual(r["doc"]["data"]["stopped_kind"], "error") + + +class KnobsAndAB(unittest.TestCase): + def ab(self, extra, faults="", kill_after=None): + d = tempfile.mkdtemp(prefix="dry-ab-") + live = os.path.join(d, "live.conf") + return d, live + + def test_ab_adopt_close_and_knob_restored(self): + # DRY's live switch file is created by the harness; the knob must come back byte for byte + r = run_case(["ab", "--knob", "live:group_x=1", "--n", "2", "--arm-settle", "0.2", "--secs", "3", "--settle-max", "2", "--threshold", "0.04"], "") + d = r["doc"] + self.assertEqual(r["rc"], 0, r["out"][-800:]) + self.assertIn(d["data"]["decision"]["decision"], ("ADOPT", "CLOSE")) + self.assertEqual([a["arm"] for a in d["data"]["arms"]], ["A", "B", "B", "A"]) + self.assertIn("AB live:group_x=1:", d["data"]["line"]) + self.assertIn("bar 4.0%", d["data"]["line"]) + self.assertIn("rule", d["params"]) + self.assertIn("restored: live switches file", r["out"]) + live = os.path.join(r["state"], "live.conf") + with open(live) as f: + self.assertEqual(f.read(), "surface_coherent=1\n") + self.assertEqual(sims_booted(r["state"]), []) + + def test_ab_sigterm_mid_arm_restores_the_knob(self): + r = run_case(["ab", "--knob", "live:group_x=1", "--n", "2", "--arm-settle", "0.2", "--secs", "6", "--settle-max", "2"], "", None, kill_after=18) + self.assertEqual(r["rc"], 143, r["out"][-600:]) + live = os.path.join(r["state"], "live.conf") + with open(live) as f: + self.assertEqual(f.read(), "surface_coherent=1\n") + self.assertEqual(sims_booted(r["state"]), []) + + def test_ab_needs_a_knob(self): + r = run_case(["ab", "--n", "1", "--secs", "3"], "") + self.assertNotEqual(r["rc"], 0) + self.assertIn("a knob is needed", r["out"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/bench/tests/test_dry_closed.py b/src/bench/tests/test_dry_closed.py new file mode 100644 index 0000000..c947eb9 --- /dev/null +++ b/src/bench/tests/test_dry_closed.py @@ -0,0 +1,138 @@ +"""Fail closed: a settle that reaches its cap, a booted count that does not match, a frame counter that answers nothing, a +foreign simulator, and result files that are never found by pattern. Real `run` processes against the stub guest.""" +import concurrent.futures +import os +import re +import unittest + +from macbench import core +from tests.test_dry import FAST, run_case, sections, sims_booted + +HERE = os.path.dirname(os.path.abspath(__file__)) +SWEEP = ["scale", "--n", "1,2,3"] + FAST +_R = {} + + +def results(): + if not _R: + cases = {"clean": (SWEEP, ""), "unsettled": (SWEEP, "unsettled@2"), "lost": (SWEEP, "lostsim@2"), "counter": (SWEEP, "counterfail@2"), + "foreign": (SWEEP, "foreign@2"), + "ab_lost": (["ab", "--knob", "live:group_x=1", "--n", "2", "--arm-settle", "0.2", "--secs", "3", "--settle-max", "2"], "lostsim@2"), + "soak_counter": (["soak", "--soak-n", "2", "--soak-minutes", "0.2", "--soak-every", "5", "--secs", "3", "--settle-max", "2"], "counterfail@2")} + with concurrent.futures.ThreadPoolExecutor(8) as ex: + futs = {k: ex.submit(run_case, a, f) for k, (a, f) in cases.items()} + _R.update({k: f.result() for k, f in futs.items()}) + return _R + + +class SettleCap(unittest.TestCase): + def test_a_settle_that_reaches_its_cap_fails_the_run_and_names_the_busy_processes(self): + r = results()["unsettled"] + self.assertNotEqual(r["rc"], 0, r["out"][-600:]) + s = sections(r["doc"], "settle")[0] + self.assertEqual(s["status"], "FAIL") + self.assertIn("NOT SETTLED at N=2", s["line"]) + self.assertIn("diagnosticd", s["line"]) # the top processes by CPU at that moment + self.assertIn("diagnosticd", [t["cmd"] for t in s["detail"]["top"]]) + self.assertEqual([st["n"] for st in r["doc"]["data"]["steps"]], [1]) # nothing was measured on top of it + self.assertEqual(sections(r["doc"], "scale")[0]["status"], "FAIL") + self.assertEqual(sims_booted(r["state"]), []) + + def test_a_clean_sweep_settles_every_step_and_says_how_long(self): + r = results()["clean"] + self.assertEqual(r["rc"], 0, r["out"][-600:]) + for st in r["doc"]["data"]["steps"]: + self.assertTrue(st["settle"]["settled"]) + self.assertIsNotNone(st["settle"]["waited_s"]) + + +class BootedCount(unittest.TestCase): + def test_the_count_is_the_guests_not_the_expected_number(self): + for st in results()["clean"]["doc"]["data"]["steps"]: + self.assertEqual(st["booted"], st["n"]) + self.assertEqual(st["expected_booted"], st["n"]) + + def test_a_mismatch_before_the_window_fails_the_run(self): + r = results()["lost"] + self.assertNotEqual(r["rc"], 0, r["out"][-600:]) + s = sections(r["doc"], "booted_check")[0] + self.assertEqual(s["status"], "FAIL") + self.assertIn("1 of the bench's simulators are booted, expected 2", s["line"]) + self.assertEqual(s["detail"], {"booted": 1, "expected": 2}) # the measured number is recorded + self.assertEqual([st["n"] for st in r["doc"]["data"]["steps"]], [1]) + self.assertEqual(sims_booted(r["state"]), []) + + def test_ab_arms_and_every_other_window_are_counted_too(self): + r = results()["ab_lost"] + self.assertNotEqual(r["rc"], 0, r["out"][-600:]) + self.assertEqual(sections(r["doc"], "booted_check")[0]["status"], "FAIL") + self.assertEqual(r["doc"]["data"].get("arms", []), []) + live = os.path.join(r["state"], "live.conf") + with open(live) as f: + self.assertEqual(f.read(), "surface_coherent=1\n") + + def test_a_simulator_outside_the_bench_makes_the_run_yield(self): + r = results()["foreign"] + self.assertNotEqual(r["rc"], 0, r["out"][-600:]) + said = " | ".join(x["line"] for x in r["doc"]["sections"] if x["status"] == "FAIL") # the precheck, the guard or the monitor: whoever sees it first + self.assertRegex(said, r"YIELDING|someone needs the Mac|outside the bench are booted") + self.assertEqual([st["n"] for st in r["doc"]["data"]["steps"]], [1]) + self.assertEqual(sims_booted(r["state"]), []) + + +class FrameCounter(unittest.TestCase): + def test_a_counter_that_answers_nothing_fails_the_window(self): + r = results()["counter"] + self.assertNotEqual(r["rc"], 0, r["out"][-600:]) + st = r["doc"]["data"]["steps"][-1] + self.assertEqual(st["n"], 2) + self.assertEqual(st["gates"]["counter"]["status"], "FAIL") + self.assertRegex(st["gates"]["counter"]["detail"], r"sim [01]: simfps: no surface") + self.assertEqual(r["doc"]["gates"]["counter"]["status"], "FAIL") + self.assertIn("gate failed at N=2: counter", r["doc"]["data"]["summary"]["stopped"]) + self.assertEqual(sims_booted(r["state"]), []) + + def test_a_soak_window_is_judged_too(self): + r = results()["soak_counter"] + self.assertEqual(r["doc"]["gates"]["counter"]["status"], "FAIL") + self.assertEqual(sections(r["doc"], "gate:counter")[0]["status"], "FAIL") + self.assertNotEqual(r["rc"], 0, r["out"][-600:]) + + def test_gate_unit(self): + ok = {"role": "busy", "simfps": {"fps": 60}, "raw": "1.0 5 255\n1.1 6 255\n"} + self.assertEqual(core.counter_gate([ok, dict(ok, role="static", raw="1.0 -1 255\n")], "all-busy")["status"], "PASS") + for bad in (dict(ok, simfps=None), dict(ok, simfps_error="hung"), dict(ok, raw="1.0 -1 255\n1.1 -1 255\n"), dict(ok, raw="")): + self.assertEqual(core.counter_gate([ok, bad], "all-busy")["status"], "FAIL", bad) + self.assertEqual(core.counter_gate([], "idle")["status"], "SKIP") + + +class ResultFilesAreNeverFoundByPattern(unittest.TestCase): + """The base2 sweep read `f-busyK-*` and also got `f-busyK-unread-*`. Here every per-window file is opened by the exact + path its owner built (a per-call temp directory; the window's own dict), nothing looks files up by prefix or glob.""" + + def sources(self): + for d in ("../macbench", "../guest"): + for f in sorted(os.listdir(os.path.join(HERE, d))): + if f.endswith(".py"): + with open(os.path.join(HERE, d, f)) as fh: + yield f, fh.read() + + def test_no_module_selects_files_by_pattern(self): + for name, text in self.sources(): + self.assertNotRegex(text, r"\bimport glob\b|\bfrom glob\b|\bfnmatch\b|\.glob\(|\brglob\b", name) + # the only directory listings: crash reports (exact suffix), Simlock's source set (twice: a UUID pattern, device.plist read by exact + # path), the simslim config folder (one profile), /proc and /sys in hostprobe. A new listing has to be argued for here. + allowed = {"benchguest.py": 4, "hostprobe.py": 2} + for name, text in self.sources(): + n = len(re.findall(r"os\.listdir\(|os\.scandir\(", text)) + self.assertLessEqual(n, allowed.get(name, 0), name) + + def test_two_windows_of_one_step_keep_their_own_files(self): + with open(os.path.join(HERE, "..", "guest", "benchguest.py")) as fh: + src = fh.read() + self.assertEqual(len(re.findall(r'tempfile\.mkdtemp\(prefix="benchwin"\)', src)), 1) # one fresh directory per window + self.assertIn('raws[u] = os.path.join(tmp, "raw-%d.txt" % i)', src) # exact name per simulator, kept in a dict + + +if __name__ == "__main__": + unittest.main() diff --git a/src/bench/tests/test_dry_guards.py b/src/bench/tests/test_dry_guards.py new file mode 100644 index 0000000..e10b4b7 --- /dev/null +++ b/src/bench/tests/test_dry_guards.py @@ -0,0 +1,97 @@ +"""The guards end to end in DRY: the host-disk floors (start 18, boot another 14, abort and clean up under 11), a disk that +cannot be read, a restarted VM, and the one cleanup after an abort from the monitor thread.""" +import concurrent.futures +import json +import os +import unittest + +from tests.test_dry import FAST, run_case, sections, sims_booted +from tests.test_dry_state import calls, files + +SWEEP = ["scale", "--n", "1,2,3"] + FAST +_R = {} + + +def results(): + if not _R: + cases = {"diskstart": (SWEEP, "diskstart"), "disklow": (SWEEP, "disklow@2"), "diskabort": (SWEEP, "diskabort@2"), + "diskunreadable": (SWEEP, "diskunreadable"), "qemugone": (SWEEP, "qemugone@2"), + "transition": (SWEEP, "transition@1"), "stockdisk": (["scale", "--n", "2", "--stock-devices"] + FAST[:], "disklow@2"), "clean": (SWEEP, "")} + with concurrent.futures.ThreadPoolExecutor(8) as ex: + futs = {k: ex.submit(run_case, a, f) for k, (a, f) in cases.items()} + _R.update({k: f.result() for k, f in futs.items()}) + return _R + + +class HostDisk(unittest.TestCase): + def test_a_run_does_not_start_under_18_gib(self): + r = results()["diskstart"] + self.assertNotEqual(r["rc"], 0) + self.assertIn("16.0 GiB is under the 18 GiB floor", sections(r["doc"], "stopped")[0]["line"]) + self.assertEqual([c for c in calls(r["state"]) if "clone" in c or "create" in c], []) # not even a seed + + def test_another_simulator_is_not_booted_under_14_gib(self): + r = results()["disklow"] + self.assertEqual([s["n"] for s in r["doc"]["data"]["steps"]], [1, 2]) # with 2 booted the disk reads 12.5: the third is refused + self.assertIn("12.5 GiB is under the 14 GiB floor to boot another simulator", r["doc"]["data"]["summary"]["stopped"]) + self.assertEqual(sims_booted(r["state"]), []) + + def test_with_stock_devices_the_guard_stands_before_each_device(self): + r = results()["stockdisk"] + creates = [c for c in calls(r["state"]) if "create" in c] + self.assertEqual(len(creates), 1) # the second device was refused before it was made + self.assertIn("under the 14 GiB floor", " ".join(s["line"] for s in r["doc"]["sections"] if s["name"] in ("stopped", "scale"))) + self.assertEqual(sims_booted(r["state"]), []) + self.assertEqual(files(r["state"], "sim-keep"), []) + + def test_a_disk_that_cannot_be_read_refuses(self): + r = results()["diskunreadable"] + self.assertNotEqual(r["rc"], 0) + self.assertIn("cannot be read", r["out"]) + + def test_under_11_gib_while_waiting_the_run_is_aborted_and_cleaned_up(self): + r = results()["diskabort"] + self.assertEqual(r["rc"], 8, r["out"][-800:]) # the monitor thread owns the exit code + ab = sections(r["doc"], "abort")[0] + self.assertEqual(ab["status"], "FAIL") + self.assertIn("9.0 GiB is under the 11 GiB floor", ab["line"]) + self.assertEqual(sections(r["doc"], "cleanup")[0]["status"], "PASS") # verified, after the abort + self.assertEqual(sims_booted(r["state"]), []) + self.assertEqual(files(r["state"], "sim-keep"), []) + self.assertEqual([s["n"] for s in r["doc"]["data"]["steps"]], [1]) # what was measured is kept + + def test_a_restarted_vm_aborts_the_run(self): + r = results()["qemugone"] + self.assertEqual(r["rc"], 9, r["out"][-800:]) + self.assertIn("QEMU", sections(r["doc"], "abort")[0]["line"]) + self.assertEqual(sims_booted(r["state"]), []) + + def test_a_bench_device_in_transition_is_not_a_foreign_simulator(self): + r = results()["transition"] # a launchd_sim that names a bench-set device in any state is the bench's own + self.assertEqual(r["rc"], 0, r["out"][-800:]) + self.assertEqual(sections(r["doc"], "abort") + sections(r["doc"], "foreign"), []) + self.assertEqual([s["n"] for s in r["doc"]["data"]["steps"]], [1, 2, 3]) + for s in r["doc"]["data"]["steps"]: + self.assertEqual(s["stability"]["foreign_booted"], 0) + + def test_an_abort_is_not_a_race_with_the_main_thread(self): + """The monitor thread writes the result while the main thread, whose calls it killed, unwinds: the result must always be + whole, the cleanup verified and the exit code the monitor's, every time.""" + cases = [(SWEEP, "diskabort@2", 8), (SWEEP, "qemugone@2", 9)] * 6 + with concurrent.futures.ThreadPoolExecutor(8) as ex: + rs = list(ex.map(lambda c: (run_case(c[0], c[1]), c[2]), cases)) + for r, code in rs: + self.assertEqual(r["rc"], code, r["out"][-600:]) + self.assertIsNotNone(r["doc"], r["out"][-600:]) + self.assertEqual(sections(r["doc"], "cleanup")[0]["status"], "PASS") + self.assertEqual([s["name"] for s in r["doc"]["sections"]].count("abort"), 1) + self.assertEqual([s["name"] for s in r["doc"]["sections"] if s["name"] in ("stopped", "error")], []) + + def test_a_clean_run_is_not_aborted(self): + r = results()["clean"] + self.assertEqual(r["rc"], 0, r["out"][-500:]) + self.assertEqual(sections(r["doc"], "abort"), []) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/bench/tests/test_dry_seed.py b/src/bench/tests/test_dry_seed.py new file mode 100644 index 0000000..139f5b0 --- /dev/null +++ b/src/bench/tests/test_dry_seed.py @@ -0,0 +1,266 @@ +"""The production seed, in DRY: made from a Simlock source device once, reused, never touching Simlock's set; every bench +simulator a clone of it with the seed's Slim disables; the explicit stock path; every way a seed can fail.""" +import json +import os +import shutil +import tempfile +import unittest + +from guest import benchguest_stub as stub +from tests.test_dry import FAST, run_case, sections +from tests.test_dry_state import calls, files + +ONE = ["scale", "--n", "1,2"] + FAST +_CACHE = {} + + +def parallel(jobs): + """Runs {name: fn} in threads once and caches the results: every DRY run is its own process, they do not share anything + but their own state directory.""" + import concurrent.futures + todo = {k: f for k, f in jobs.items() if k not in _CACHE} + with concurrent.futures.ThreadPoolExecutor(8) as ex: + futs = {k: ex.submit(f) for k, f in todo.items()} + for k, f in futs.items(): + _CACHE[k] = f.result() + return {k: _CACHE[k] for k in jobs} + + +def verbs(state, word): + return [c for c in calls(state) if word in c] + + +def devs(state): + with open(os.path.join(state, "guest", "state.json")) as f: + return json.load(f)["devs"] + + +def manifest(state): + with open(os.path.join(state, "guest", "set", ".bench-manifest.json")) as f: + return json.load(f) + + +def new_state(): + return tempfile.mkdtemp(prefix="dry-seed-") + + +def job_seed2(): + st = new_state() + r1 = run_case(ONE, "", state=st) + n1 = len([c for c in calls(st) if c[0].endswith("simslim")]) + return {"state": st, "r1": r1, "n_slim1": n1, "r2": run_case(ONE, "", state=st)} + + +def job_fail(fault, args=None): + def f(): + st = new_state() + return {"state": st, "r": run_case(args or ONE, fault, state=st), "next": None} + return f + + +def job_fail_then_ok(fault, args=None): + def f(): + st = new_state() + r = run_case(args or ONE, fault, state=st) + snap = {"devs": devs(st), "keep": files(st, "sim-keep")} + return {"state": st, "r": r, "snap": snap, "next": run_case(ONE, "", state=st)} + return f + + +def job_stale(): + st = new_state() + out = {"state": st, "first": run_case(ONE, "", state=st)} + p = os.path.join(st, "guest", "set", ".bench-manifest.json") + m = manifest(st) + for t in m["seeds"].values(): + t["app_stamp"] = "stale" + with open(p, "w") as f: + json.dump(m, f) + out["stale"] = run_case(ONE, "", state=st) + out["rebuilt"] = run_case(ONE + ["--rebuild-seed"], "", state=st) + return out + + +def job_seedcmd(): + st = new_state() + out = {"state": st, "seed": run_case(["seed", "--secs", "3"], "", state=st)} + out["ndevs"] = len(devs(st)) + out["cleanup"] = run_case(["cleanup", "--seeds"], "", state=st) + return out + + +JOBS = { + "seed2": job_seed2, + "nosource": job_fail_then_ok("nosource"), + "nosourceset": job_fail("nosourceset"), + "slimfail": job_fail_then_ok("slimfail"), + "slimfew": job_fail_then_ok("slimfew"), + "noicons": lambda: (lambda st: {"state": st, "r": run_case(ONE, "noicons", state=st)})(new_state()), + "noinstall": job_fail_then_ok("noinstall"), + "stale": job_stale, + "seedcmd": job_seedcmd, + "stock": lambda: (lambda st: {"state": st, "r": run_case(ONE + ["--stock-devices"], "nosourceset", state=st)})(new_state()), + "lifecycle": lambda: (lambda st: {"state": st, "r": run_case(["lifecycle", "--cycles", "2", "--storm", "2", "--secs", "3"], "", state=st)})(new_state()), +} + + +class Seed(unittest.TestCase): + @classmethod + def setUpClass(cls): + d = parallel(JOBS)["seed2"] + cls.state, cls.r1, cls.r2, cls.n_slim1 = d["state"], d["r1"], d["r2"], d["n_slim1"] + + def test_the_seed_is_made_once_from_the_simlock_source_then_reused(self): + self.assertEqual((self.r1["rc"], self.r2["rc"]), (0, 0), self.r1["out"][-500:] + self.r2["out"][-500:]) + self.assertEqual(self.n_slim1, 1) + self.assertEqual(len([c for c in calls(self.state) if c[0].endswith("simslim")]), 1) # the second run did not make another + src = os.path.join(self.state, "guest", "simlock-sources") + from_source = [c for c in calls(self.state) if c[:4] == ["xcrun", "simctl", "--set", src]] + self.assertEqual(len(from_source), 1) + self.assertEqual(from_source[0][4:6], ["clone", "5A5A5A5A-0000-4000-8000-000000000003"]) # the 1x type's source + self.assertIn("made from a Simlock source device", sections(self.r1["doc"], "seed")[0]["line"]) + self.assertIn("reused", sections(self.r2["doc"], "seed")[0]["line"]) + self.assertFalse(self.r1["doc"]["env"]["guest"]["seed"]["reused"]) + self.assertTrue(self.r2["doc"]["env"]["guest"]["seed"]["reused"]) + + def test_the_seed_is_kept_shut_down_and_everything_else_is_gone(self): + d = devs(self.state) + self.assertEqual([v["name"] for v in d.values()], ["bench-seed-iPhone-17-Pro-1x"]) + self.assertEqual({v["state"] for v in d.values()}, {"Shutdown"}) + self.assertEqual(manifest(self.state)["seeds"]["dev.macbox.CoreSimulator.SimDeviceType.iPhone-17-Pro-1x"]["complete"], True) + self.assertEqual(files(self.state, "sim-keep"), []) + ov = files(self.state, "overrides") # only the seed's own Slim disables stay + self.assertEqual(len(ov), 1) + self.assertEqual(ov[0].split(".")[-1], list(d)[0]) + + def test_simlocks_set_is_read_only(self): + src = os.path.join(self.state, "guest", "simlock-sources") + fresh = tempfile.mkdtemp(prefix="dry-src-") + stub.make_source_set(os.path.join(fresh, "s")) + + def tree(root): + out = [] + for dp, _, fs in os.walk(root): + for f in fs: + with open(os.path.join(dp, f), "rb") as fh: + out.append((os.path.relpath(os.path.join(dp, f), root), fh.read())) + return sorted(out) + self.assertEqual(tree(src), tree(os.path.join(fresh, "s"))) + with open(os.path.join(self.state, "guest", "state.json")) as f: + self.assertEqual(json.load(f).get("source_violations", []), []) + for c in calls(self.state): + self.assertNotIn("config.json", " ".join(c)) + if c[:2] == ["xcrun", "simctl"] and c[3] == src: + self.assertEqual(c[4], "clone") # never boot, shutdown, delete, create... in it + + def test_every_clone_has_the_seeds_slim_disables_before_its_first_boot(self): + seed = list(devs(self.state))[0] + cs = calls(self.state) + boots = [c[-1] for c in cs if c[:2] == ["xcrun", "simctl"] and c[4:5] == ["boot"]] + clone_udids = [u for u in boots if u != seed] + self.assertTrue(clone_udids) + for u in clone_udids: + boot_at = next(i for i, c in enumerate(cs) if c[4:6] == ["boot", u]) + checked = [i for i, c in enumerate(cs) if c[0] == "plutil" and c[-1].endswith("SimDevice.%s/disabled.plist" % u)] + self.assertTrue(checked and checked[0] < boot_at, u) + for st in self.r1["doc"]["data"]["steps"]: + self.assertEqual(len(st["sims"]), st["n"]) + + def test_the_seed_is_slimmed_after_one_boot_and_before_the_app(self): + cs = calls(self.state) + kinds = [] + for c in cs: + if c[0].endswith("simslim"): + kinds.append("slim") + elif c[:2] == ["xcrun", "simctl"] and c[4:5] == ["boot"]: + kinds.append("boot") + elif c[:2] == ["xcrun", "simctl"] and c[4:5] == ["install"]: + kinds.append("install") + self.assertEqual(kinds[:3], ["boot", "slim", "install"]) + slim = next(c for c in cs if c[0].endswith("simslim")) + self.assertEqual(slim[1:4], ["--set", os.path.join(self.state, "guest", "set"), "on"]) + self.assertIn("--profile", slim) + + +class SeedFailures(unittest.TestCase): + def failing(self, key, needle): + d = parallel(JOBS)[key] + r = d["r"] + self.assertNotEqual(r["rc"], 0, r["out"][-500:]) + self.assertIn(needle, r["out"]) + self.assertEqual(d["snap"]["devs"], {}, "a failed seed must not stay in the set") # deleted again: never reusable + self.assertEqual(d["snap"]["keep"], []) + self.assertEqual(r["doc"]["data"]["steps"], []) + self.assertEqual(d["next"]["rc"], 0, d["next"]["out"][-500:]) # and the next run is not blocked + return r + + def test_no_source_device_of_the_type_says_so_and_offers_the_stock_path(self): + r = self.failing("nosource", "Simlock has no source device of type iPhone-17-Pro-1x (it has: iPhone-17)") + self.assertIn("--stock-devices", r["out"]) + self.assertEqual([c for c in calls(r["state"]) if "create" in c], []) # it did not fall back to stock devices by itself + + def test_no_simlock_at_all_says_so(self): + r = parallel(JOBS)["nosourceset"]["r"] + self.assertNotEqual(r["rc"], 0) + self.assertIn("Simlock is optional", r["out"]) + self.assertIn("--stock-devices", r["out"]) + + def test_simslim_failure(self): + self.failing("slimfail", "simslim failed on the seed") + + def test_a_seed_that_is_not_slim_enough_fails(self): + self.failing("slimfew", "not Slim after simslim: 20 services off") + + def test_a_seed_without_the_widgets_icons_folder_is_recorded_not_refused(self): + r = parallel(JOBS)["noicons"]["r"] # the folder is not in the verified recipe: an unknown, never a gate + self.assertEqual(r["rc"], 0, r["out"][-500:]) + self.assertIs(r["doc"]["env"]["guest"]["seed"]["icons"], False) + + def test_the_app_must_be_in_the_seed(self): + self.failing("noinstall", "not installed in the seed") + + +class SeedLifecycle(unittest.TestCase): + def test_a_stale_app_stops_the_run_and_rebuild_seed_fixes_it(self): + d = parallel(JOBS)["stale"] + self.assertEqual(d["first"]["rc"], 0) + self.assertNotEqual(d["stale"]["rc"], 0) + self.assertIn("--rebuild-seed", d["stale"]["out"]) + self.assertEqual(d["rebuilt"]["rc"], 0, d["rebuilt"]["out"][-500:]) + self.assertEqual(len([c for c in calls(d["state"]) if c[0].endswith("simslim")]), 2) + self.assertFalse(d["rebuilt"]["doc"]["env"]["guest"]["seed"]["reused"]) + + def test_the_seed_command_and_cleanup_seeds(self): + d = parallel(JOBS)["seedcmd"] + self.assertEqual(d["seed"]["rc"], 0, d["seed"]["out"][-500:]) + self.assertEqual(d["ndevs"], 1) + self.assertEqual(d["cleanup"]["rc"], 0, d["cleanup"]["out"]) + self.assertEqual(devs(d["state"]), {}) + self.assertEqual(files(d["state"], "overrides"), []) + + def test_stock_devices_are_an_explicit_path_and_say_so(self): + d = parallel(JOBS)["stock"] + r, state = d["r"], d["state"] # needs no Simlock at all (nosourceset) + self.assertEqual(r["rc"], 0, r["out"][-600:]) + cs = calls(state) + self.assertEqual(len([c for c in cs if "create" in c]), 2) + self.assertEqual([c for c in cs if c[0].endswith("simslim")], []) + self.assertIn("fresh stock devices", r["doc"]["env"]["guest"]["device_type"]) + self.assertEqual(r["doc"]["env"]["guest"]["device_source"], "stock devices") + self.assertEqual(devs(state), {}) + self.assertEqual(files(state, "sim-keep"), []) + + def test_the_lifecycle_cycles_clone_the_seed(self): + d = parallel(JOBS)["lifecycle"] + r, state = d["r"], d["state"] + self.assertEqual(r["rc"], 0, r["out"][-600:]) + med = r["doc"]["data"]["median_s"] + self.assertIn("clone_s", med) + self.assertNotIn("create_s", med) + self.assertNotIn("install_s", med) + self.assertEqual([c for c in calls(state) if "create" in c], []) + self.assertEqual(len(devs(state)), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/bench/tests/test_dry_state.py b/src/bench/tests/test_dry_state.py new file mode 100644 index 0000000..8171e90 --- /dev/null +++ b/src/bench/tests/test_dry_state.py @@ -0,0 +1,140 @@ +"""DRY runs that follow each other in one stub guest: what a run leaves, what the next one refuses, the idle guard. +Each case is a real `run` process (stub guest, fake host); nothing real is touched.""" +import json +import os +import signal +import subprocess +import sys +import tempfile +import time +import unittest + +from tests.test_dry import FAST, RUN, run_case, sections, simctl_calls + +ONE = ["scale", "--n", "1,2"] + FAST + + +def calls(state): + with open(os.path.join(state, "guest", "calls.log")) as f: + return [json.loads(l) for l in f] + + +def keep_calls(state): + return [c for c in calls(state) if c[0].endswith("sim-idle-guard")] + + +def files(state, sub): + try: + return sorted(os.listdir(os.path.join(state, "guest", sub))) + except OSError: + return [] + + +class CleanupAndLeftovers(unittest.TestCase): + def test_a_cleanup_that_cannot_delete_says_so_exits_nonzero_and_blocks_the_next_run(self): + state = tempfile.mkdtemp(prefix="dry-state-") + r1 = run_case(ONE, "cantdelete", state=state) + self.assertEqual(r1["rc"], 20, r1["out"][-800:]) + c = sections(r1["doc"], "cleanup")[0] + self.assertEqual(c["status"], "FAIL") + self.assertIn("could not clean up", c["line"]) + self.assertNotIn("CLEANUP PASS", r1["out"]) + self.assertTrue(os.path.exists(os.path.join(state, "guest", "set", ".bench-run.lock"))) + n_before = len([x for x in calls(state) if "clone" in x or "create" in x]) + r2 = run_case(ONE, "", state=state) # the next run refuses to start + self.assertEqual(r2["rc"], 2, r2["out"][-800:]) + self.assertIn("run `macbox bench cleanup`", r2["doc"]["sections"][0]["line"]) + self.assertEqual(n_before, len([x for x in calls(state) if "clone" in x or "create" in x])) # it made nothing + r3 = run_case(["cleanup"], "", state=state) # the explicit command cleans, verified + self.assertEqual(r3["rc"], 0, r3["out"]) + self.assertIn("CLEANUP PASS", r3["out"]) + r4 = run_case(ONE, "", state=state) + self.assertEqual(r4["rc"], 0, r4["out"][-800:]) + + def test_cleanup_command_fails_loudly_when_it_cannot_delete(self): + state = tempfile.mkdtemp(prefix="dry-state-") + run_case(ONE, "cantdelete", state=state) + r = run_case(["cleanup"], "cantdelete", state=state) + self.assertEqual(r["rc"], 20) + self.assertIn("CLEANUP FAIL", r["out"]) + + +class IdleGuard(unittest.TestCase): + def test_every_bench_simulator_is_kept_for_the_run_and_the_keep_files_are_removed(self): + state = tempfile.mkdtemp(prefix="dry-state-") + r = run_case(ONE, "", state=state) + self.assertEqual(r["rc"], 0, r["out"][-600:]) + made = {c[2] for c in keep_calls(state)} + self.assertEqual(len(made), 3) # the seed (made in this run) and the two clones + for c in keep_calls(state): + self.assertEqual(c[1], "keep") + self.assertEqual(c[3], str(95)) # --deadline 5400 s = 90 min + 5 + self.assertEqual(files(state, "sim-keep"), []) # cleanup removed ~/.sim-keep/ + self.assertEqual(r["doc"]["env"]["guest"]["idle_guard"], "kept") + + def test_minutes_are_capped_at_240(self): + state = tempfile.mkdtemp(prefix="dry-state-") + r = run_case(ONE + ["--deadline", "30000"], "", state=state) + self.assertEqual(r["rc"], 0, r["out"][-600:]) + self.assertTrue(keep_calls(state)) + self.assertEqual({c[3] for c in keep_calls(state)}, {"240"}) + + def test_missing_tool_is_recorded_and_the_run_goes_on(self): + r = run_case(ONE, "noguard") + self.assertEqual(r["rc"], 0, r["out"][-600:]) + self.assertEqual(r["doc"]["env"]["guest"]["idle_guard"], "absent") + s = sections(r["doc"], "idle_guard")[0] + self.assertEqual(s["status"], "INFO") + self.assertIn("ABSENT", s["line"]) + + def test_a_refusing_tool_stops_the_run(self): + r = run_case(ONE, "keepfail") + self.assertNotEqual(r["rc"], 0) + self.assertIn("sim-idle-guard keep", r["out"]) + + +if __name__ == "__main__": + unittest.main() + + +def devices_of(state): + with open(os.path.join(state, "guest", "state.json")) as f: + return json.load(f)["devs"] + + +def kill_mid_seed(state): + """A run killed outright (no signal handler, no exit trap, the guest helper gone with it) while its seed is half made: + what an ssh drop or a SIGKILL leaves behind.""" + d = tempfile.mkdtemp(prefix="dry-kill-") + env = dict(os.environ, DRY="1", BENCH_STUB_FAULT="slowslim", BENCH_DRY_KEEP=state, BENCH_DEADLINE_MARGIN="3") + p = subprocess.Popen([sys.executable, RUN] + ONE + ["--out", os.path.join(d, "out")], env=env, stdout=subprocess.DEVNULL, + stderr=subprocess.STDOUT, start_new_session=True) + t0 = time.time() + try: + while time.time() - t0 < 60 and not any(c[0].endswith("simslim") for c in (calls(state) if os.path.exists(os.path.join(state, "guest", "calls.log")) else [])): + time.sleep(0.2) + finally: + os.killpg(p.pid, signal.SIGKILL) + p.wait() + return any(c[0].endswith("simslim") for c in calls(state)) + + +class HalfMadeSeed(unittest.TestCase): + def test_a_killed_seed_is_refused_then_cleanup_removes_it_and_the_next_run_works(self): + state = tempfile.mkdtemp(prefix="dry-half-") + self.assertTrue(kill_mid_seed(state), "the run never reached simslim") + left = devices_of(state) + self.assertEqual(len(left), 1) # the half-made seed + self.assertEqual(files(state, "sim-keep"), list(left)) + r = run_case(ONE, "", state=state) # the next run refuses to start on it + self.assertNotEqual(r["rc"], 0) + self.assertIn("never completed", r["out"]) + self.assertIn("macbox bench cleanup", r["out"]) + c = run_case(["cleanup"], "", state=state) # a plain cleanup (no --seeds) removes it + self.assertEqual(c["rc"], 0, c["out"][-600:]) + self.assertIn("CLEANUP PASS", c["out"]) + self.assertEqual(devices_of(state), {}) + self.assertEqual(files(state, "sim-keep"), []) + self.assertFalse(os.path.exists(os.path.join(state, "guest", "set", ".bench-run.lock"))) + ok = run_case(ONE, "", state=state) + self.assertEqual(ok["rc"], 0, ok["out"][-600:]) diff --git a/src/bench/tests/test_modules.py b/src/bench/tests/test_modules.py new file mode 100644 index 0000000..29c89e5 --- /dev/null +++ b/src/bench/tests/test_modules.py @@ -0,0 +1,377 @@ +import importlib.util +import json +import os +import sys +import tempfile +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..")) +from macbench import analysis, chart, compare, gates, hostprobe, schema # noqa: E402 +from tests import helpers # noqa: E402 + +spec = importlib.util.spec_from_file_location("benchguest", os.path.join(HERE, "..", "guest", "benchguest.py")) +bg = importlib.util.module_from_spec(spec) +spec.loader.exec_module(bg) + +VM_STAT = """Mach Virtual Memory Statistics: (page size of 16384 bytes) +Pages free: 573102. +Pages active: 1833909. +Pages inactive: 100000. +Pages speculative: 10000. +Pages wired down: 946936. +Pages occupied by compressor: 52949. +Pageins: 123456. +Pageouts: 12. +Swapins: 0. +Swapouts: 7. +""" +PS = """ 1 0 4000 0:01.50 /sbin/launchd + 200 1 20000 2:03.25 launchd_sim /x/set/AAAA-1/data + 201 200 150000 1-00:00:10.00 SpringBoard + 300 1 90000 0:10.00 SimMetalHost -d BBBB-2 + 301 300 1000 0:00.10 helper + 400 1 1000 0:00.50 unrelated AAAA-1x +""" + + +def wr(path, mode, text): + with open(path, mode) as f: + f.write(text) + + +class SlimProfile(unittest.TestCase): + def test_explicit_env_then_the_only_file_else_an_error_that_says_what_to_pass(self): + import tempfile + d = tempfile.mkdtemp() + old = (bg.SLIM_DIR, bg.SLIM_PROFILE) + try: + bg.SLIM_DIR, bg.SLIM_PROFILE = d, None + with self.assertRaises(bg.GuestError) as e: + bg.resolve_profile() + self.assertIn("--slim-profile", str(e.exception)) + wr(os.path.join(d, "a.json"), "w", "{}") + self.assertEqual(bg.resolve_profile(), os.path.join(d, "a.json")) + wr(os.path.join(d, "b.json"), "w", "{}") + with self.assertRaises(bg.GuestError): + bg.resolve_profile() # two: no guessing + self.assertEqual(bg.resolve_profile("/x/p.json"), "/x/p.json") + bg.SLIM_PROFILE = "/env/p.json" + self.assertEqual(bg.resolve_profile(), "/env/p.json") + finally: + bg.SLIM_DIR, bg.SLIM_PROFILE = old + + +class GuestParsers(unittest.TestCase): + def test_vm_stat(self): + m = bg.mem_summary(bg.parse_vm_stat(VM_STAT)) + self.assertEqual(m["swapouts"], 7) + self.assertAlmostEqual(m["free_mb"], (573102 + 10000) * 16384 / 1048576.0, places=1) + self.assertAlmostEqual(m["compressed_mb"], 52949 * 16384 / 1048576.0, places=1) + self.assertEqual(bg.parse_vm_stat(VM_STAT)["page"], 16384) + + def test_available_memory_counts_cache_and_defaults_to_4k_pages(self): + m = bg.mem_summary(bg.parse_vm_stat(VM_STAT)) + self.assertAlmostEqual(m["avail_mb"], (573102 + 10000 + 100000) * 16384 / 1048576.0, places=1) # + inactive + self.assertGreater(m["avail_mb"], m["free_mb"]) + nohead = bg.parse_vm_stat(VM_STAT.replace("(page size of 16384 bytes)", "")) + if os.uname()[4] == "x86_64": + # without the header (x86 guest): 573102 free pages are 2.2 GiB at 4 KiB, not 8.7 GiB + self.assertEqual(nohead["page"], 4096) + self.assertAlmostEqual(573102 * 4096 / 1073741824.0, 2.19, places=2) + + def test_observer_is_nobodys_simulator_tree(self): + procs = bg.parse_ps(PS + " 500 1 2000 0:03.00 xcrun simctl --set /x AAAA-1 BBBB-2\n 501 500 2000 0:09.00 simfps AAAA-1\n") + own = bg.own_pids(procs, me=501) + self.assertEqual(own, {500, 501}) + trees = bg.sim_trees(procs, ["AAAA-1", "BBBB-2"], own) + self.assertNotIn(500, trees["AAAA-1"] | trees["BBBB-2"]) + self.assertNotIn(501, trees["AAAA-1"]) + later = [dict(p, cpu_s=p["cpu_s"] + 1.0) for p in procs] + per, tot = bg.attribute(procs, later, ["AAAA-1", "BBBB-2"], 2.0, own) + self.assertAlmostEqual(tot["observer_cpu_pct"], 100.0, places=1) # two processes x 1 s over 2 s + self.assertEqual(per["AAAA-1"]["procs"], 3) + + def test_settle_ignores_the_load_by_whole_command(self): + for cmd in ("/Library/Developer/CoreSimulator/Volumes/iOS 26.5.simruntime/Contents/bin/backboardd", + "/x/SimMetalHost -d AAAA-1", "simfps -set /x", "/usr/bin/ssh host", "/usr/bin/python3 benchguest.py settle"): + self.assertTrue(bg.SETTLE_SKIP.search(cmd), cmd) + for cmd in ("/usr/libexec/mds", "/System/Library/Frameworks/Foo.framework/assetsd", "/usr/sbin/cfprefsd daemon"): + self.assertFalse(bg.SETTLE_SKIP.search(cmd), cmd) + + def test_count_launchd_sim_sees_every_device_set(self): + text = PS + " 600 1 20000 0:01.00 /Library/x/launchd_sim /other/set/CCCC-3/data\n" + saved = bg.ps_text + bg.ps_text = lambda: text + try: + self.assertEqual(bg.count_launchd_sim(), 2) + finally: + bg.ps_text = saved + + def test_foreign_counts_a_bench_device_that_is_not_booted_as_the_benchs_own(self): + text = PS + " 600 1 20000 0:01.00 /Library/x/launchd_sim /bench/set/CCCC-3/data\n" # CCCC-3 is booting: state Shutdown + devs = [{"udid": "CCCC-3", "state": "Shutdown"}, {"udid": "AAAA-1", "state": "Booted"}, {"udid": "BBBB-2", "state": "Booted"}] + saved = bg.ps_text + bg.ps_text = lambda: text + try: + self.assertEqual(bg.count_launchd_sim(), 2) + self.assertEqual(bg.foreign_count(devs), 0) + self.assertEqual(bg.foreign_count(devs[1:2]), 1) # the same process with no device of the set behind it: foreign + finally: + bg.ps_text = saved + + def test_qemu_env_value_reads_the_process_environ(self): + with tempfile.TemporaryDirectory() as root: + os.makedirs(os.path.join(root, "proc", "77")) + wr(os.path.join(root, "proc", "77", "environ"), "w", "A=1\0REIMS_VGPU_CENSUS=/run/x/coherent\0REIMS_VGPU_COHERENT=1\0") + self.assertEqual(hostprobe.qemu_env_value(77, "REIMS_VGPU_CENSUS", root), "/run/x/coherent") + self.assertIsNone(hostprobe.qemu_env_value(77, "NOPE", root)) + self.assertIsNone(hostprobe.qemu_env_value(78, "A", root)) + + def test_cputime(self): + self.assertAlmostEqual(bg.parse_cputime("0:01.50"), 1.5) + self.assertAlmostEqual(bg.parse_cputime("2:03.25"), 123.25) + self.assertAlmostEqual(bg.parse_cputime("1-00:00:10.00"), 86410.0) + + def test_process_trees_and_attribution(self): + procs = bg.parse_ps(PS) + trees = bg.sim_trees(procs, ["AAAA-1", "BBBB-2"]) + self.assertEqual(trees["AAAA-1"], {200, 201, 400}) # a command line that names it, and children + self.assertEqual(trees["BBBB-2"], {300, 301}) + later = [dict(p, cpu_s=p["cpu_s"] + 2.0) for p in procs] + per, tot = bg.attribute(procs, later, ["AAAA-1", "BBBB-2"], 4.0) + self.assertEqual(per["BBBB-2"]["procs"], 2) + self.assertAlmostEqual(per["BBBB-2"]["cpu_pct"], 100.0, places=1) # 2 processes x 2 s over 4 s + self.assertAlmostEqual(tot["total_cpu_pct"], 6 * 2.0 / 4 * 100, places=1) + + def test_pace(self): + recs = bg.parse_pace('{"at": 100.5, "counter": 300, "ticks": 300, "fps": 59.9}\nnot json\n{"at": 105.5, "counter": 600, "ticks": 300, "fps": 60.0}\n') + self.assertEqual(recs, [(100.5, 300, 59.9, 300), (105.5, 600, 60.0, 300)]) + ps = analysis.pace_summary(recs + [(110.5, 900, 60.1, 300)], 102.0, 108.0, 101.0, 109.0) + self.assertEqual(ps["app_ticks"], 600) # records at or before the first frame (300) to the first after the last (900) + self.assertAlmostEqual(ps["app_tick_fps"], 60.0, delta=0.2) + + +class HostParsers(unittest.TestCase): + def test_task_stat_with_awkward_names(self): + fields = ["S", "1", "1", "0", "0", "0", "0", "0", "0", "0", "0", "300", "50", "0", "0", "20", "0", "1", "0", "100", "0", "0"] + ["0"] * 19 + ["120", "0"] + text = "123 (CPU 3/KVM (x)) " + " ".join(fields) + comm, ut, st, gt = hostprobe.parse_task_stat(text) + self.assertEqual((comm, ut, st), ("CPU 3/KVM (x)", 300, 50)) + self.assertEqual(gt, int(fields[40])) + + def test_thread_delta_splits_user_guest_sys(self): + a = {"1": ("CPU 0/KVM", 1000, 100, 800), "2": ("CPU 1/KVM", 1000, 100, 800), "3": ("metal2vulkan-sc", 100, 10, 0)} + b = {"1": ("CPU 0/KVM", 1400, 120, 1100), "2": ("CPU 1/KVM", 1400, 120, 1100), "3": ("metal2vulkan-sc", 150, 30, 0)} + d = hostprobe.thread_delta(a, b, 2.0, hz=100) + # user (host side) = (400-300) + (400-300) + 50 = 250 ticks; guest = 600; sys = 20+20+20 = 60: over 2 s at 100 Hz + self.assertAlmostEqual(d["user"], 1.25) + self.assertAlmostEqual(d["guest"], 3.0) + self.assertAlmostEqual(d["sys"], 0.3) + self.assertAlmostEqual(d["cores"], 4.55) + self.assertEqual(d["top"][0]["name"], "vCPU") + self.assertEqual(d["top"][0]["threads"], 2) + + def test_fdinfo_and_fake_proc_tree(self): + with tempfile.TemporaryDirectory() as root: + def put(rel, text): + p = root + rel + os.makedirs(os.path.dirname(p), exist_ok=True) + with open(p, "w") as f: + f.write(text) + put("/proc/7/fdinfo/10", "pos: 0\ndrm-client-id:\t5\ndrm-engine-render:\t1000 ns\n") + put("/proc/7/fdinfo/11", "pos: 0\ndrm-client-id:\t5\ndrm-engine-render:\t1000 ns\n") # a dup of the same client + put("/proc/7/fdinfo/12", "pos: 0\ndrm-client-id:\t6\ndrm-engine-render:\t500 ns\n") + put("/proc/7/fdinfo/13", "pos: 0\nflags: 02\n") + self.assertEqual(hostprobe.gpu_render_ns(7, root), 1500) + self.assertIsNone(hostprobe.gpu_render_ns(8, root)) + put("/proc/7/cgroup", "0::/machine/vm.scope\n") + put("/sys/fs/cgroup/machine/vm.scope/memory.current", "4294967296\n") + put("/sys/fs/cgroup/machine/vm.scope/memory.swap.current", "1048576\n") + put("/sys/fs/cgroup/machine/vm.scope/memory.events", "low 0\nhigh 0\nmax 2\noom 0\noom_kill 0\n") + put("/proc/meminfo", "MemTotal: 1000000 kB\nMemAvailable: 500000 kB\nSwapTotal: 0 kB\nSwapFree: 0 kB\n") + put("/proc/vmstat", "pswpin 10\npswpout 20\n") + put("/proc/7/status", "VmRSS:\t 2048 kB\n") + a = hostprobe.mem_snapshot(7, root) + self.assertEqual(a["cg_current"], 4294967296) + self.assertEqual(a["cg_swap"], 1048576) + self.assertEqual(a["qemu_rss_kb"], 2048) + self.assertEqual((a["mem_avail_kb"], a["mem_total_kb"]), (500000, 1000000)) + put("/proc/7/maps", "7f00 r-xp 0 00:00 0 /opt/mesa/lib/libvulkan_intel.so\n") + self.assertTrue(hostprobe.mesa_is_custom(7, root)) + put("/proc/7/maps", "7f00 r-xp 0 00:00 0 /usr/lib/x86_64-linux-gnu/libvulkan_intel.so\n") + self.assertFalse(hostprobe.mesa_is_custom(7, root)) + put("/proc/vmstat", "pswpin 15\npswpout 120\n") + put("/sys/fs/cgroup/machine/vm.scope/memory.swap.current", "3145728\n") + b = hostprobe.mem_snapshot(7, root) + d = hostprobe.mem_delta(a, b, 10) + self.assertEqual((d["pswpin_pages"], d["pswpout_pages"]), (5, 100)) + self.assertAlmostEqual(d["cgroup_swap_delta_mb"], 2.0) + put("/proc/7/environ", "A=1\0REIMS_VGPU_CENSUS=/some/dir\0REIMS_VGPU_COHERENT=1\0") + self.assertEqual(hostprobe.qemu_env_switches(7, root), {"REIMS_VGPU_CENSUS": "", "REIMS_VGPU_COHERENT": "1"}) + + def test_gpu_error_regex(self): + self.assertTrue(hostprobe.GPU_ERRORS.search("i915 0000:00:02.0: [drm] GPU HANG: ecode 12")) + self.assertTrue(hostprobe.GPU_ERRORS.search("Fence expired on rcs0")) + self.assertFalse(hostprobe.GPU_ERRORS.search("i915: frequency changed to 1200")) + + +class Gates(unittest.TestCase): + def setUp(self): + self.d = tempfile.mkdtemp() + self.census = os.path.join(self.d, "coherent.jsonl") + self.log = os.path.join(self.d, "qemu.log") + self.hang = os.path.join(self.d, "hang") + wr(self.log, "w", "old line\ncommand buffer failed (before the mark)\n") + with open(self.census, "w") as f: + f.write(json.dumps({"id": 1, "overwrite_risk": 3, "sync_errors": 0}) + "\n") # history: before the mark + f.write(json.dumps({"process": 1, "late_copies_in_busy": 0}) + "\n") + + def test_deltas_only(self): + c = gates.Coherent(self.d, self.log, self.hang) + m = c.mark() + r = c.check(m) + self.assertEqual(r["coherent_census"]["status"], "SKIP") # nothing new: not a pass + self.assertEqual(r["qemu_log"]["status"], "PASS") # the old failure is before the mark + with open(self.census, "a") as f: + f.write(json.dumps({"id": 1, "overwrite_risk": 0, "sync_errors": 0}) + "\n") + self.assertEqual(c.check(m)["coherent_census"]["status"], "PASS") + with open(self.census, "a") as f: + f.write(json.dumps({"id": 1, "overwrite_risk": 1, "sync_errors": 0}) + "\n") + f.write(json.dumps({"process": 1, "late_copies_in_busy": 2}) + "\n") + r = c.check(m) + self.assertEqual(r["coherent_census"]["status"], "FAIL") + self.assertEqual(r["coherent_process"]["status"], "FAIL") + wr(self.log, "a", "m2v: command buffer failed\n") + wr(self.hang, "w", "x") + r = c.check(m) + self.assertEqual(r["qemu_log"]["status"], "FAIL") + self.assertEqual(r["gpu_hang"]["status"], "FAIL") + + def test_unreadable_is_skip_never_pass(self): + r = gates.Coherent(None, None, None).check({"n": 0, "proc": {k: 0 for k in gates.PROCESS_GATES}, "log_off": None, "hang": False}) + self.assertTrue(all(g["status"] == "SKIP" for g in r.values()), r) + self.assertEqual(gates.kernel_gate(None, "no journal")["status"], "SKIP") + self.assertEqual(gates.kernel_gate(0, "")["status"], "PASS") + self.assertEqual(gates.kernel_gate(1, "x")["status"], "FAIL") + + def test_screen_gates(self): + ok = {"ok": True, "hash": "aa", "tear": False, "tear_applicable": True, "w": 402, "h": 874} + by = [{"idx": 0, "role": "busy", "shots": [ok, dict(ok, tear=True)]}, {"idx": 1, "role": "static", "shots": [ok, ok]}] + r = gates.shots_gates(by, lambda w, h: ("aa", "k")) + self.assertEqual(r["tears"]["status"], "FAIL") + self.assertEqual(r["pixels"]["status"], "PASS") + self.assertEqual(gates.shots_gates(by, lambda w, h: ("bb", "k"))["pixels"]["status"], "FAIL") + self.assertEqual(gates.shots_gates(by, lambda w, h: (None, "k"))["pixels"]["status"], "SKIP") + by[1]["shots"][1] = dict(ok, hash="cc") + self.assertEqual(gates.shots_gates(by, None)["pixels"]["status"], "FAIL") # the same screen, two pictures + by2 = [{"idx": 0, "role": "busy", "shots": [dict(ok, tear_applicable=False)]}] + self.assertEqual(gates.shots_gates(by2, None)["tears"]["status"], "SKIP") + + def test_leak_gate(self): + self.assertEqual(gates.leak_gate("x", [1, 2, 3, 4], [0, 50, 100, 150], 20, "MB")["status"], "FAIL") + self.assertEqual(gates.leak_gate("x", [1, 2, 3, 4], [0, 1, 2, 3], 20, "MB")["status"], "PASS") + self.assertEqual(gates.leak_gate("x", [1], [0], 20, "MB")["status"], "SKIP") + # one step up, then flat (a host bringing swapped pages back): above the bar, but not a steady growth + self.assertEqual(gates.leak_gate("x", [1, 2, 3], [90, 570, 520], 20, "MB")["status"], "SKIP") + self.assertEqual(gates.leak_gate("x", [1, 2, 3, 4, 5], [0, 480, 470, 490, 480], 20, "MB")["status"], "SKIP") + + +class Schema(unittest.TestCase): + def test_fake_result_validates_and_is_private(self): + doc = helpers.fake_scale_result() + self.assertEqual(schema.validate(doc), []) + + def test_privacy_catches_paths_ips_hosts(self): + doc = helpers.fake_scale_result() + doc["env"]["x"] = "/var/lib/something/live.conf" + self.assertTrue(any("path" in p for p in schema.validate(doc))) + doc["env"]["x"] = "connected to 192.168.1.20" + self.assertTrue(any("ip" in p for p in schema.validate(doc))) + import socket + doc["env"]["x"] = "ran on %s" % socket.gethostname() + self.assertTrue(any("host" in p for p in schema.validate(doc))) + + def test_scrub_then_publishable(self): + import socket + doc = helpers.fake_scale_result() + doc["env"]["x"] = "ssh %s from %s at 10.0.0.1" % (socket.gethostname(), os.path.expanduser("~") + "/work") + clean = schema.scrubber(["my-vm-alias"]) + doc["env"]["y"] = "my-vm-alias" + d = tempfile.mkdtemp() + p = os.path.join(d, "r.json") + problems = schema.write(doc, p, clean) + self.assertEqual(problems, []) + back = schema.load(p) + self.assertTrue(back["publishable"]) + self.assertNotIn(socket.gethostname(), json.dumps(back)) + self.assertEqual(back["env"]["y"], "") + + def test_write_keeps_data_even_when_not_publishable(self): + doc = helpers.fake_scale_result() + doc["env"]["x"] = "/var/private/path" + p = os.path.join(tempfile.mkdtemp(), "r.json") + problems = schema.write(doc, p) + self.assertTrue(problems) + back = schema.load(p) + self.assertFalse(back["publishable"]) + self.assertEqual(len(back["data"]["steps"]), 3) + + def test_validate_rejects_bad_shapes(self): + self.assertTrue(schema.validate({"schema": 99})) + doc = helpers.fake_scale_result() + del doc["data"]["steps"][0]["system"] + self.assertTrue(any("system" in p for p in schema.validate(doc))) + + +class CompareAndChart(unittest.TestCase): + def test_compare_flags_regressions_beyond_noise(self): + a = helpers.fake_scale_result(fps=(60, 58, 56)) + b = helpers.fake_scale_result(fps=(60, 58, 40)) + text, worse = compare.compare(a, b) + self.assertGreater(worse, 0) + self.assertIn("WORSE", text) + same, w0 = compare.compare(a, helpers.fake_scale_result(fps=(60, 58, 56))) + self.assertEqual(w0, 0, same) + c = helpers.fake_scale_result(fps=(61, 58.5, 56.5)) # +1.7%, +0.9%: inside the 4% noise + self.assertEqual(compare.compare(a, c)[1], 0) + + def test_compare_max_n(self): + a = helpers.fake_scale_result(fps=(60, 58, 56)) + b = helpers.fake_scale_result(fps=(60, 58, 40)) + text, _ = compare.compare(a, b) + self.assertIn("max sustainable N smooth60", text) + + def test_chart_is_self_contained(self): + html = chart.scale_chart(helpers.fake_scale_result(ns=(1, 2, 3, 4), fps=(60, 59, 55, 30))) + self.assertIn("")[0] + "") + + +class AgentAndMotion(unittest.TestCase): + def test_motion_metrics_ignore_idle_gaps(self): + t = [i / 60 for i in range(60)] + [5 + i / 60 for i in range(60)] + m = analysis.motion_metrics(t) + self.assertEqual(m["stretches"], 2) + self.assertAlmostEqual(m["fps"], 60.0, delta=0.5) + self.assertEqual(m["over2x"], 0) + + def test_agent_metrics(self): + raw = [(10.0 + i / 60, i, 255) for i in range(300)] + acts = [{"op": "launch", "t0": 10.5, "t1": 10.9, "rc": 0}, {"op": "screenshot", "t0": 12.0, "t1": 12.4, "rc": 0}, + {"op": "terminate", "t0": 13, "t1": 13.1, "rc": 0}] + m = analysis.agent_metrics(acts, raw) + self.assertAlmostEqual(m["launch_to_frame_ms_p50"], 17, delta=17) + self.assertAlmostEqual(m["screenshot_ms_p50"], 400, delta=1) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/bench/tests/test_pure.py b/src/bench/tests/test_pure.py new file mode 100644 index 0000000..bb2eb1b --- /dev/null +++ b/src/bench/tests/test_pure.py @@ -0,0 +1,297 @@ +import os +import sys +import unittest + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..")) +from macbench import abrule, analysis, gates, png, stats, strip, targets, tears # noqa: E402 +from tests import helpers # noqa: E402 + + +class CrashReports(unittest.TestCase): + NEW = ["searchd-2026-11-17-065234.ips", "SpringBoard-2026-11-17-065301.ips", "GlassBench_2026-11-17-065302_Mac.crash", + "PassbookWidgetsExtension-iPhone-2026-11-17-065303.ips", "com.apple.CoreSimulator.CoreSimulatorService-2026-11-17-070000.ips"] + + def test_process_name_of_a_report(self): + self.assertEqual(gates.crash_proc("searchd-2026-11-17-065234.ips"), "searchd") + self.assertEqual(gates.crash_proc("PassbookWidgetsExtension-iPhone-2026-11-17-065303.ips"), "PassbookWidgetsExtension-iPhone") + self.assertEqual(gates.crash_proc("GlassBench_2026-11-17-065302_Mac.crash"), "GlassBench") + self.assertEqual(gates.crash_proc("odd.ips"), "odd") + + def test_only_the_bench_and_the_graphics_path_are_judged(self): + judged, listed = gates.crash_split(self.NEW) + self.assertEqual([gates.crash_proc(r) for r in judged], ["GlassBench", "SpringBoard", "com.apple.CoreSimulator.CoreSimulatorService"]) + self.assertEqual([gates.crash_proc(r) for r in listed], ["PassbookWidgetsExtension-iPhone", "searchd"]) + + def test_more_names_and_strict(self): + judged, listed = gates.crash_split(self.NEW, extra=["SearchD", " "]) + self.assertIn("searchd-2026-11-17-065234.ips", judged) + judged, listed = gates.crash_split(self.NEW, strict=True) + self.assertEqual((len(judged), listed), (5, [])) + + def test_gate_names_what_it_did_not_judge(self): + g = gates.crash_gate(["searchd-2026-11-17-065234.ips"]) + self.assertEqual(g["status"], "PASS") + self.assertIn("not judged: 1 other report(s): searchd-2026-11-17-065234.ips", g["detail"]) + g = gates.crash_gate(self.NEW) + self.assertEqual(g["status"], "FAIL") + self.assertIn("3 new crash report(s)", g["detail"]) + self.assertEqual(gates.crash_gate([])["status"], "PASS") + + +class Stats(unittest.TestCase): + def test_percentile_matches_simfps_rule(self): + v = list(range(1, 101)) + self.assertEqual(stats.percentile(v, 0.5), 50) # sorted[int(99*0.5)] = sorted[49] + self.assertEqual(stats.percentile(v, 0.95), 95) + self.assertEqual(stats.percentile(v, 0.99), 99) + self.assertEqual(stats.percentile(v, 1.0), 100) + self.assertIsNone(stats.percentile([], 0.5)) + self.assertEqual(stats.percentile([7], 0.99), 7) + + def test_frame_stats(self): + t = [i / 60 for i in range(121)] # 2 s at 60 fps + m = stats.frame_stats(t) + self.assertAlmostEqual(m["fps"], 60.0, delta=0.1) + self.assertEqual(m["over2x"], 0) + t2 = t[:60] + [x + 0.05 for x in t[60:]] # one 66 ms gap + m2 = stats.frame_stats(t2) + self.assertEqual(m2["over2x"], 1) + self.assertAlmostEqual(m2["longest_gap_s"], 0.067, places=2) + self.assertEqual(stats.frame_stats([1.0])["fps"], 0.0) + + def test_stretches(self): + t = [0, .01, .02, .03, 1.0, 1.01, 1.02] + self.assertEqual(stats.stretches(t, 250), [(0, .03, 4), (1.0, 1.02, 3)]) + + def test_jain_and_minmax(self): + self.assertAlmostEqual(stats.jain([60, 60, 60]), 1.0) + self.assertAlmostEqual(stats.jain([60, 0, 0]), 1 / 3) + self.assertAlmostEqual(stats.jain([60, 30]), 0.9, places=6) + self.assertEqual(stats.jain([0, 0]), 1.0) + self.assertAlmostEqual(stats.min_max_ratio([30, 60]), 0.5) + self.assertIsNone(stats.jain([])) + + def test_slope(self): + b, a, r2 = stats.slope([1, 2, 3, 4], [10, 20, 30, 40]) + self.assertAlmostEqual(b, 10) + self.assertAlmostEqual(a, 0) + self.assertAlmostEqual(r2, 1) + self.assertEqual(stats.slope([1], [1]), (None, None, None)) + self.assertEqual(stats.slope([2, 2], [1, 2]), (None, None, None)) + b, _, _ = stats.slope([1, 2, 3], [5, None, 9]) # None is skipped + self.assertAlmostEqual(b, 2) + + +class Strip(unittest.TestCase): + def test_decode_every_counter_and_scale(self): + for width in (402, 1206): + for c in (0, 1, 2, 255, 21845, 43690, 65535): + self.assertEqual(strip.read_counter_image(helpers.strip_image(c, width)), c, (width, c)) + + def test_no_strip_is_unreadable(self): + img = png.Image(402, 900, 3, [bytes([128] * 402 * 3)] * 900) + self.assertEqual(strip.read_counter_image(img), -1) + + def test_png_roundtrip_and_hash(self): + img = helpers.strip_image(1234, 402) + back = png.read_png(png.write_png(img)) + self.assertEqual(back.pixel_hash(), img.pixel_hash()) + self.assertEqual(strip.read_counter_image(back), 1234) + other = helpers.strip_image(1235, 402) + self.assertNotEqual(other.pixel_hash(), img.pixel_hash()) + + def test_analyze_clean_with_repeats_and_wrap(self): + s = [(0.00, 65534), (0.01, 65534), (0.02, 65535), (0.03, 0), (0.04, 0), (0.05, 1)] + a = strip.analyze_counters(s) + self.assertEqual([c for _, c in a["unique"]], [65534, 65535, 0, 1]) + self.assertEqual((a["repeats"], a["lost"], a["backwards"], a["jumps"], a["span_ticks"]), (2, 0, 0, 0, 3)) + self.assertTrue(strip.strip_gate(a, app_ticks=3)[0]) + + def test_lost_frames_are_counted_not_failed(self): + a = strip.analyze_counters([(0, 10), (0.03, 13), (0.05, 14)]) + self.assertEqual(a["lost"], 2) + self.assertTrue(strip.strip_gate(a)[0]) + + def test_out_of_order_fails(self): + a = strip.analyze_counters([(0, 10), (0.02, 12), (0.04, 11), (0.06, 13)]) + self.assertEqual(a["backwards"], 1) + ok, why = strip.strip_gate(a) + self.assertFalse(ok) + self.assertIn("out of order", why) + + def test_jump_and_garbage_fail(self): + a = strip.analyze_counters([(0, 10), (1, 5000)]) + self.assertEqual(a["jumps"], 1) + self.assertFalse(strip.strip_gate(a)[0]) + a = strip.analyze_counters([(0, 10), (0.1, 60)]) + self.assertFalse(strip.strip_gate(a, app_ticks=10)[0]) # counter moved 50, the app says 10 ticks + self.assertTrue(strip.strip_gate(a, app_ticks=50)[0]) + + def test_unreadable_share(self): + s = [(i / 60, i) for i in range(50)] + [(1.0, -1)] * 10 + self.assertFalse(strip.strip_gate(strip.analyze_counters(s))[0]) + + +class Tears(unittest.TestCase): + def test_clean_screenshot_has_no_tear(self): + r = tears.detect_tears(helpers.card_image(603)) + self.assertTrue(r["applicable"]) + self.assertFalse(r["tear"], r) + + def test_torn_screenshot_is_found(self): + r = tears.detect_tears(helpers.card_image(603, tear_at=700)) + self.assertTrue(r["tear"], r) + + def test_a_gap_sliding_under_the_navigation_bar_is_not_a_tear(self): + # the bar's glass ends at 420 px (3x): a gap half under it showed as a short gap at a wrong distance + for scrolled in (190, 200, 216, 230, 245): + r = tears.detect_tears(helpers.card_image(603, scrolled=scrolled, bar=True)) + self.assertTrue(r["applicable"], scrolled) + self.assertFalse(r["tear"], (scrolled, r)) + r = tears.detect_tears(helpers.card_image(603, scrolled=216, bar=True, tear_at=700)) + self.assertTrue(r["tear"]) + + def test_no_cards_is_not_applicable(self): + r = tears.detect_tears(helpers.card_image(603, with_cards=False)) + self.assertFalse(r["applicable"]) + + +class Latency(unittest.TestCase): + def raw(self, flips, rate=60, secs=10): + """frames at `rate`; the marker (luma) flips at each time in `flips`""" + out, lum = [], 255 + for i in range(int(secs * rate)): + t = 100.0 + i / rate + if flips and t >= flips[0]: + lum = 0 if lum == 255 else 255 + flips = flips[1:] + out.append((t, i, lum)) + return out + + def test_latency_and_misses(self): + raw = self.raw([102.080, 105.250]) # answers 80 ms and 250 ms after the injections + inj = [(102.0, 102.12), (105.0, 105.14), (108.0, 108.1)] # the third is never answered + r = analysis.probe_latencies(raw, inj, timeout_s=2.0) + self.assertEqual(r["misses"], 1) + self.assertAlmostEqual(r["latencies_ms"][0], 83, delta=17) + self.assertAlmostEqual(r["latencies_ms"][1], 250, delta=17) + self.assertEqual(r["latencies_ms"][2], 2000.0) # a miss counts at the timeout + self.assertEqual(r["max_ms"], 2000.0) + self.assertAlmostEqual(r["call_p50_ms"], 120, delta=1) + + def test_a_second_look_answers_a_probe_and_is_not_a_frame(self): + # a still screen: the surface that appeared at 103.07 was read too early (old marker), the second look saw the flip + raw = [(100.0, -1, 255), (103.07, -1, 255), (103.10, analysis.SECOND_LOOK, 0)] + r = analysis.probe_latencies(raw, [(103.0, 103.1)]) + self.assertEqual((r["misses"], r["second_looks"]), (0, 1)) + self.assertAlmostEqual(r["latencies_ms"][0], 100, delta=1) + busy = [(100 + i / 60, i % 65536, 255) for i in range(120)] + [(100.5001, analysis.SECOND_LOOK, 0)] + m = analysis.sim_metrics(0, "busy", sorted(busy), []) + self.assertEqual(m["strip"]["unreadable"], 0) # the second look is not an unreadable frame + + def test_static_screen_uses_the_start_line(self): + raw = [(100.0, -1, 255), (103.07, -1, 0)] # no frames between: a static screen + r = analysis.probe_latencies(raw, [(103.0, 103.1)]) + self.assertEqual(r["misses"], 0) + self.assertAlmostEqual(r["latencies_ms"][0], 70, delta=1) + + def test_sim_metrics_busy_has_frames_and_probe(self): + raw = [(100 + i / 60, i % 65536, 255) for i in range(300)] + m = analysis.sim_metrics(0, "busy", raw, [], app_ticks=300) + self.assertAlmostEqual(m["fps"], 60, delta=0.5) + self.assertTrue(m["strip_gate"]["ok"]) + s = analysis.sim_metrics(1, "static", raw, [(101.0, 101.1)]) + self.assertNotIn("fps", s) + self.assertEqual(s["input_misses"], 1) + + +class Targets(unittest.TestCase): + def sims(self, fps, p95=20.0, n=3): + return [{"idx": i, "role": "busy", "fps": f, "p95_ms": p95} for i, f in enumerate(fps)] + + def test_parse(self): + t = targets.parse_target("smooth: fps>=55, p95<=25") + self.assertEqual(t["conds"], [("fps", ">=", 55.0), ("p95", "<=", 25.0)]) + with self.assertRaises(ValueError): + targets.parse_target("x: bogus>=1") + with self.assertRaises(ValueError): + targets.parse_target("no colon") + + def test_every_sim_must_pass(self): + t = targets.parse_target("t: fps>=55") + self.assertTrue(targets.evaluate(t, self.sims([60, 58, 56]))["pass"]) + r = targets.evaluate(t, self.sims([60, 58, 40])) + self.assertFalse(r["pass"]) + self.assertEqual(r["failing"][0]["sim"], 2) + + def test_unknown_when_nothing_to_judge(self): + t = targets.parse_target("t: input_p95<=150") + self.assertIsNone(targets.evaluate(t, self.sims([60]))["pass"]) + t = targets.parse_target("t: fps>=55, input_p95<=150") # one condition has no value anywhere: unknown, not a pass + self.assertIsNone(targets.evaluate(t, self.sims([60]))["pass"]) + self.assertFalse(targets.evaluate(t, self.sims([40]))["pass"]) # but a failed condition still fails it + + def test_max_sustainable_stops_at_first_fail(self): + steps = [{"n": n, "sims": self.sims([f] * n), "gates_ok": True} for n, f in ((1, 60), (2, 58), (3, 50), (4, 60))] + t = targets.parse_target("t: fps>=55") + r = targets.max_sustainable(steps, t) + self.assertEqual((r["max_n"], r["first_fail"]), (2, 3)) # 4 passes but 3 failed: not sustainable + self.assertTrue(r["evaluated"][4]) + + def test_gates_and_instability_veto(self): + steps = [{"n": 1, "sims": self.sims([60]), "gates_ok": True}, + {"n": 2, "sims": self.sims([60, 60]), "gates_ok": False}] + self.assertEqual(targets.max_sustainable(steps, targets.parse_target("t: fps>=55"))["max_n"], 1) + steps = [{"n": 1, "sims": self.sims([60]), "stable": False}] + self.assertIsNone(targets.max_sustainable(steps, targets.parse_target("t: fps>=55"))["max_n"]) + + def test_default_targets_parse(self): + for spec in targets.DEFAULT_TARGETS: + targets.parse_target(spec) + + def test_summary_and_fairness(self): + doc = helpers.fake_scale_result() + sm = doc["data"]["summary"] + self.assertEqual(sm["max_n"]["smooth60"]["max_n"], 2) + self.assertEqual(sm["max_n"]["smooth30"]["max_n"], 3) + self.assertAlmostEqual(doc["data"]["steps"][0]["fairness"]["jain"], 1.0) + sc = sm["standing_cost"] + self.assertAlmostEqual(sc["qemu_cpu_cores"]["per_sim"], 1.0, places=2) + self.assertAlmostEqual(sc["guest_mem_used_mb"]["per_sim"], 900.0, places=1) + + +class ABRule(unittest.TestCase): + def test_adopt_clear_win(self): + r = abrule.decide(100, 110, 100, 111) # +10, +11 + self.assertEqual(r["decision"], "ADOPT") + + def test_close_below_bar(self): + self.assertEqual(abrule.decide(100, 102, 100, 103)["decision"], "CLOSE") + + def test_close_when_pairs_disagree(self): + r = abrule.decide(100, 112, 100, 98) + self.assertEqual(r["decision"], "CLOSE") + self.assertTrue(any("disagree" in w for w in r["why"])) + + def test_lower_is_better(self): + self.assertEqual(abrule.decide(10, 8, 10, 8.5, higher_better=False)["decision"], "ADOPT") + self.assertEqual(abrule.decide(10, 12, 10, 12, higher_better=False)["decision"], "CLOSE") + + def test_tail_worsening_closes(self): + r = abrule.decide(100, 110, 100, 110, tails={"p95": (20, 25, 20, 26, False)}) + self.assertEqual(r["decision"], "CLOSE") + r = abrule.decide(100, 110, 100, 110, tails={"p95": (20, 20.2, 20, 20.3, False)}) + self.assertEqual(r["decision"], "ADOPT") + + def test_missing_values_close(self): + self.assertEqual(abrule.decide(0, 1, 100, 110)["decision"], "CLOSE") + + def test_decision_line(self): + r = abrule.decide(100, 110, 100, 111) + line = abrule.decision_line("knob=1", "distinct fps", ((100, 100), (110, 111)), r, 0.04) + self.assertIn("ADOPT", line) + self.assertIn("bar 4.0%", line) + + +if __name__ == "__main__": + unittest.main() diff --git a/src/bench/tests/test_runctx.py b/src/bench/tests/test_runctx.py new file mode 100644 index 0000000..d4713bc --- /dev/null +++ b/src/bench/tests/test_runctx.py @@ -0,0 +1,116 @@ +"""The run's safety, without the rest: the watchdog, the one trap, knobs.""" +import os +import signal +import subprocess +import sys +import tempfile +import time +import unittest + + +def rd(path): + with open(path) as f: + return f.read() + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..")) +from macbench.runctx import LiveKnob, Run # noqa: E402 + +SCRIPT = r''' +import sys, time +sys.path.insert(0, %r) +from macbench.runctx import Run +r = Run(6, print, margin=2) +r.teardown.append(("t", lambda: print("TEARDOWN RAN", flush=True))) +r.undo.append(("u", lambda: print("UNDO RAN", flush=True))) +r.on_deadline = lambda: print("PARTIAL WRITTEN", flush=True) +r.install() +mode = sys.argv[1] +if mode == "hang": + time.sleep(60) +elif mode == "child": + p = r.popen(["sleep", "60"]) + print("CHILD", p.pid, flush=True) + time.sleep(60) +''' + + +class RunSafety(unittest.TestCase): + def go(self, mode, sig=None, after=1.0): + p = subprocess.Popen([sys.executable, "-c", SCRIPT % os.path.join(HERE, ".."), mode], stdout=subprocess.PIPE, text=True) + if sig: + time.sleep(after) + p.send_signal(sig) + out, _ = p.communicate(timeout=30) + return p.returncode, out + + def test_deadline_runs_partial_write_then_cleanup_and_exits_124(self): + t0 = time.time() + rc, out = self.go("hang") + self.assertEqual(rc, 124) + self.assertLess(time.time() - t0, 10) + self.assertLess(out.index("PARTIAL WRITTEN"), out.index("TEARDOWN RAN")) + self.assertLess(out.index("TEARDOWN RAN"), out.index("UNDO RAN")) # simulators down first, then knobs restored + + def test_sigterm_runs_cleanup_once_and_kills_children(self): + rc, out = self.go("child", signal.SIGTERM) + self.assertEqual(rc, 143) + self.assertEqual(out.count("TEARDOWN RAN"), 1) + pid = int(out.split("CHILD")[1].split()[0]) + time.sleep(0.2) + with self.assertRaises(ProcessLookupError): + os.kill(pid, 0) + + def test_need_refuses_what_does_not_fit(self): + from macbench.runctx import DeadlineError + r = Run(100, margin=10) + r.need(40, "x") + with self.assertRaises(DeadlineError): + r.need(80, "y") + + def test_live_knob_restores_bytes(self): + d = tempfile.mkdtemp() + p = os.path.join(d, "live.conf") + with open(p, "w") as f: + f.write("a=1\n# comment\nb=2\nk=old\n") + r = Run(100) + k = LiveKnob(p, "k", "new", r) + k.set() + self.assertEqual(rd(p).splitlines()[-1], "k=new") + self.assertNotIn("k=old", rd(p)) + k.unset() + self.assertEqual(rd(p), "a=1\n# comment\nb=2\nk=old\n") + k.set() + r.cleanup() + self.assertEqual(rd(p), "a=1\n# comment\nb=2\nk=old\n") + + +if __name__ == "__main__": + unittest.main() + + +class GuardForeign(unittest.TestCase): + def test_a_simulator_outside_the_bench_refuses_the_run(self): + from macbench import core, fakes + + class A: + min_host_free_gib, disk_boot_gib, min_guest_free_gib, allow_foreign, disk_path = 1.0, 1.0, 1.0, False, "/" + max_compressor_mb = 2048.0 + + class G: + def __init__(self, foreign): + self.foreign = foreign + + def call(self, args, timeout): + return {"mem": {"free_mb": 8000, "avail_mb": 9000, "swapouts": 0, "compressed_mb": 0}, "pressure": 1, "booted": 0, "foreign_booted": self.foreign} + + class Ctx: + a, est_sim_mb, host, swap0 = A(), 1000.0, fakes.FakeHost(), None + c = Ctx() + c.guest = G(1) + self.assertIn("outside the bench", core.guard(c, 1, "x")) + c.a.allow_foreign = True + self.assertIsNone(core.guard(c, 1, "x")) + c.guest = G(0) + c.a.allow_foreign = False + self.assertIsNone(core.guard(c, 1, "x")) diff --git a/src/bench/tests/test_runguards.py b/src/bench/tests/test_runguards.py new file mode 100644 index 0000000..95a7318 --- /dev/null +++ b/src/bench/tests/test_runguards.py @@ -0,0 +1,241 @@ +"""The guards (host disk floors, guest swap and compressor, the VM still the same and answering, someone else's simulator) and +the monitor that applies them while the run waits. Pure: fake host, fake guest, no VM.""" +import argparse +import os +import subprocess +import sys +import tempfile +import time +import unittest + +HERE = os.path.dirname(os.path.abspath(__file__)) +sys.path.insert(0, os.path.join(HERE, "..")) +from macbench import cli, core, hostprobe # noqa: E402 + + +class FakeHost: + dry = True + + def __init__(self, disk=500.0, same=True, mem=40.0): + self.disk, self.same, self.mem = disk, same, mem + + def mem_avail_gb(self): + return self.mem + + def disk_free_gb(self, path): + return self.disk + + def same_qemu(self): + return self.same + + +class FakeGuest: + def __init__(self, mem=None, foreign=0, alive=True, pressure=1): + self.mem = mem or {"avail_mb": 9000, "free_mb": 5000, "swapouts": 7, "compressed_mb": 200} + self.foreign, self.up, self.pressure, self.calls = foreign, alive, pressure, [] + + def call(self, args, timeout=60, counted=True): + self.calls.append(args[0]) + return {"mem": self.mem, "foreign_booted": self.foreign, "pressure": self.pressure, "booted": 0} + + quick = call + + def alive(self): + self.calls.append("alive") + return self.up + + +def ns(**kw): + a = cli.parser().parse_args(["scale"]) + for k, v in kw.items(): + setattr(a, k, v) + return a + + +class Ctx: + def __init__(self, host=None, guest=None, **kw): + self.a = ns(**kw) + self.host, self.guest = host or FakeHost(), guest or FakeGuest() + self.est_sim_mb, self.swap0, self.in_window, self.taken = 1100.0, 7, False, True + + +class DiskFloors(unittest.TestCase): + def test_the_defaults_are_the_outage_lessons(self): + a = ns() + self.assertEqual((a.disk_start_gib, a.disk_boot_gib, a.disk_abort_gib, a.max_compressor_mb), (18.0, 14.0, 11.0, 2048.0)) + + def test_start_boot_and_abort_floors(self): + for disk, start, boot, keep_going in ((20, True, True, True), (17.9, False, True, True), (13.9, False, False, True), (10.9, False, False, False)): + c = Ctx(host=FakeHost(disk=disk)) + self.assertEqual(core.start_reason(c) is None, start, disk) + self.assertEqual(core.guard(c, 1, "x") is None, boot, disk) + self.assertEqual(core.disk_reason(c, c.a.disk_abort_gib, "go on") is None, keep_going, disk) + + def test_an_unreadable_disk_refuses_everything(self): + c = Ctx(host=FakeHost(disk=None)) + self.assertIn("cannot be read", core.start_reason(c)) + self.assertIn("cannot be read", core.guard(c, 1, "x")) + self.assertIn("cannot be read", monitor_result(c)[0]) + self.assertEqual(monitor_result(c)[1], 8) + + +class GuestGuards(unittest.TestCase): + def test_swapouts_may_not_grow_and_the_compressor_is_capped(self): + self.assertIsNone(core.guard(Ctx(), 1, "x")) + c = Ctx(guest=FakeGuest(mem={"avail_mb": 9000, "free_mb": 5000, "swapouts": 8, "compressed_mb": 200})) + self.assertIn("swap-outs grew from 7 to 8", core.guard(c, 1, "x")) + c = Ctx(guest=FakeGuest(mem={"avail_mb": 9000, "free_mb": 5000, "swapouts": 7, "compressed_mb": 2100})) + self.assertIn("compressor holds 2100 MB", core.guard(c, 1, "x")) + + def test_someone_elses_simulator_refuses_a_boot_unless_allowed(self): + self.assertIn("outside the bench", core.guard(Ctx(guest=FakeGuest(foreign=1)), 1, "x")) + self.assertIsNone(core.guard(Ctx(guest=FakeGuest(foreign=1), allow_foreign=True), 1, "x")) + + +def monitor_result(ctx): + ctx.a.watch_guest_every = 0.0 + return core.monitor_check(ctx)() + + +class Monitor(unittest.TestCase): + def test_quiet_run(self): + self.assertIsNone(monitor_result(Ctx())) + + def test_disk_under_the_abort_floor_ends_the_run_with_8(self): + r = monitor_result(Ctx(host=FakeHost(disk=10.5))) + self.assertEqual(r[1], 8) + self.assertIn("under the 11 GiB floor", r[0]) + + def test_a_restarted_qemu_ends_the_run_with_9(self): + self.assertEqual(monitor_result(Ctx(host=FakeHost(same=False)))[1], 9) + + def test_a_vm_that_stops_answering_twice_in_a_row_ends_the_run_with_9(self): + c = Ctx(guest=FakeGuest(alive=False)) + check = core.monitor_check(c) + c.a.watch_guest_every = 0.0 + self.assertIsNone(check()) # one miss is not a verdict + self.assertEqual(check()[1], 9) + c = Ctx(guest=FakeGuest(alive=False)) + c.a.watch_guest_every = 0.0 + check = core.monitor_check(c) + check() + c.guest.up = True + self.assertIsNone(check()) # an answer resets the count + c.guest.up = False + self.assertIsNone(check()) + + def test_someone_elses_simulator_makes_the_run_yield_with_7_when_two_polls_in_a_row_see_it(self): + c = Ctx(guest=FakeGuest(foreign=1)) + c.a.watch_guest_every = 0.0 + check = core.monitor_check(c) + self.assertIsNone(check()) # the bench's own boots and shutdowns race the count: one sighting is not a verdict + self.assertEqual(check()[1], 7) + self.assertIsNone(monitor_result(Ctx(guest=FakeGuest(foreign=1), allow_foreign=True))) + + def test_a_sighting_that_goes_away_resets_the_count(self): + c = Ctx(guest=FakeGuest(foreign=1)) + c.a.watch_guest_every = 0.0 + check = core.monitor_check(c) + self.assertIsNone(check()) + c.guest.foreign = 0 + self.assertIsNone(check()) + c.guest.foreign = 1 + self.assertIsNone(check()) + self.assertEqual(check()[1], 7) + + def test_swap_growth_and_the_compressor_end_the_run_with_10(self): + self.assertEqual(monitor_result(Ctx(guest=FakeGuest(mem={"swapouts": 9, "compressed_mb": 1})))[1], 10) + self.assertEqual(monitor_result(Ctx(guest=FakeGuest(mem={"swapouts": 7, "compressed_mb": 9999})))[1], 10) + + def test_a_measurement_window_is_left_alone(self): + c = Ctx(guest=FakeGuest(foreign=1)) + c.in_window = True + self.assertIsNone(monitor_result(c)) + self.assertEqual(c.guest.calls, []) # no observer load: the guest was not asked + c.host.disk = 5 + self.assertEqual(monitor_result(c)[1], 8) # but the host-side checks go on + + +class LifecycleGuard(unittest.TestCase): + def test_every_lifecycle_cycle_boots_behind_the_guard(self): + from macbench import others + stopped, cycles, seen = [], [], [] + c = Ctx(guest=FakeGuest()) + c.a = cli.parser().parse_args(["lifecycle", "--cycles", "3", "--storm", "0"]) + c.sims, c.seed, c.type_id, c.runtime, c.stock = [], {"seed": "S"}, "t", "r", False + c.marks, c.doc = {}, {"data": {}, "gates": {}, "sections": []} + c.coh = type("Coh", (), {"mark": lambda self: 0, "check": lambda self, m: {}})() + c.host.snap = lambda: {"mem": {}} + c.section = lambda name, status, line, extra=None: stopped.append((name, status, line)) + c.save = lambda: None + orig = (core.setup_guest, core.capture_env, core.guard, core.keep_minutes) + core.setup_guest, core.capture_env, core.keep_minutes = (lambda ctx: {}), (lambda ctx, info: None), (lambda ctx: 30) + core.guard = lambda ctx, k, label: (seen.append(label), "disk is low" if len(seen) == 2 else None)[1] + c.guest.call = lambda args, timeout=60, counted=True: (cycles.append(args[0]) or {"timings": {"clone_s": 1.0}, "mem": {"used_mb": 1, "compressed_mb": 1}}) + try: + others.run_lifecycle(c) + finally: + core.setup_guest, core.capture_env, core.guard, core.keep_minutes = orig + self.assertEqual(seen[:2], ["before lifecycle cycle 1", "before lifecycle cycle 2"]) + self.assertEqual(cycles.count("cycle"), 1) # the second was refused + self.assertIn(("stopped", "FAIL", "STOPPED: disk is low"), stopped) + + +class QemuIdentity(unittest.TestCase): + def test_reads_a_real_process(self): + h = hostprobe.Host(os.getpid()) + self.assertIsNotNone(h.born) + self.assertTrue(h.same_qemu()) + + def test_same_pid_with_another_start_time_is_another_vm(self): + root = tempfile.mkdtemp() + os.makedirs(os.path.join(root, "proc", "42")) + + def stat(start): + with open(os.path.join(root, "proc", "42", "stat"), "w") as f: + f.write("42 (qemu-system-x86) S " + " ".join(["1"] * 18) + " %s 5\n" % start) + stat(1000) + h = hostprobe.Host(42, root=root, hz=100) + self.assertTrue(h.same_qemu()) + stat(1000) + self.assertTrue(h.same_qemu()) + stat(7777) + self.assertFalse(h.same_qemu()) + os.remove(os.path.join(root, "proc", "42", "stat")) + self.assertFalse(h.same_qemu()) + + +SCRIPT = r''' +import sys, time, threading +sys.path.insert(0, %r) +from macbench.runctx import Run +r = Run(60, print, margin=2) +r.teardown.append(("t", lambda: (time.sleep(0.5), print("TEARDOWN RAN", flush=True)))) +r.on_abort = lambda reason: print("ABORT RECORDED", reason, flush=True) +r.on_deadline = lambda reason=None: print("PARTIAL WRITTEN", reason, flush=True) +r.final = lambda: print("FINAL", flush=True) +r.install() +t0 = time.time() +r.watch("x", lambda: ("disk gone", 8) if time.time() - t0 > 0.3 else None, 0.1) +try: + time.sleep(30) # the main thread is blocked in a "call" +finally: + r.cleanup() # and runs the one cleanup too when it is woken: it must wait for the monitor's + r.join_abort() +''' + + +class AbortFromAMonitor(unittest.TestCase): + def test_partial_result_cleanup_then_exit_code(self): + t0 = time.time() + p = subprocess.run([sys.executable, "-c", SCRIPT % os.path.join(HERE, "..")], capture_output=True, text=True, timeout=40) + self.assertEqual(p.returncode, 8, p.stdout + p.stderr) + out = p.stdout + self.assertLess(time.time() - t0, 20) + for word in ("disk gone", "PARTIAL WRITTEN", "TEARDOWN RAN", "FINAL"): + self.assertIn(word, out) + self.assertLess(out.index("TEARDOWN RAN"), out.index("FINAL")) # the closing sections come after the cleanup + + +if __name__ == "__main__": + unittest.main()