From 176d2f21c6d02d75cf1b57d9a588f862a46a3669 Mon Sep 17 00:00:00 2001 From: harry Date: Fri, 6 Nov 2026 18:06:18 +0900 Subject: [PATCH] docs: a performance model for devices we do not have Add scripts/perf_model.py, which fits time = fixed + per_pixel x megapixels on the M3 Max from the C API runs of release 0.11.0 and scales the per-pixel term by memory bandwidth to estimate other chips. The iPhone 17, measured the same way, checks it: the model reads 6 to 16 % above its measurements, so the table is no better than 15 %. - BENCHMARKS.md gains "Estimates for other devices": a table kept apart from the measurements and marked as estimated, with its basis, its check and its limits. `perf_model.py --write` rewrites the table. - Paced decoding (the GPU's power states) is not modelled and stays measured (#81). - ARCHITECTURE.md records the choice as D71. Refs #159. Claude-Session: https://claude.ai/code/session_019fJGoCTvP5e7n599q3ZQ42 --- ARCHITECTURE.md | 18 +++++ BENCHMARKS.md | 32 ++++++++ scripts/perf_model.py | 182 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 232 insertions(+) create mode 100755 scripts/perf_model.py diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 73c5f0f..1541fb3 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1826,3 +1826,21 @@ The last frame's detail, always, gains everything on a still picture and loses u **Consequence.** - Rust callers change: a `match` on those enums gains a wildcard arm, and a struct literal of those types becomes a default changed member by member. The recording calls take their new names, and `set_partial_share`'s result is handled. - emberwave's own crates and tests have changed with it, and so does emberwave-stream when it moves to the next release. + +### 2026-11-06 — D71: A performance model for devices we do not have + +**Context.** Measuring on a phone needs the user's phone, unattended, with the keychain unlocked, and the user will not keep supplying devices. A release should still say roughly what it costs elsewhere (#159). + +**Decision.** `scripts/perf_model.py` fits `time = fixed + per_pixel x megapixels` for each way (decode into the planes, decode into RGB, encode) on the Apple M3 Max, from the C API runs of 2026-10-26 and 2026-10-27 (release 0.11.0), and estimates other chips by scaling the per-pixel term by their memory bandwidth against the M3 Max's 400 GB/s, keeping the fixed term. BENCHMARKS.md has the result as its own section, "Estimates for other devices", apart from the measured tables and marked as estimated. + +**Why bandwidth.** On the one other device measured the same way, the iPhone 17, the per-pixel cost is 5.2 to 5.6 times the M3 Max's. The memory bandwidths, 400 and 68 GB/s, are 5.9 times apart; the GPU cores, 40 and 5, 8 times. A frame is mostly coefficients and planes moving through memory. + +**Limits, stated beside the table.** +- The model overestimates the iPhone 17 by 6 to 16 %, always above, so the table is no better than 15 %. +- Three points per line show that the lines are straight, not that they predict, and the check is one device. +- Paced decoding (60 frames a second) is not modelled: its extra 1.7 to 3.0 ms on the M3 Max and 4.6 to 5.9 ms on the iPhone 17 comes from the GPU's power states, and stays measured (#81). +- The per-pixel term carries the frame's bytes, which grow with the size in the benchmark streams. + +**Not done.** Calibrating the bandwidth term on the iPhone (a factor of 0.88 to 0.95) for other chips: it would fit one device and be an extrapolation for all the others. + +**Consequence.** No code in the library changes. A new measurement of the M3 Max or the iPhone is entered into the script's table, and the table in BENCHMARKS.md is rewritten with `scripts/perf_model.py --write BENCHMARKS.md`. A device measured later is added as a check. diff --git a/BENCHMARKS.md b/BENCHMARKS.md index 135a984..e64190f 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -349,6 +349,38 @@ Since decision D53, on Apple silicon a decode that waits for its frame writes th The GPU encoder does the same with a picture's planes since decision D54, where it used to write them into an upload buffer and copy them over (D30): 12 MB at 4K. Measured the same way on 2026-10-29 from 20:24 to 20:36, with the code of [#107](https://github.com/with-developer/emberwave/pull/107): a scratch program, not kept, encoded the scene's 20 pictures from CPU memory on the two contexts in turn, a frame each, within the budgets of the tables above; the frames were the same bytes either way. At 4K, two runs of four and six rounds of 300 frames of each: frame after frame, 1.88 to 2.00 ms in place against 1.91 to 2.03 ms, 0.02 to 0.04 ms less in every round, 0.03 ms in the median round; at 60 frames a second, 4.63 to 5.88 ms against 4.90 to 6.05 ms, 0.04 ms less in the median round, in eight rounds of ten, from 0.09 ms more to 0.28 ms less. At 1080p, one run of four rounds, the difference was within 0.01 ms frame after frame. The upload buffer, as large as the planes, is gone too. +## Estimates for other devices + +A model, not measurements. `scripts/perf_model.py` fits a frame's time on the two devices that were measured, the Apple M3 Max and the iPhone 17, as a fixed cost plus a cost per megapixel, and estimates other chips from the M3 Max's line by their memory bandwidth. It was written to say roughly what a change or a device costs without having the device (decision D71, issue #159). Run `scripts/perf_model.py` for the fits and the check, and `scripts/perf_model.py --write BENCHMARKS.md` to rewrite the table. + + +Estimated by `scripts/perf_model.py`, not measured. Milliseconds per frame, frame after frame. + +| Chip | Memory GB/s | Decode to planes, 1080p | Decode to planes, 4K | Decode to RGB, 1080p | Decode to RGB, 4K | Encode, 1080p | Encode, 4K | +|---|---|---|---|---|---|---|---| +| iPhone 17 (A19) (the check) | 68 | 1.37 | 4.81 | 1.77 | 6.26 | 2.60 | 9.01 | +| Apple M1 | 68.25 | 1.37 | 4.79 | 1.76 | 6.23 | 2.59 | 8.98 | +| Apple M2 | 100 | 1.00 | 3.34 | 1.29 | 4.34 | 1.91 | 6.27 | +| Apple M3 | 100 | 1.00 | 3.34 | 1.29 | 4.34 | 1.91 | 6.27 | +| Apple M4 | 120 | 0.87 | 2.82 | 1.12 | 3.66 | 1.67 | 5.30 | +| Apple M4 Pro | 273 | 0.51 | 1.37 | 0.64 | 1.76 | 0.99 | 2.59 | +| Apple M3 Max (40-core GPU) (fitted on) | 400 | 0.42 | 1.00 | 0.52 | 1.29 | 0.82 | 1.91 | +| Apple M4 Max (32-core GPU) | 410 | 0.41 | 0.98 | 0.52 | 1.26 | 0.81 | 1.88 | +| Apple M4 Max (40-core GPU) | 546 | 0.37 | 0.79 | 0.46 | 1.02 | 0.73 | 1.52 | + + +The table is estimated, never measured, from the means of [the C API runs of 2026-10-26 and 2026-10-27](#against-hevc-on-the-same-device) (release 0.11.0), frame after frame. The two rows named in it are the M3 Max, which the lines are fitted on, so its row is the fit and not the measurement, and the iPhone 17, which the model is checked against. Measured, the iPhone 17 takes 1.27 and 4.52 ms to decode into the planes at 1080p and 4K, 1.62 and 5.84 ms into RGB, and 2.24 and 7.89 ms to encode. The M3 Max takes 0.42 and 1.00, 0.54 and 1.30, and 0.82 and 1.91 ms. + +- **What it rests on.** Three sizes, each the middle of the runs' lowest and highest mean, per way and device: a straight line `time = fixed + per_pixel x megapixels`. The lines are within 0.04 ms of the means at every size (the worst, 4 %, is the M3 Max's decode into RGB), which shows that the lines are straight. Three points and two parameters cannot show that they predict. The per-pixel term also carries the bytes of the frame, which grow with the size in these streams, 400, 700 and 1200 kB. +- **Fixed costs are alike.** 0.22 and 0.18 ms for decoding into the planes, 0.27 and 0.21 for RGB and 0.46 and 0.43 for encoding, on the M3 Max and the iPhone 17: submitting the work and learning that it is done, which any Metal client pays ([Reading the numbers](#reading-the-numbers)). The estimates keep the M3 Max's. +- **The per-pixel cost follows the memory.** The iPhone 17's is 5.6, 5.5 and 5.2 times the M3 Max's for the three ways. The memory bandwidths, 400 and 68 GB/s, are 5.9 times apart, and the GPU cores, 40 and 5, 8 times. A frame is mostly coefficients and planes moving through memory, so the estimates scale the per-pixel term by the bandwidth. +- **Checked on one device, and a little high.** Scaling the M3 Max's per-pixel term by 5.9 puts the iPhone 17 6 to 8 % above its measurements for decoding into the planes, 7 to 9 % for RGB and 11 to 16 % for encoding, at all three sizes, always above. Read the estimates as at most that much too high for chips like the A19, and not as better than 15 % anywhere. The bandwidth of the iPhone 17 is the figure the project uses, 68 GB/s; Apple does not publish it. The M-series figures are Apple's announcements. +- **What it does not give.** + - Decoding at a stream's pace. At 60 frames a second a decode takes 4.6 to 5.9 ms more than frame after frame on the iPhone 17 and 1.7 to 3.0 ms more on the M3 Max, from the GPU's power states. It hardly follows the size and has to be measured on the device (issue #81). + - Chips with more bandwidth than the M3 Max's, as the M4 Max, are extrapolations beyond the fitted line's range, and other GPUs differ in compute, caches and drivers. The check is one comparison against one other device. + - The browser: WebGPU adds a share of its own, 0.2 to 0.7 ms on the M3 Max and 0.7 to 1.8 ms on the iPhone 17. + - Heat, battery and low-power modes. + ## Picture quality What a frame's budget buys, on a hard picture: a 4K frame of the game Crimson Desert, grass, pines and a stone wall in daylight, from a screenshot taken in the game, a 3840x2160 PNG. It was converted to 8-bit 4:2:0 in BT.709's video range, each chroma sample the mean of its four pixels. On 2026-10-29, on the Apple M3 Max of the tables above, `emberwave-cli encode` encoded it within five budgets on the CPU, which makes the same bytes on every machine, and `emberwave-cli decode` decoded it again, built from the code of release 0.11.0. The GPU encoder (`--gpu`), on 2026-10-27 and 2026-10-29, gave the same sizes and the same PSNR to two decimals. PSNR is per plane, over the whole picture, against the 4:2:0 picture that the encoder took. `scripts/quality_figure.py` does all this from the 4:2:0 picture, prints this table and draws the README's picture, which enlarges the region that loses the most at the smallest budget. The frame itself is not in the repository (decision D49). diff --git a/scripts/perf_model.py b/scripts/perf_model.py new file mode 100755 index 0000000..6d2b95e --- /dev/null +++ b/scripts/perf_model.py @@ -0,0 +1,182 @@ +#!/usr/bin/env python3 +"""A performance model for emberwave: estimates, never measurements. + + time = fixed + per_pixel * pixels + +The two terms are fitted per way (decode into planes, decode into RGB, encode) +on the Apple M3 Max, from numbers BENCHMARKS.md already holds. The iPhone 17's +numbers, measured the same way, then check the model: its per-pixel cost should +sit near the M3 Max's times the ratio of their memory bandwidths, and its fixed +cost near the M3 Max's. Estimates for other chips scale the per-pixel term by +bandwidth and keep the fixed term. + +What this cannot give: decoding at a stream's pace (the GPU's power states, see +issue #81), thermal throttling, or anything that is not memory-bound. Every +figure it prints is a model output; report it as "estimated", with this +script's name and the date of the measurements below. + +Usage: + scripts/perf_model.py # fit, check, estimates, as text + scripts/perf_model.py --markdown # the estimated table for BENCHMARKS.md + scripts/perf_model.py --write BENCHMARKS.md # put that table between the file's perf-model markers +Standard library only. +""" + +import argparse +import sys + +# Frame sizes of the benchmark streams (BENCHMARKS.md, "What is measured"). +SIZES = {"1080p": 1920 * 1080, "1440p": 2560 * 1440, "4K": 3840 * 2160} +# Bytes per frame of those streams: the per-pixel term also carries the data. +BUDGET_KB = {"1080p": 400, "1440p": 700, "4K": 1200} + +# Mean milliseconds per frame, frame after frame, through the C API in the +# benchmark app (BENCHMARKS.md, "Against HEVC on the same device"), XCFramework +# of release 0.11.0, measured 2026-10-26 and 2026-10-27. Each value is the +# lowest and the highest of the runs; the model takes their middle. The M3 Max's +# 1440p decode into RGB leaves out the run with a few slow frames (0.85 ms): +# the section's own text gives 0.69 to 0.70 ms for the others. +MEASURED = { + "Apple M3 Max": { + "decode to planes": {"1080p": (0.42, 0.43), "1440p": (0.56, 0.56), "4K": (1.00, 1.01)}, + "decode to RGB": {"1080p": (0.53, 0.56), "1440p": (0.69, 0.70), "4K": (1.29, 1.30)}, + "encode": {"1080p": (0.81, 0.82), "1440p": (1.11, 1.12), "4K": (1.90, 1.92)}, + }, + "iPhone 17": { + "decode to planes": {"1080p": (1.27, 1.27), "1440p": (2.10, 2.11), "4K": (4.51, 4.54)}, + "decode to RGB": {"1080p": (1.61, 1.62), "1440p": (2.71, 2.72), "4K": (5.83, 5.84)}, + "encode": {"1080p": (2.24, 2.24), "1440p": (3.82, 3.84), "4K": (7.88, 7.90)}, + }, +} +MEASURED_ON = "2026-10-26 and 2026-10-27, release 0.11.0" + +# Memory bandwidth in GB/s, the figure of Apple's own announcement for each chip +# (the A19's 68 GB/s is the figure the project has used since 2026-11-04; Apple +# does not publish it). Chips with several GPU configurations are listed by the +# one whose bandwidth differs. +BANDWIDTH = [ + ("iPhone 17 (A19)", 68.0, "check"), + ("Apple M1", 68.25, ""), + ("Apple M2", 100.0, ""), + ("Apple M3", 100.0, ""), + ("Apple M4", 120.0, ""), + ("Apple M4 Pro", 273.0, ""), + ("Apple M3 Max (40-core GPU)", 400.0, "fit"), + ("Apple M4 Max (32-core GPU)", 410.0, ""), + ("Apple M4 Max (40-core GPU)", 546.0, ""), +] +REFERENCE = "Apple M3 Max" +REFERENCE_BANDWIDTH = 400.0 +CHECK = "iPhone 17" +CHECK_BANDWIDTH = 68.0 + + +def middle(pair): + return (pair[0] + pair[1]) / 2 + + +def fit(points): + """Least squares for t = a + b * megapixels; returns (a, b, worst error in %).""" + xs = [SIZES[k] / 1e6 for k in points] + ys = [middle(points[k]) for k in points] + n = len(xs) + mx, my = sum(xs) / n, sum(ys) / n + b = sum((x - mx) * (y - my) for x, y in zip(xs, ys)) / sum((x - mx) ** 2 for x in xs) + a = my - b * mx + worst = max(abs(a + b * x - y) / y * 100 for x, y in zip(xs, ys)) + return a, b, worst + + +def predict(a, b, bandwidth_ratio, megapixels): + """Fixed term kept, per-pixel term scaled by how much slower the memory is.""" + return a + b * bandwidth_ratio * megapixels + + +def text_report(): + fits = {dev: {way: fit(pts) for way, pts in ways.items()} for dev, ways in MEASURED.items()} + ratio = REFERENCE_BANDWIDTH / CHECK_BANDWIDTH + print(f"Measured {MEASURED_ON}; sizes 1080p/1440p/4K = " + + "/".join(f"{SIZES[k] / 1e6:.2f}" for k in SIZES) + " Mpixel") + print("\nFit, t = fixed + per_pixel x megapixels (ms, ms per Mpixel):") + print(f"{'device':14} {'way':18} {'fixed':>6} {'per Mpx':>8} {'worst error':>12}") + for dev, ways in fits.items(): + for way, (a, b, worst) in ways.items(): + print(f"{dev:14} {way:18} {a:6.2f} {b:8.3f} {worst:11.1f}%") + print(f"\nCheck on the {CHECK} (bandwidth ratio {REFERENCE_BANDWIDTH:.0f}/{CHECK_BANDWIDTH:.0f} = {ratio:.2f}):") + print(f"{'way':18} {'per-pixel ratio':>16} {'fixed iPhone/Mac':>17} " + f"{'1080p err':>10} {'1440p err':>10} {'4K err':>8}") + for way in MEASURED[CHECK]: + a_m, b_m, _ = fits[REFERENCE][way] + a_p, b_p, _ = fits[CHECK][way] + errs = [] + for size in SIZES: + est = predict(a_m, b_m, ratio, SIZES[size] / 1e6) + got = middle(MEASURED[CHECK][way][size]) + errs.append((est - got) / got * 100) + print(f"{way:18} {b_p / b_m:16.2f} {a_p / a_m:17.2f} " + + " ".join(f"{e:+9.1f}%" for e in errs)) + print("\n(error = model minus measured, on the iPhone's own measurements)") + print("\nEstimates, ms per frame, frame after frame (1080p / 4K):") + ways = list(MEASURED[REFERENCE]) + print(f"{'chip':28} {'GB/s':>6} " + " ".join(f"{w:>18}" for w in ways)) + for chip, bw, note in BANDWIDTH: + row = [] + for way in ways: + a, b, _ = fits[REFERENCE][way] + cells = [predict(a, b, REFERENCE_BANDWIDTH / bw, SIZES[size] / 1e6) for size in ("1080p", "4K")] + row.append(f"{cells[0]:7.2f} / {cells[1]:6.2f}") + print(f"{chip:28} {bw:6.1f} " + " ".join(f"{c:>18}" for c in row) + + (f" ({note})" if note else "")) + + +def markdown_table(out_lines=None): + fits = {way: fit(pts) for way, pts in MEASURED[REFERENCE].items()} + ways = [("decode to planes", "Decode to planes"), ("decode to RGB", "Decode to RGB"), ("encode", "Encode")] + out = [] + out.append("| Chip | Memory GB/s | " + " | ".join( + f"{label}, {size}" for _, label in ways for size in ("1080p", "4K")) + " |") + out.append("|---|---|" + "---|" * (len(ways) * 2)) + for chip, bw, note in BANDWIDTH: + cells = [] + for way, _ in ways: + a, b, _w = fits[way] + for size in ("1080p", "4K"): + cells.append(f"{predict(a, b, REFERENCE_BANDWIDTH / bw, SIZES[size] / 1e6):.2f}") + tag = {"fit": " (fitted on)", "check": " (the check)"}.get(note, "") + out.append(f"| {chip}{tag} | {bw:g} | " + " | ".join(cells) + " |") + return "\n".join(out) + + +BEGIN = "" + + +def write_into(path): + """Replace what stands between the markers; the begin marker's line is kept.""" + text = open(path, encoding="utf-8").read() + head, _, rest = text.partition(BEGIN) + marker_rest, found, tail = rest.partition(END) + if not found: + sys.exit(f"{path}: no perf-model markers") + begin_line = BEGIN + marker_rest.split("\n", 1)[0] + "\n" + body = f"Estimated by `scripts/perf_model.py`, not measured. Milliseconds per frame, frame after frame.\n\n{markdown_table()}\n" + with open(path, "w", encoding="utf-8") as f: + f.write(head + begin_line + body + END + tail) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--markdown", action="store_true", help="print the estimated table for BENCHMARKS.md") + parser.add_argument("--write", metavar="FILE", help="put the table between FILE's perf-model markers") + args = parser.parse_args() + if args.write: + write_into(args.write) + elif args.markdown: + print(markdown_table()) + else: + text_report() + return 0 + + +if __name__ == "__main__": + sys.exit(main())