From b9aaa01ed1867105bdcc43f9b72340d2332f3f8e Mon Sep 17 00:00:00 2001 From: harry Date: Fri, 6 Nov 2026 18:37:37 +0900 Subject: [PATCH 1/3] bench: decode beside a display loop on the Mac Issue #81 asks how long a decode takes inside a client's display loop, where a real client draws the last picture at the screen's refresh while frames arrive at a stream's pace. The benchmark app's Mac front-end gains a -display mode: a borderless window over a screen, a CVDisplayLink and a CAMetalLayer draw the last decoded picture at the refresh, while emberwave (RGB texture, Y/Cb/Cr textures) and HEVC (NV12) decode as frames arrive. Each way is also run without the loop in the same run, so the loop's cost is a difference within one run. The report adds the time from a frame's arrival until the display presented it (the drawable's presentedTime). The stream runs at 59.5 frames a second so that frames drift across the refreshes, as a stream's do against a display; frames locked to the refresh would all show one latency. CADisplayLink stays at 60 Hz on a 120 Hz screen for an app that is not in front, so a CVDisplayLink is used. The shared benchmark code gains only a PictureSink type, a settable frame rate in FrameClock, and entry points that decode an HEVC stream paced with a sink; the iOS app is not changed. Claude-Session: https://claude.ai/code/session_019fJGoCTvP5e7n599q3ZQ42 --- .../ios/EmberwaveBench/Benchmark.swift | 24 +- .../ios/EmberwaveBench/VideoToolbox.swift | 64 +++- .../ios/EmberwaveBenchMac/DisplayLoop.swift | 353 ++++++++++++++++++ .../ios/EmberwaveBenchMac/DisplayRun.swift | 267 +++++++++++++ .../examples/ios/EmberwaveBenchMac/main.swift | 24 +- scripts/bench_app_on_mac.sh | 11 +- 6 files changed, 736 insertions(+), 7 deletions(-) create mode 100644 crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/DisplayLoop.swift create mode 100644 crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/DisplayRun.swift diff --git a/crates/emberwave-ffi/examples/ios/EmberwaveBench/Benchmark.swift b/crates/emberwave-ffi/examples/ios/EmberwaveBench/Benchmark.swift index 84f4dd9..61b9d6b 100644 --- a/crates/emberwave-ffi/examples/ios/EmberwaveBench/Benchmark.swift +++ b/crates/emberwave-ffi/examples/ios/EmberwaveBench/Benchmark.swift @@ -53,6 +53,23 @@ struct Stats: CustomStringConvertible { } } +/// How a decoded picture sits in Metal textures. +enum PictureLayout { + /// One texture of colours. + case rgb + /// Y, Cb and Cr, a texture each: emberwave's plane textures. + case planes + /// Y, and Cb and Cr interleaved: VideoToolbox's NV12. + case nv12 +} + +/// Takes a picture the moment its decode is done: when its frame arrived on +/// `now()`'s clock, its textures, and whether the frame is one of those +/// timed, not the warm-up's. A display loop draws what it is given. +typealias PictureSink = ( + _ arrived: Double, _ layout: PictureLayout, _ textures: [MTLTexture], _ timed: Bool +) -> Void + /// A picture size and the bytes a frame of it gets: those of the native /// benchmarks (BENCHMARKS.md) and of the browser's benchmark page. struct PictureSize { @@ -97,13 +114,18 @@ struct FrameClock { private var due: Double? private static let timers = DispatchQueue(label: "frame clock", qos: .userInteractive) static let spins = ProcessInfo.processInfo.environment["EMBERWAVE_BENCH_SPIN"] == "1" + /// Frames a second. A display-loop run detunes it a little from the + /// stream's 60, so that the frames drift across the display's refreshes + /// as a stream's do, where frames locked to the refresh would all have the + /// same latency, one of many. + static var rate = Double(streamFrameRate) mutating func wait() { guard let previous = due else { due = now() return } - let next = max(previous + 1000 / Double(streamFrameRate), now()) + let next = max(previous + 1000 / FrameClock.rate, now()) due = next let delay = next - now() guard delay > 0 else { diff --git a/crates/emberwave-ffi/examples/ios/EmberwaveBench/VideoToolbox.swift b/crates/emberwave-ffi/examples/ios/EmberwaveBench/VideoToolbox.swift index da7e247..de55bc8 100644 --- a/crates/emberwave-ffi/examples/ios/EmberwaveBench/VideoToolbox.swift +++ b/crates/emberwave-ffi/examples/ios/EmberwaveBench/VideoToolbox.swift @@ -266,6 +266,51 @@ private func timeHEVC( pacedBGRA: pacedBGRA) } +/// An HEVC stream of `pictures`, ready to decode as often as wanted: a keyframe +/// and then the frames that depend on it, as `timeHEVC` makes them, and the +/// parameter sets to decode them. +struct HEVCStream { + let frames: [[UInt8]] + let format: CMVideoFormatDescription +} + +/// Encodes the stream of `pictures` at `rate` for `hevcPaced`. +func makeHEVCStream( + _ size: PictureSize, _ rate: HEVCRate, _ pictures: [[[UInt8]]], _ plan: Plan +) throws -> HEVCStream { + let encoder = try settingUp { try HEVCEncoder(size, rate, pictures) } + defer { encoder.invalidate() } + var warmed = 0 + let warm = now() + while warmed < plan.warmupFrames || now() - warm < plan.warmupMs { + _ = try encoder.encode() + warmed += 1 + } + var frames = [try encoder.encode(keyframe: true).bytes] + for _ in 0.. (stats: Stats, hardware: Bool?) { + let decoder = try settingUp { + try HEVCDecoder(stream.format, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, device) + } + defer { decoder.invalidate() } + let stats = try decoder.time(stream.frames, plan, paced: true, sink: sink) + return (stats, decoder.hardware) +} + /// The value of a VideoToolbox session's property, `nil` if it has none. private func property(_ session: VTSession, _ key: CFString) -> CFTypeRef? { var value: CFTypeRef? @@ -562,7 +607,7 @@ private final class HEVCDecoder { private let format: CMVideoFormatDescription private let pixelFormat: OSType private let textureCache: CVMetalTextureCache - private var shown: [CVMetalTexture] = [] + private(set) var shown: [CVMetalTexture] = [] init(_ format: CMVideoFormatDescription, _ pixelFormat: OSType, _ device: MTLDevice) throws { self.format = format @@ -711,12 +756,23 @@ private final class HEVCDecoder { return textures } + /// Hands the picture just decoded to `sink`, in the textures it was wrapped in. + private func offer(_ sink: PictureSink?, _ arrived: Double, timed: Bool) { + guard let sink else { + return + } + let textures = shown.compactMap { CVMetalTextureGetTexture($0) } + sink(arrived, pixelFormat == kCVPixelFormatType_32BGRA ? .rgb : .nv12, textures, timed) + } + /// Times the decoding of `frames`, a keyframe and the frames after it: a /// warm-up over them in order, untimed, then the keyframe, untimed, which /// the decoder can take after any frame, and the others, each timed; one /// after the other, or with `paced` as a stream's frames arrive, the /// warm-up too. - func time(_ frames: [[UInt8]], _ plan: Plan, paced: Bool = false) throws -> Stats { + func time(_ frames: [[UInt8]], _ plan: Plan, paced: Bool = false, sink: PictureSink? = nil) + throws -> Stats + { var clock = FrameClock() let warm = now() var index = 0 @@ -724,7 +780,9 @@ private final class HEVCDecoder { if paced { clock.wait() } + let arrived = now() _ = try decode(frames[index % frames.count]) + offer(sink, arrived, timed: false) index += 1 } if paced { @@ -737,7 +795,9 @@ private final class HEVCDecoder { if paced { clock.wait() } + let arrived = now() let (picture, time) = try decode(frame) + offer(sink, arrived, timed: true) times.append(time) last.insert(picture, at: 0) if last.count > 4 { diff --git a/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/DisplayLoop.swift b/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/DisplayLoop.swift new file mode 100644 index 0000000..98a3e06 --- /dev/null +++ b/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/DisplayLoop.swift @@ -0,0 +1,353 @@ +// A display loop for the Mac's measurements (issue #81): a borderless window +// over one screen that draws the last decoded picture, scaled to the screen, +// at the screen's refresh, through a CAMetalLayer and a CVDisplayLink, as a +// client's player does. (A CADisplayLink of an app that is not in front stays +// at 60 Hz on a 120 Hz screen; a CVDisplayLink follows the screen.) A paced decode runs beside it, and the loop notes when +// each frame first reaches the screen: the time from the frame's arrival until +// the display shows it. + +import AppKit +import CoreVideo +import Foundation +import Metal +import QuartzCore + +// Draws a picture over the whole screen: one triangle, and a fragment shader +// for each layout. The colours are BT.709, video range, as the pictures' are; +// whether they are exact does not matter to the timing. +private let shaderSource = """ + #include + using namespace metal; + + struct Vertex { + float4 position [[position]]; + float2 uv; + }; + + vertex Vertex vertex_main(uint id [[vertex_id]]) { + float2 corner = float2((id << 1) & 2, id & 2); + Vertex out; + out.position = float4(corner * 2.0 - 1.0, 0.0, 1.0); + out.uv = float2(corner.x, 1.0 - corner.y); + return out; + } + + constexpr sampler linearSampler(filter::linear, address::clamp_to_edge); + + float3 bt709(float y, float cb, float cr) { + float luma = (y * 255.0 - 16.0) / 219.0; + float u = (cb * 255.0 - 128.0) / 224.0; + float v = (cr * 255.0 - 128.0) / 224.0; + return float3(luma + 1.5748 * v, luma - 0.1873 * u - 0.4681 * v, luma + 1.8556 * u); + } + + fragment float4 rgb_main(Vertex in [[stage_in]], texture2d color [[texture(0)]]) { + return float4(color.sample(linearSampler, in.uv).rgb, 1.0); + } + + fragment float4 planes_main( + Vertex in [[stage_in]], texture2d y [[texture(0)]], + texture2d cb [[texture(1)]], texture2d cr [[texture(2)]] + ) { + return float4(bt709(y.sample(linearSampler, in.uv).r, + cb.sample(linearSampler, in.uv).r, + cr.sample(linearSampler, in.uv).r), 1.0); + } + + fragment float4 nv12_main( + Vertex in [[stage_in]], texture2d y [[texture(0)]], texture2d cbcr [[texture(1)]] + ) { + float2 chroma = cbcr.sample(linearSampler, in.uv).rg; + return float4(bt709(y.sample(linearSampler, in.uv).r, chroma.x, chroma.y), 1.0); + } + """ + +/// What a display loop saw during one run. +struct DisplayOutcome { + /// Milliseconds from a frame's arrival until the display presented it, for + /// the frames that were drawn at all, those timed only. + let toScreen: [Double] + /// Frames offered by the decode, those timed only, and how many of them the + /// loop never drew because a newer one had come before its next refresh. + let offered: Int + let skipped: Int + /// Refreshes drawn; the time between the display link's calls, the median + /// and the number of calls that came more than half an interval later, + /// a sign of a loop that cannot keep up. + let refreshes: Int + let interval: Double + let late: Int + /// Calls of the display link, and how many of them found no picture yet or + /// no drawable: what to look at when nothing was drawn. + let ticks: Int + let noPicture: Int + let noDrawable: Int +} + +final class DisplayLoop: NSObject { + /// The screen the loop runs on, and its refresh rate in the mode it is in. + let screenName: String + let maximumRate: Int + let screen: NSScreen + + private let device: MTLDevice + private let queue: MTLCommandQueue + private let window: NSWindow + private let view: NSView + private let layer = CAMetalLayer() + private var pipelines: [PictureLayout: MTLRenderPipelineState] = [:] + private var link: CVDisplayLink? + private let displayID: CGDirectDisplayID + private var divider = 1 + private var counter = 0 + + private struct Offered { + var sequence: Int + var arrived: Double + var layout: PictureLayout + var textures: [MTLTexture] + var timed: Bool + } + + // The decode thread offers pictures, the main thread draws them and + // Metal's threads report when they were presented. + private let lock = NSLock() + private var latest: Offered? + private var sequence = 0 + private var drawn = -1 + private var offeredTimed = 0 + private var skipped = 0 + private var refreshes = 0 + private var intervals: [Double] = [] + private var ticks = 0 + private var noPicture = 0 + private var noDrawable = 0 + private var toScreen: [Double] = [] + private var lastTick = 0.0 + + /// The screens, the one with the highest refresh rate first. + static func screens() -> [NSScreen] { + NSScreen.screens.sorted { $0.maximumFramesPerSecond > $1.maximumFramesPerSecond } + } + + /// Makes the window on `screen`, which is not shown until `start`. Call on + /// the main thread. + init(device: MTLDevice, screen: NSScreen) throws { + self.screen = screen + displayID = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? CGDirectDisplayID ?? 0 + guard let queue = device.makeCommandQueue() else { + throw EmberwaveFailure(description: "no command queue") + } + self.device = device + self.queue = queue + screenName = screen.localizedName + maximumRate = screen.maximumFramesPerSecond + + // In global coordinates: a rectangle given with a `screen` would be + // relative to it, which puts a screen that is not the main one's + // window off every screen. + window = NSWindow( + contentRect: screen.frame, styleMask: .borderless, backing: .buffered, defer: false) + // Over the menu bar and the Dock, as a full-screen client is, and + // out of the way of the mouse. + window.level = .screenSaver + window.isOpaque = true + window.backgroundColor = .black + window.ignoresMouseEvents = true + window.hasShadow = false + view = NSView(frame: NSRect(origin: .zero, size: screen.frame.size)) + layer.device = device + layer.pixelFormat = .bgra8Unorm + // Two drawables, as a client that minds latency has: the default three let a frame wait a refresh longer in the queue. + layer.maximumDrawableCount = 2 + layer.framebufferOnly = true + layer.frame = view.bounds + layer.contentsScale = screen.backingScaleFactor + layer.drawableSize = CGSize( + width: screen.frame.width * screen.backingScaleFactor, + height: screen.frame.height * screen.backingScaleFactor) + view.wantsLayer = true + view.layer = layer + window.contentView = view + super.init() + + let library: MTLLibrary + do { + library = try device.makeLibrary(source: shaderSource, options: nil) + } catch { + throw EmberwaveFailure(description: "the display shaders: \(error)") + } + for (layout, fragment) in [ + (PictureLayout.rgb, "rgb_main"), (.planes, "planes_main"), (.nv12, "nv12_main"), + ] { + let descriptor = MTLRenderPipelineDescriptor() + descriptor.vertexFunction = library.makeFunction(name: "vertex_main") + descriptor.fragmentFunction = library.makeFunction(name: fragment) + descriptor.colorAttachments[0].pixelFormat = .bgra8Unorm + pipelines[layout] = try device.makeRenderPipelineState(descriptor: descriptor) + } + } + + /// Whether the window is on the screen now: a screen that is off, or showing + /// another input, never takes it. + var isOnScreen: Bool { window.occlusionState.contains(.visible) } + + /// The picture size the screen is drawn at, for the report. + var drawableSize: CGSize { layer.drawableSize } + + /// Takes a picture from the decode, on any thread. + func offer( + _ arrived: Double, _ layout: PictureLayout, _ textures: [MTLTexture], _ timed: Bool + ) { + lock.lock() + sequence += 1 + latest = Offered( + sequence: sequence, arrived: arrived, layout: layout, textures: textures, timed: timed) + if timed { + offeredTimed += 1 + } + lock.unlock() + } + + /// Shows the window and draws at `rate` frames a second, or the screen's + /// refresh if that is lower, until `stop`: on a screen of a higher refresh + /// than `rate`, every second or third refresh and so on. Returns the rate + /// drawn at. Call on the main thread. + func start(rate: Int) -> Int { + lock.lock() + latest = nil + sequence = 0 + drawn = -1 + offeredTimed = 0 + skipped = 0 + refreshes = 0 + intervals = [] + ticks = 0 + noPicture = 0 + noDrawable = 0 + toScreen = [] + lastTick = 0 + lock.unlock() + divider = max(1, Int((Double(maximumRate) / Double(rate)).rounded())) + counter = 0 + NSApp.activate() + window.orderFrontRegardless() + // The window server takes a moment to put it on the screen. + let shown = Date(timeIntervalSinceNow: 2) + while !window.occlusionState.contains(.visible), Date() < shown { + RunLoop.main.run(until: Date(timeIntervalSinceNow: 0.05)) + } + var link: CVDisplayLink? + CVDisplayLinkCreateWithCGDisplay(displayID, &link) + if let link { + // On Core Video's thread; the window itself stays the main thread's. + CVDisplayLinkSetOutputHandler(link) { [unowned self] _, _, _, _, _ in + self.counter += 1 + if self.counter % self.divider == 0 { + self.tick() + } + return kCVReturnSuccess + } + CVDisplayLinkStart(link) + } + self.link = link + return maximumRate / divider + } + + /// Stops drawing, hides the window and says what the loop saw, once the + /// last pictures have been presented. Call on the main thread. + func stop() -> DisplayOutcome { + if let link { + CVDisplayLinkStop(link) + } + link = nil + // The last handlers run on Metal's threads a refresh or two later. + RunLoop.main.run(until: Date(timeIntervalSinceNow: 0.2)) + window.orderOut(nil) + lock.lock() + defer { lock.unlock() } + latest = nil + let median = intervals.sorted()[safe: intervals.count / 2] ?? 0 + let late = intervals.filter { $0 > median * 1.5 }.count + return DisplayOutcome( + toScreen: toScreen, offered: offeredTimed, skipped: skipped, refreshes: refreshes, + interval: median, late: late, ticks: ticks, noPicture: noPicture, + noDrawable: noDrawable) + } + + private func tick() { + let began = now() + lock.lock() + let frame = latest + let previousTick = lastTick + lastTick = began + lock.unlock() + lock.lock() + ticks += 1 + if previousTick > 0 { + intervals.append(began - previousTick) + } + lock.unlock() + guard let frame else { + lock.lock() + noPicture += 1 + lock.unlock() + return + } + guard let drawable = layer.nextDrawable(), let buffer = queue.makeCommandBuffer(), + let pipeline = pipelines[frame.layout] + else { + lock.lock() + noDrawable += 1 + lock.unlock() + return + } + let pass = MTLRenderPassDescriptor() + pass.colorAttachments[0].texture = drawable.texture + pass.colorAttachments[0].loadAction = .dontCare + pass.colorAttachments[0].storeAction = .store + guard let encoder = buffer.makeRenderCommandEncoder(descriptor: pass) else { + return + } + encoder.setRenderPipelineState(pipeline) + for (index, texture) in frame.textures.enumerated() { + encoder.setFragmentTexture(texture, index: index) + } + encoder.drawPrimitives(type: .triangle, vertexStart: 0, vertexCount: 3) + encoder.endEncoding() + + // A picture's first refresh is the one whose presentation is its way + // to the screen. + let fresh = frame.sequence != drawn + lock.lock() + refreshes += 1 + if fresh { + if drawn >= 0, frame.timed { + skipped += frame.sequence - drawn - 1 + } + drawn = frame.sequence + } + lock.unlock() + if fresh, frame.timed { + let arrived = frame.arrived + drawable.addPresentedHandler { [weak self] drawable in + // 0 for a drawable that was never presented. + guard drawable.presentedTime > 0, let self else { + return + } + let latency = drawable.presentedTime * 1000 - arrived + self.lock.lock() + self.toScreen.append(latency) + self.lock.unlock() + } + } + buffer.present(drawable) + buffer.commit() + } +} + +extension Array { + subscript(safe index: Int) -> Element? { + indices.contains(index) ? self[index] : nil + } +} diff --git a/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/DisplayRun.swift b/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/DisplayRun.swift new file mode 100644 index 0000000..3bd785f --- /dev/null +++ b/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/DisplayRun.swift @@ -0,0 +1,267 @@ +// The Mac's display-loop measurement (issue #81, `-display`): decodes at a +// stream's pace, 60 frames a second, as in the report's other paced rows, but +// with a display loop (DisplayLoop.swift) drawing the last picture over a +// screen at the screen's refresh, 60 and 120 Hz where the screen has both. Each +// way is also run without the loop, in the same run, so that the loop's cost is +// the difference and not a comparison with another day. Besides the time to +// decode, it reports the time from the frame's arrival until the display +// presented it. + +import AppKit +import Emberwave +import Foundation +import Metal + +extension Emberwave { + /// Encodes `pictures`, of `size`, on the GPU within its budget: the frames + /// of a stream. + func encodeFrames(_ size: PictureSize, _ pictures: [[[UInt8]]]) throws -> [[UInt8]] { + var encoder: OpaquePointer? + try check( + emberwave_encoder_create( + context, UInt32(size.width), UInt32(size.height), UInt32(EMBERWAVE_CHROMA_420), nil, + &encoder), + "emberwave_encoder_create") + defer { emberwave_encoder_destroy(encoder) } + var output = [UInt8](repeating: 0, count: size.budget) + return try pictures.map { picture in + var written = 0 + try picture[0].withUnsafeBufferPointer { y in + try picture[1].withUnsafeBufferPointer { cb in + try picture[2].withUnsafeBufferPointer { cr in + try output.withUnsafeMutableBufferPointer { frame in + try check( + emberwave_encoder_encode( + encoder, + EmberwaveSourcePlane(data: y.baseAddress, stride: size.width, size: y.count), + EmberwaveSourcePlane( + data: cb.baseAddress, stride: size.width / 2, size: cb.count), + EmberwaveSourcePlane( + data: cr.baseAddress, stride: size.width / 2, size: cr.count), + frame.baseAddress, size.budget, &written), + "emberwave_encoder_encode") + } + } + } + } + return Array(output[.. Stats { + guard let device else { + throw EmberwaveFailure(description: "no Metal device") + } + var decoder: OpaquePointer? + try check( + emberwave_decoder_create( + context, UInt32(size.width), UInt32(size.height), UInt32(EMBERWAVE_CHROMA_420), + &decoder), + "emberwave_decoder_create") + defer { emberwave_decoder_destroy(decoder) } + + var owned: [OpaquePointer?] = [] + defer { + for texture in owned { + emberwave_texture_destroy(texture) + } + } + var sets: [[OpaquePointer?]] = [] + var metal: [[MTLTexture]] = [] + for _ in 0..<3 { + var set: [OpaquePointer?] = [] + let planes = layout == .rgb ? [nil] : [EMBERWAVE_PLANE_Y, EMBERWAVE_PLANE_CB, EMBERWAVE_PLANE_CR] + for plane in planes { + var texture: OpaquePointer? + if let plane { + try check( + emberwave_texture_create_plane(decoder, UInt32(plane), &texture), + "emberwave_texture_create_plane") + } else { + try check( + emberwave_texture_create(decoder, UInt32(EMBERWAVE_TEXTURE_BGRA8), &texture), + "emberwave_texture_create") + } + owned.append(texture) + set.append(texture) + } + sets.append(set) + metal.append(try set.map { try metalTexture($0, on: device, "display texture") }) + } + + func step(_ index: Int, timed: Bool) throws -> Double { + let start = now() + emberwave_decoder_clear(decoder) + try frames[index % frames.count].withUnsafeBufferPointer { + try check( + emberwave_decoder_push_packet(decoder, $0.baseAddress, $0.count), + "emberwave_decoder_push_packet") + } + let set = sets[index % sets.count] + if layout == .rgb { + try check( + emberwave_decoder_decode_texture(decoder, nil, set[0]), + "emberwave_decoder_decode_texture") + } else { + try check( + emberwave_decoder_decode_plane_textures(decoder, set[0], set[1], set[2]), + "emberwave_decoder_decode_plane_textures") + } + let end = now() + sink?(start, layout, metal[index % metal.count], timed) + return end - start + } + + var clock = FrameClock() + let warm = now() + var index = 0 + while index < plan.warmupFrames || now() - warm < plan.warmupMs { + clock.wait() + _ = try step(index, timed: false) + index += 1 + } + var times: [Double] = [] + for offset in 0.. Stats +} + +/// Runs the whole measurement and returns its report. Call off the main +/// thread, which has to run the application's loop. +func runDisplayBenchmark(_ plan: Plan, progress: (String) -> Void) throws -> String { + FrameClock.rate = displayStreamRate + let emberwave = try Emberwave() + guard let device = emberwave.device else { + throw EmberwaveFailure(description: "no Metal device") + } + // A loop on each kind of screen: the screens' refresh rates, 60 Hz + // and 120 Hz, which the Mac may have more than one of. + let screens = DispatchQueue.main.sync { DisplayLoop.screens() } + var loops: [(rate: Int, loop: DisplayLoop, how: String)] = [] + let isBuiltin = { (screen: NSScreen) -> Bool in + let id = screen.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? CGDirectDisplayID ?? 0 + return CGDisplayIsBuiltin(id) != 0 + } + for rate in [60, 120] { + let candidates = screens.filter { $0.maximumFramesPerSecond >= rate } + // The slowest such screen, the built-in one of equals. + guard + let screen = candidates.min(by: { + ($0.maximumFramesPerSecond, isBuiltin($0) ? 0 : 1) + < ($1.maximumFramesPerSecond, isBuiltin($1) ? 0 : 1) + }) + else { + continue + } + var loop = try DispatchQueue.main.sync { try DisplayLoop(device: device, screen: screen) } + var how = "\(loop.screenName) at \(loop.maximumRate) Hz" + // A screen that takes no window, off or on another input, is left for + // every n-th refresh of a faster one. + let takes = DispatchQueue.main.sync { () -> Bool in + _ = loop.start(rate: rate) + defer { _ = loop.stop() } + return loop.isOnScreen + } + if !takes, let faster = screens.first(where: { $0.maximumFramesPerSecond > rate && isBuiltin($0) }) { + loop = try DispatchQueue.main.sync { try DisplayLoop(device: device, screen: faster) } + how = "every \(loop.maximumRate / rate)nd refresh of \(loop.screenName) at \(loop.maximumRate) Hz" + } + loops.append((rate, loop, how)) + } + var lines = [ + "emberwave display-loop benchmark, \(ISO8601DateFormatter().string(from: Date()))", + "Device: \(deviceDescription()), \(ProcessInfo.processInfo.operatingSystemVersionString)", + "GPU: \(emberwave.adapter); emberwave \(String(cString: emberwave_version()))", + "Screens: " + loops.map { + "\($0.rate) Hz: \($0.how), drawn at " + + "\(Int($0.loop.drawableSize.width))x\(Int($0.loop.drawableSize.height))" + }.joined(separator: "; ") + + "; Low Power Mode: \(ProcessInfo.processInfo.isLowPowerModeEnabled ? "on" : "off")", + "\(plan.frames) frames after at least \(plan.warmupFrames) and \(Int(plan.warmupMs)) ms of warm-up, " + + "\(plan.pictures) pictures in turn, arriving \(displayStreamRate) a second, a little under a stream's " + + "\(streamFrameRate) so that they drift across the refreshes; HEVC at Moonlight's rate", + "Per frame (ms): mean / median / 99th percentile. decode: from the frame's arrival until " + + "its picture is in textures; to screen: from the frame's arrival until the display " + + "presented it (presentedTime of its drawable, the first refresh that drew it). " + + "skipped: timed frames no refresh drew, because a newer one came first. " + + "refreshes: the median time between the display link's calls, and how many calls came " + + "more than half an interval late.", + ] + for size in pictureSizes { + progress("\(size.name): making \(plan.pictures) test pictures") + let pictures = makePictures(size, plan) + let frames = try emberwave.encodeFrames(size, pictures) + let rate = hevcRates(size)[0] + progress("\(size.name): encoding HEVC at \(rate.bitsPerSecond / 1_000_000) Mbit/s") + let stream = try? makeHEVCStream(size, rate, pictures, plan) + if stream == nil { + lines.append("\(size.name): HEVC not measured, VideoToolbox could not make the stream") + } + var ways = [ + Way(name: "emberwave into an RGB texture") { + try emberwave.pacedDecode(size, plan, frames, .rgb, sink: $0) + }, + Way(name: "emberwave into Y, Cb, Cr textures") { + try emberwave.pacedDecode(size, plan, frames, .planes, sink: $0) + }, + ] + if let stream { + ways.append( + Way(name: "HEVC into NV12 textures") { + try hevcPaced(stream, plan, device: device, sink: $0).stats + }) + } + lines.append("\(size.name), thermal state \(thermalState()):") + for refresh in [nil] + loops.map({ Optional($0.rate) }) { + lines.append(refresh.map { " display loop at \($0) Hz:" } ?? " no display loop:") + for way in ways { + progress("\(size.name): \(way.name), \(refresh.map { "display \($0) Hz" } ?? "no display")") + guard let refresh, let loop = loops.first(where: { $0.rate == refresh })?.loop else { + lines.append(row(way.name, "decode \(try way.run(nil))", indent: 4)) + continue + } + _ = DispatchQueue.main.sync { loop.start(rate: refresh) } + let decode: Stats + do { + decode = try way.run(loop.offer) + } catch { + _ = DispatchQueue.main.sync { loop.stop() } + throw error + } + let seen = DispatchQueue.main.sync { loop.stop() } + var text = "decode \(decode)" + if seen.toScreen.isEmpty { + text += "; nothing reached the screen" + } else { + text += "; to screen \(Stats(seen.toScreen, total: seen.toScreen.reduce(0, +)))" + } + text += "; skipped \(seen.skipped) of \(seen.offered); refreshes every " + + String(format: "%.2f ms (%.0f Hz), late %d of %d", seen.interval, 1000 / max(seen.interval, 0.001), seen.late, seen.refreshes) + text += "; ticks \(seen.ticks), no picture \(seen.noPicture), no drawable \(seen.noDrawable)" + lines.append(row(way.name, text, indent: 4)) + } + } + } + return lines.joined(separator: "\n") +} diff --git a/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/main.swift b/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/main.swift index 9eacd37..d9e1295 100644 --- a/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/main.swift +++ b/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/main.swift @@ -5,9 +5,31 @@ // scripts/bench_app_on_mac.sh builds and runs it. With -autorun, it makes the // short run of CI, and exits with status 1 if anything failed. +import AppKit import Foundation -let plan = CommandLine.arguments.contains("-autorun") ? Plan.short : Plan() +let plan = CommandLine.arguments.contains("-autorun") || CommandLine.arguments.contains("-short") ? Plan.short : Plan() +// -display: decodes beside a display loop over a screen (issue #81), which +// needs the application's run loop on the main thread, so the work goes to +// another one. It covers a screen for a few minutes. +if CommandLine.arguments.contains("-display") { + let application = NSApplication.shared + application.setActivationPolicy(.regular) + Thread.detachNewThread { + do { + let report = try runDisplayBenchmark(plan) { + FileHandle.standardError.write(Data(($0 + "\n").utf8)) + } + print(report) + exit(0) + } catch { + print("failed: \(error)") + exit(1) + } + } + application.run() +} + do { let emberwave = try Emberwave() var results: [SizeResult] = [] diff --git a/scripts/bench_app_on_mac.sh b/scripts/bench_app_on_mac.sh index e83874e..8d2d701 100755 --- a/scripts/bench_app_on_mac.sh +++ b/scripts/bench_app_on_mac.sh @@ -6,10 +6,14 @@ # report of a phone's run (see BENCHMARKS.md). It builds the front-end, # examples/ios/EmberwaveBenchMac, with swiftc into target/bench-app-mac. # -# Usage: scripts/bench_app_on_mac.sh [-autorun] +# Usage: scripts/bench_app_on_mac.sh [-autorun | -display] # # -autorun the short run of CI: every size briefly, and the three texture # checks; fails unless the report is complete and they pass +# -display instead of the report above, decodes at 60 frames a second beside +# a display loop (issue #81): a borderless window covers the screen +# with the highest refresh rate while it runs, a few minutes, and +# draws the last picture at 60 and 120 Hz. Leave the Mac alone. # # Needs macOS with Xcode, and the XCFramework with its library for the Mac, # which scripts/build_xcframework.sh makes. A full run takes a few minutes: @@ -28,15 +32,16 @@ fi mkdir -p "$BUILD" # The system libraries are words for the shell to split. # shellcheck disable=SC2046 -xcrun --sdk macosx swiftc -O -swift-version 5 -target arm64-apple-macos13.0 \ +xcrun --sdk macosx swiftc -O -swift-version 5 -target arm64-apple-macos14.0 \ -I "$SLICE/Headers/Emberwave" \ "$APP/EmberwaveBench/Benchmark.swift" "$APP/EmberwaveBench/Scene.swift" \ "$APP/EmberwaveBench/VideoToolbox.swift" "$APP/EmberwaveBenchMac/main.swift" \ + "$APP/EmberwaveBenchMac/DisplayLoop.swift" "$APP/EmberwaveBenchMac/DisplayRun.swift" \ "$SLICE/libemberwave_ffi.a" $(cat target/xcframework/native-static-libs-macos-arm64.txt) \ -o "$BUILD/emberwave-bench" if [ "${1:-}" != -autorun ]; then - exec "$BUILD/emberwave-bench" + exec "$BUILD/emberwave-bench" "$@" fi "$BUILD/emberwave-bench" -autorun | tee "$BUILD/autorun.log" for check in texture plane queue; do From 7a33540a685383aff8f7660c04f814bbd33f6ace Mon Sep 17 00:00:00 2001 From: harry Date: Fri, 6 Nov 2026 18:38:53 +0900 Subject: [PATCH 2/3] docs: the Mac's decodes beside a display loop (#81, D72) BENCHMARKS.md gets the measurements of 2026-11-06: decode times with and without a display loop at 60 and 120 Hz, and the time to the screen, for emberwave and HEVC. D72 records how the loop is built and why, and what was not done (the iPhone half, a real client's pipeline). The READMEs' roadmap notes the Mac's half of #81 as done. Claude-Session: https://claude.ai/code/session_019fJGoCTvP5e7n599q3ZQ42 --- ARCHITECTURE.md | 18 ++++++++++ BENCHMARKS.md | 38 +++++++++++++++++++-- README.ko.md | 2 +- README.md | 2 +- crates/emberwave-ffi/examples/ios/README.md | 2 +- 5 files changed, 57 insertions(+), 5 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 1541fb3..6fe7807 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1844,3 +1844,21 @@ The last frame's detail, always, gains everything on a still picture and loses u **Not done.** Calibrating the bandwidth term on the iPhone (a factor of 0.88 to 0.95) for other chips: it would fit one device and be an extrapolation for all the others. **Consequence.** No code in the library changes. A new measurement of the M3 Max or the iPhone is entered into the script's table, and the table in BENCHMARKS.md is rewritten with `scripts/perf_model.py --write BENCHMARKS.md`. A device measured later is added as a check. + +### 2026-11-06 — D72: Timing decodes beside a display loop on the Mac + +**Context.** The paced rows of BENCHMARKS.md time a decode with the GPU idle between frames. A client draws at every refresh instead, and its decodes share the GPU with that. Issue #81 asked what a decode then takes, for emberwave and for HEVC, and the earlier probes (a second queue blitting a texture, a kick ahead of each frame, a raised QoS) had not settled it. + +**Decision.** The benchmark app's Mac front-end gets a `-display` mode (`scripts/bench_app_on_mac.sh -display`), in `EmberwaveBenchMac/DisplayLoop.swift` and `DisplayRun.swift`. A borderless window covers one screen while it runs; a `CVDisplayLink` and a `CAMetalLayer` with two drawables draw the last decoded picture at the screen's refresh. The decode runs on another thread at a stream's pace. Every way (emberwave into Y, Cb and Cr textures and into RGB, HEVC at Moonlight's rate into NV12) runs without the loop and then with a 60 Hz and a 120 Hz loop in the same run, so the loop's cost is a difference within one run. The report adds the time from a frame's arrival to the display's presentation of it. + +**Why these choices.** +- A `CVDisplayLink`, where a `CADisplayLink` of an app that is not in front stayed at 60 calls a second on the 120 Hz screen whatever frame rate range it asked for (62.5 a second in a minimal program, against 124 for the `CVDisplayLink`). +- The stream arrives at 59.5 frames a second, not 60, so that frames drift across the refreshes. At 60 every frame had the same phase against the display, and the latency of a run was one number between 12 and 25 ms, set by where the run happened to start. +- The 60 Hz loop draws every second refresh of the 120 Hz screen: the Mac's 60 Hz external monitor took no window (it never reported the window on screen), so the run falls back to the built-in screen when a screen does not take the window. +- Two drawables, as a client that minds latency has. + +**Result.** Into the planes, emberwave decoded as quickly as or up to 0.8 ms quicker than HEVC into NV12 in the loop (0 to 0.3 ms at 1080p, 0.5 to 0.8 ms at 4K) (BENCHMARKS.md, "In a display loop, on the Mac"). The loop cost emberwave 0.2 to 0.5 ms and HEVC nothing, and did not keep the GPU's clock up. The time to the screen was 17 to 21 ms for all ways and does not tell them apart. + +**Not done.** The iPhone half of #81, which needs the phone. A real client's pipeline, with its other threads and a heavier draw. HEVC into BGRA in the loop. + +**Consequence.** The shared benchmark code gains a `PictureSink` type, a settable `FrameClock.rate`, and entry points that decode a prepared HEVC stream paced with a sink; the iOS app's behaviour is unchanged. The script's deployment target is macOS 14. diff --git a/BENCHMARKS.md b/BENCHMARKS.md index e64190f..ce28246 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -245,7 +245,7 @@ Two more runs with `EMBERWAVE_BENCH_SPIN=1`, at 60 frames a second, the CPU awak - at FP32: 1.47 to 1.48 ms against 0.38 to 0.39 ms at 1080p, and 2.31 to 2.63 ms against 0.82 ms at 4K. That is 2.8 to 3.9 times as long, and its GPU time grows 2.1 to 3.2 times. The slowdown belongs to wavelet decoding on this GPU at this cadence, not to emberwave. On the same streams at that cadence, emberwave's decodes into textures still took 1.3 to 1.5 times as long as upstream's at FP32, more than frame after frame (issue #86). Decision D48 in [`ARCHITECTURE.md`](ARCHITECTURE.md) splits that time up: about 1 ms of it is Metal's own at that cadence, which any decoder pays. -- **What a client sees.** A client draws every display refresh, and its network threads wake between frames, so it lies between these cadences. On this Mac, emberwave's decoding is quicker than HEVC's only when the GPU is kept busy. Issue #81 is to time decodes within a client's display loop. +- **What a client sees.** A client draws every display refresh, and its network threads wake between frames, so it lies between these cadences. On this Mac, emberwave's decoding is quicker than HEVC's only when the GPU is kept busy. [In a display loop](#in-a-display-loop-on-the-mac) times that on this Mac. - **HEVC's decoding grows with the bytes.** At emberwave's bytes per frame it took 2.4 to 4.7 times as long as at Moonlight's rates, 12.3 to 12.7 ms at 4K. Raising HEVC's bit rate costs the client time. - **After a loss**, an HEVC client needs a keyframe: 600 to 730 kB at 4K, which took 5.6 to 7.8 ms to decode. Every PyroWave frame is one. - **BGRA** is VideoToolbox's colour conversion, which Moonlight does not use: it draws NV12. Into RGB, emberwave took 27 to 49 % less than HEVC into BGRA at 60 frames a second. @@ -296,10 +296,44 @@ The two runs' means differed by 4 % or less, except in five rows. HEVC's encodin - NVENC's streams, and AV1, which Apple's devices decode but do not encode (issue #82); - 10-bit video; - FFmpeg's wrapper around VideoToolbox, through which Moonlight on the Mac decodes; -- the wait for the display, and power; +- the wait for the display on the iPhone, and power; - that HEVC's decoder leaves the GPU free to draw while emberwave's decoding uses it; - the pictures' quality. The report's luma PSNR only checks that the stream decodes to the pictures: about 40 dB at Moonlight's rates and 43 dB at emberwave's bytes on both devices. The grain, fresh in every picture, dominates it. [Picture quality](#picture-quality) sets a key frame of HEVC beside emberwave's frames on a picture of a game. +## In a display loop, on the Mac + +A client draws the last picture at every refresh of its screen, and its decodes share the GPU with that. `scripts/bench_app_on_mac.sh -display` times decodes beside such a loop (issue #81, decision D72): a borderless window covers a screen while the run lasts, about three minutes, and a `CVDisplayLink` and a `CAMetalLayer` draw the last decoded picture over it at the screen's refresh, one textured triangle with two drawables. Frames arrive on the decode's thread as a stream's do and are decoded into textures, emberwave into Y, Cb and Cr textures and into an RGB texture, HEVC at Moonlight's rate into NV12 textures, 300 frames after at least 30 and a second of warm-up. Each way also runs without the loop in the same run. The loop is a 60 Hz one, every second refresh of the Mac's 120 Hz screen, and a 120 Hz one at every refresh. The stream arrives at 59.5 frames a second, a little under 60, so that its frames drift across the refreshes as a stream's do against a display: frames locked to the refresh would all take one latency, whichever phase they happened to have. + +Measured on 2026-11-06 from 18:30 to 18:36 on the Apple M3 Max of the tables above, macOS 27.0.1, on mains power, in two runs one after the other, with the library of release 0.18.1 (the code of `main` at `d6bc2b2`). The Mac was not quiet: its load average was 4 to 7 with other programs running. Mean milliseconds per frame, from a frame's arrival until its picture is in textures, the lowest and the highest of the two runs: + +| Decode | no display loop | loop at 60 Hz | loop at 120 Hz | +|---|---|---|---| +| 1080p, emberwave into Y, Cb, Cr textures | 1.64 to 1.80 | 1.93 to 2.04 | 1.84 to 2.08 | +| 1080p, HEVC into NV12 textures | 2.11 to 2.14 | 2.14 to 2.17 | 2.07 to 2.13 | +| 1080p, emberwave into an RGB texture | 2.06 to 2.19 | 2.39 to 2.61 | 2.35 to 2.53 | +| 1440p, emberwave into Y, Cb, Cr textures | 2.12 to 2.17 | 2.22 to 2.40 | 2.22 to 2.24 | +| 1440p, HEVC into NV12 textures | 2.50 to 2.61 | 2.68 to 2.73 | 2.63 to 2.77 | +| 1440p, emberwave into an RGB texture | 2.80 to 2.90 | 2.64 to 3.06 | 3.08 to 3.26 | +| 4K, emberwave into Y, Cb, Cr textures | 3.01 to 3.03 | 3.25 to 3.27 | 3.54 to 3.55 | +| 4K, HEVC into NV12 textures | 4.02 to 4.05 | 4.00 to 4.04 | 4.03 to 4.07 | +| 4K, emberwave into an RGB texture | 3.91 to 4.26 | 4.29 to 4.36 | 4.51 to 4.70 | + +And from a frame's arrival until the display presented it (the `presentedTime` of the drawable that first drew it), mean milliseconds, the lowest and the highest of the two runs: + +| To the screen | loop at 60 Hz | loop at 120 Hz | +|---|---|---| +| 1080p, emberwave, Y, Cb, Cr | 17.50 to 19.55 | 17.77 to 18.48 | +| 1080p, HEVC, NV12 | 17.30 to 18.44 | 18.02 to 18.20 | +| 4K, emberwave, Y, Cb, Cr | 18.80 to 19.58 | 19.46 to 19.89 | +| 4K, HEVC, NV12 | 19.18 to 19.53 | 20.02 to 20.14 | + +- **Into the planes, emberwave decodes quicker than HEVC in the loop too.** At 4K it took 3.25 to 3.55 ms against 4.00 to 4.07 ms, at 1440p 2.22 to 2.40 against 2.63 to 2.77, at 1080p 1.84 to 2.08 against 2.07 to 2.17: a gap of 0 to 0.3 ms at 1080p, 0.3 to 0.5 ms at 1440p and 0.5 to 0.8 ms at 4K, run by run. The runs of release 0.11.0 at 60 frames a second, in the section above, had the two about even or HEVC ahead, 3.77 to 4.04 against 3.71 to 3.85 ms at 4K; the library has changed since (decisions D52 to D54 and the paced work of #86). +- **The loop costs emberwave a little and HEVC nothing.** Beside the loop emberwave's decode into the planes took 0.2 to 0.4 ms longer at 1080p and 0.2 to 0.5 ms at 4K; HEVC's, on the media engine, did not move. The loop did not keep the GPU's clock up: the paced decodes did not get quicker beside it, they shared the GPU with its draw (a first probe, with a second queue blitting a texture, said the same, #81). +- **The way to the screen does not tell the two apart.** Every way took 17 to 21 ms to reach the screen, the ways within about 2 ms of each other and well within the spread of one run, whose 99th percentiles were 22 to 40 ms. The picture waits for the next refresh, and the display takes a further two or so. A decode that is up to 0.8 ms quicker is a small part of that. +- **Frames that no refresh drew.** The 60 Hz loop, at every second refresh of a 120 Hz screen, skipped 6 to 24 of 300 frames for each of the ways, as the drift between a 59.5 Hz stream and a 60 Hz loop puts two frames before one refresh now and then; the 120 Hz loop skipped at most 2 of 300. At 120 Hz 1 to 28 of about 710 calls of the display link came more than half an interval late. +- **Into RGB** emberwave took 0.3 to 0.7 ms longer than into the planes at 1080p and about 1 ms longer at 4K, the colour conversion's cost; HEVC into BGRA was not timed in this mode, so there is no like-for-like row. +- **Limits.** The Mac alone; the iPhone half of #81 needs the phone. The 60 Hz loop is a 60 frames a second client on a 120 Hz screen: the Mac's one 60 Hz screen, an external monitor, took no window (its window was never on screen), and a real 60 Hz display would add up to a refresh more to the way to the screen. The loop draws one triangle with a texture, a client draws more, and a streaming client has other threads busy. The Mac was not quiet. + ## Planes stored into textures Since decision D52, on Metal a decode into Y, Cb and Cr textures has the inverse transform's last levels store the planes into them, where it used to pack them into a buffer and copy them in (D41). Measured on 2026-10-29 from 19:00 to 19:20 on the Apple M3 Max of the tables above, macOS 27.0.1, on mains power, the code of `main` at `5f69efb` against that of [#103](https://github.com/with-developer/emberwave/pull/103). Nobody used the Mac, but a screen recorder and other programs ran. diff --git a/README.ko.md b/README.ko.md index b4542d9..1c12573 100644 --- a/README.ko.md +++ b/README.ko.md @@ -148,7 +148,7 @@ iPhone 17의 Safari에서는 [벤치마크 페이지](https://with-developer.git 앞으로 할 일입니다. 준비되는 대로 각각 릴리스합니다. -- **실사용 환경에서 측정.** 클라이언트의 화면 출력 루프 안에서 디코딩해, 화면에 나오기까지의 시간을 Mac과 iPhone에서 잽니다([#81](https://github.com/with-developer/emberwave/issues/81)). NVENC로 만든 HEVC와 AV1 스트림을 emberwave 스트림과 나란히 디코딩하고, 움직이는 게임 영상의 화질을 같은 비트레이트에서 예측 프레임을 쓰는 HEVC와 비교합니다([#82](https://github.com/with-developer/emberwave/issues/82)). 이 비교로 색차 대역 가중치를 바꿀지도 정합니다(결정 D67). 스트림 속도에서 4K는 upstream의 FP32 디코더를 따라잡았지만 1080p는 아직 0.15~0.3 ms 느리며, 이 차이를 줄입니다([#150](https://github.com/with-developer/emberwave/issues/150)). +- **실사용 환경에서 측정.** 클라이언트의 화면 출력 루프 안에서 디코딩해, 화면에 나오기까지의 시간을 Mac과 iPhone에서 잽니다([#81](https://github.com/with-developer/emberwave/issues/81), Mac 쪽은 끝났고 결정 D72에 기록했습니다). NVENC로 만든 HEVC와 AV1 스트림을 emberwave 스트림과 나란히 디코딩하고, 움직이는 게임 영상의 화질을 같은 비트레이트에서 예측 프레임을 쓰는 HEVC와 비교합니다([#82](https://github.com/with-developer/emberwave/issues/82)). 이 비교로 색차 대역 가중치를 바꿀지도 정합니다(결정 D67). 스트림 속도에서 4K는 upstream의 FP32 디코더를 따라잡았지만 1080p는 아직 0.15~0.3 ms 느리며, 이 차이를 줄입니다([#150](https://github.com/with-developer/emberwave/issues/150)). - **1.0을 향해: 클라이언트.** emberwave가 코덱으로 남도록, 브라우저 스트리밍은 별도 저장소에서 만듭니다. 호스트 화면을 GPU에서 인코딩해 로컬 네트워크로 보내고, 브라우저 탭에서 WebGPU로 디코딩합니다. 지연 시간부터 잽니다. 그다음에는 PyroWave로 스트리밍하는 호스트들에 emberwave 인코더를 제안합니다. 1.0이 되려면 안정된 C·Rust API, 실제로 쓰이는 클라이언트 하나, 그리고 프레임이 실제 스트림의 간격으로 들어올 때 업스트림만큼 빠른 디코딩이 필요합니다. ## 저장소 구성 diff --git a/README.md b/README.md index 29960a6..7b3eae7 100644 --- a/README.md +++ b/README.md @@ -148,7 +148,7 @@ Done: the milestones to the first release, M0 to M5 ([#1](https://github.com/wit Next, each in a release when it is ready: -- **Measured in real use.** Decoding inside a client's display loop, timed to the screen, on a Mac and an iPhone ([#81](https://github.com/with-developer/emberwave/issues/81)); NVENC's HEVC and AV1 streams decoded beside emberwave's, and the quality of moving game footage against HEVC with predicted frames at the same bit rate, which also tells whether chroma's band weights should change ([#82](https://github.com/with-developer/emberwave/issues/82), decision D67); and 1080p at a stream's pace, still 0.15 to 0.3 ms behind upstream's FP32 decoder where 4K has caught up ([#150](https://github.com/with-developer/emberwave/issues/150)). +- **Measured in real use.** Decoding inside a client's display loop, timed to the screen, on a Mac and an iPhone ([#81](https://github.com/with-developer/emberwave/issues/81); the Mac's half is done, decision D72); NVENC's HEVC and AV1 streams decoded beside emberwave's, and the quality of moving game footage against HEVC with predicted frames at the same bit rate, which also tells whether chroma's band weights should change ([#82](https://github.com/with-developer/emberwave/issues/82), decision D67); and 1080p at a stream's pace, still 0.15 to 0.3 ms behind upstream's FP32 decoder where 4K has caught up ([#150](https://github.com/with-developer/emberwave/issues/150)). - **Toward 1.0, clients.** Streaming to a browser, in a repository of its own so that emberwave stays a codec: a host's screen encoded on its GPU, sent over the local network and decoded with WebGPU in a tab, its latency measured first. After that, emberwave's encoder offered to the hosts that stream PyroWave. 1.0 needs stable C and Rust APIs, one client in real use, and decoding at a stream's pace level with upstream's. ## Repository layout diff --git a/crates/emberwave-ffi/examples/ios/README.md b/crates/emberwave-ffi/examples/ios/README.md index 42a9e78..859cad0 100644 --- a/crates/emberwave-ffi/examples/ios/README.md +++ b/crates/emberwave-ffi/examples/ios/README.md @@ -67,7 +67,7 @@ Unlock the phone and leave it on the home screen first. With `-run`, the app sta ## On a Mac -`scripts/bench_app_on_mac.sh` makes the same measurements on a Mac, by the app's own code: it builds `EmberwaveBenchMac`, a command-line front-end, against the XCFramework's library for the Mac, and prints the app's report. The progress goes to the standard error. It needs the XCFramework built by `scripts/build_xcframework.sh`, which has the library for the Mac. +`scripts/bench_app_on_mac.sh` makes the same measurements on a Mac, by the app's own code: it builds `EmberwaveBenchMac`, a command-line front-end, against the XCFramework's library for the Mac, and prints the app's report. The progress goes to the standard error. It needs the XCFramework built by `scripts/build_xcframework.sh`, which has the library for the Mac. With `-display` it decodes beside a display loop instead (issue #81, decision D72): a borderless window covers a screen for about three minutes, drawing the last picture at 60 and 120 Hz, and the report adds the time to the screen; `-short` makes it brief. ## In CI From 2b533ea6712c1b9e6640e04d211f2df4ef876406 Mon Sep 17 00:00:00 2001 From: harry Date: Fri, 6 Nov 2026 19:23:05 +0900 Subject: [PATCH 3/3] bench: whole beats, a window in every row, a guard for a sleeping screen A review of the display-loop benchmark found that the mean time to the screen of the 60 Hz loop depends on the run's start phase, that the rows without a loop ran in an application with no window on the screen, and that nothing stopped a run on a sleeping or locked screen from printing rows as if it had drawn. The program now times whole beats of its stream's drift against a 60 Hz loop (119 frames, 357 for a run), keeps a black window on the screen for the rows without a loop, keeps the displays awake, and stops where the window never reaches the screen. The display link's statistics start with the first timed frame, frames not drawn are counted as timed offers minus timed draws, frames the display never presented are counted, and the HEVC stream's failures other than "unavailable" fail the run. FrameClock takes its rate as an argument. The docs say what the first runs were and what is to be repeated. Claude-Session: https://claude.ai/code/session_019fJGoCTvP5e7n599q3ZQ42 --- ARCHITECTURE.md | 2 + BENCHMARKS.md | 5 +- .../ios/EmberwaveBench/Benchmark.swift | 8 +- .../ios/EmberwaveBench/VideoToolbox.swift | 13 +-- .../ios/EmberwaveBenchMac/DisplayLoop.swift | 84 ++++++++++++++----- .../ios/EmberwaveBenchMac/DisplayRun.swift | 66 ++++++++++++--- .../examples/ios/EmberwaveBenchMac/main.swift | 7 +- scripts/bench_app_on_mac.sh | 8 +- 8 files changed, 149 insertions(+), 44 deletions(-) diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 6fe7807..3cb8df4 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1859,6 +1859,8 @@ The last frame's detail, always, gains everything on a still picture and loses u **Result.** Into the planes, emberwave decoded as quickly as or up to 0.8 ms quicker than HEVC into NV12 in the loop (0 to 0.3 ms at 1080p, 0.5 to 0.8 ms at 4K) (BENCHMARKS.md, "In a display loop, on the Mac"). The loop cost emberwave 0.2 to 0.5 ms and HEVC nothing, and did not keep the GPU's clock up. The time to the screen was 17 to 21 ms for all ways and does not tell them apart. +**After the review.** The first runs (in BENCHMARKS.md) were made before it. It found the mean time to the screen of the 60 Hz loop dependent on the run's start phase (the stream and the loop beat every 2 s, so a run times whole beats of 119 frames now), the rows without a loop running in an application with no window on the screen (a black window is kept there now), and nothing guarding against a sleeping or locked screen (the displays are kept awake, and a window that never reaches the screen stops the run). The runs are to be repeated with the changed program, which needs the screen unlocked. + **Not done.** The iPhone half of #81, which needs the phone. A real client's pipeline, with its other threads and a heavier draw. HEVC into BGRA in the loop. **Consequence.** The shared benchmark code gains a `PictureSink` type, a settable `FrameClock.rate`, and entry points that decode a prepared HEVC stream paced with a sink; the iOS app's behaviour is unchanged. The script's deployment target is macOS 14. diff --git a/BENCHMARKS.md b/BENCHMARKS.md index ce28246..db3e345 100644 --- a/BENCHMARKS.md +++ b/BENCHMARKS.md @@ -304,7 +304,7 @@ The two runs' means differed by 4 % or less, except in five rows. HEVC's encodin A client draws the last picture at every refresh of its screen, and its decodes share the GPU with that. `scripts/bench_app_on_mac.sh -display` times decodes beside such a loop (issue #81, decision D72): a borderless window covers a screen while the run lasts, about three minutes, and a `CVDisplayLink` and a `CAMetalLayer` draw the last decoded picture over it at the screen's refresh, one textured triangle with two drawables. Frames arrive on the decode's thread as a stream's do and are decoded into textures, emberwave into Y, Cb and Cr textures and into an RGB texture, HEVC at Moonlight's rate into NV12 textures, 300 frames after at least 30 and a second of warm-up. Each way also runs without the loop in the same run. The loop is a 60 Hz one, every second refresh of the Mac's 120 Hz screen, and a 120 Hz one at every refresh. The stream arrives at 59.5 frames a second, a little under 60, so that its frames drift across the refreshes as a stream's do against a display: frames locked to the refresh would all take one latency, whichever phase they happened to have. -Measured on 2026-11-06 from 18:30 to 18:36 on the Apple M3 Max of the tables above, macOS 27.0.1, on mains power, in two runs one after the other, with the library of release 0.18.1 (the code of `main` at `d6bc2b2`). The Mac was not quiet: its load average was 4 to 7 with other programs running. Mean milliseconds per frame, from a frame's arrival until its picture is in textures, the lowest and the highest of the two runs: +Measured on 2026-11-06 from 18:30 to 18:36, with the first version of the loop (see Limits), on the Apple M3 Max of the tables above, macOS 27.0.1, on mains power, in two runs one after the other, with the library of release 0.18.1 (the code of `main` at `d6bc2b2`). The Mac was not quiet: its load average was 4 to 7 with other programs running. Mean milliseconds per frame, from a frame's arrival until its picture is in textures, the lowest and the highest of the two runs: | Decode | no display loop | loop at 60 Hz | loop at 120 Hz | |---|---|---|---| @@ -330,9 +330,10 @@ And from a frame's arrival until the display presented it (the `presentedTime` o - **Into the planes, emberwave decodes quicker than HEVC in the loop too.** At 4K it took 3.25 to 3.55 ms against 4.00 to 4.07 ms, at 1440p 2.22 to 2.40 against 2.63 to 2.77, at 1080p 1.84 to 2.08 against 2.07 to 2.17: a gap of 0 to 0.3 ms at 1080p, 0.3 to 0.5 ms at 1440p and 0.5 to 0.8 ms at 4K, run by run. The runs of release 0.11.0 at 60 frames a second, in the section above, had the two about even or HEVC ahead, 3.77 to 4.04 against 3.71 to 3.85 ms at 4K; the library has changed since (decisions D52 to D54 and the paced work of #86). - **The loop costs emberwave a little and HEVC nothing.** Beside the loop emberwave's decode into the planes took 0.2 to 0.4 ms longer at 1080p and 0.2 to 0.5 ms at 4K; HEVC's, on the media engine, did not move. The loop did not keep the GPU's clock up: the paced decodes did not get quicker beside it, they shared the GPU with its draw (a first probe, with a second queue blitting a texture, said the same, #81). - **The way to the screen does not tell the two apart.** Every way took 17 to 21 ms to reach the screen, the ways within about 2 ms of each other and well within the spread of one run, whose 99th percentiles were 22 to 40 ms. The picture waits for the next refresh, and the display takes a further two or so. A decode that is up to 0.8 ms quicker is a small part of that. -- **Frames that no refresh drew.** The 60 Hz loop, at every second refresh of a 120 Hz screen, skipped 6 to 24 of 300 frames for each of the ways, as the drift between a 59.5 Hz stream and a 60 Hz loop puts two frames before one refresh now and then; the 120 Hz loop skipped at most 2 of 300. At 120 Hz 1 to 28 of about 710 calls of the display link came more than half an interval late. +- **Frames that no refresh drew.** The 60 Hz loop, at every second refresh of a 120 Hz screen, skipped 6 to 24 of 300 frames for each of the ways, as the drift between a 59.5 Hz stream and a 60 Hz loop puts two frames before one refresh now and then; the 120 Hz loop skipped at most 2 of 300. - **Into RGB** emberwave took 0.3 to 0.7 ms longer than into the planes at 1080p and about 1 ms longer at 4K, the colour conversion's cost; HEVC into BGRA was not timed in this mode, so there is no like-for-like row. - **Limits.** The Mac alone; the iPhone half of #81 needs the phone. The 60 Hz loop is a 60 frames a second client on a 120 Hz screen: the Mac's one 60 Hz screen, an external monitor, took no window (its window was never on screen), and a real 60 Hz display would add up to a refresh more to the way to the screen. The loop draws one triangle with a texture, a client draws more, and a streaming client has other threads busy. The Mac was not quiet. +- **What a review found, after these runs.** (1) The mean time to the screen of the 60 Hz loop depends on the phase where the run happens to start: the stream and the loop beat every 2 s, 119 frames, and 300 frames are two and a half beats, so the mean wait for the next refresh moves by about 1 ms from run to run; differences under 2 ms between the ways cannot be read from that column. The decode times do not depend on it. (2) The rows without a loop ran in an application with no window on the screen, which the system may throttle after some seconds; none of those rows looks inflated, but the loop's cost may be a little larger than shown. (3) The count of late calls of the display link is of the order of an idle link's own jitter, and is left out. The program has changed since: it times whole beats (357 frames, 119 with `-short`), keeps a black window on the screen for the rows without a loop, keeps the displays awake and stops where the window never reaches the screen, and counts frames the display did not present. The runs are to be repeated with it; the Mac's screen was locked when it was written. ## Planes stored into textures diff --git a/crates/emberwave-ffi/examples/ios/EmberwaveBench/Benchmark.swift b/crates/emberwave-ffi/examples/ios/EmberwaveBench/Benchmark.swift index 61b9d6b..9a78197 100644 --- a/crates/emberwave-ffi/examples/ios/EmberwaveBench/Benchmark.swift +++ b/crates/emberwave-ffi/examples/ios/EmberwaveBench/Benchmark.swift @@ -118,14 +118,18 @@ struct FrameClock { /// stream's 60, so that the frames drift across the display's refreshes /// as a stream's do, where frames locked to the refresh would all have the /// same latency, one of many. - static var rate = Double(streamFrameRate) + let rate: Double + + init(rate: Double = Double(streamFrameRate)) { + self.rate = rate + } mutating func wait() { guard let previous = due else { due = now() return } - let next = max(previous + 1000 / FrameClock.rate, now()) + let next = max(previous + 1000 / rate, now()) due = next let delay = next - now() guard delay > 0 else { diff --git a/crates/emberwave-ffi/examples/ios/EmberwaveBench/VideoToolbox.swift b/crates/emberwave-ffi/examples/ios/EmberwaveBench/VideoToolbox.swift index de55bc8..56cd001 100644 --- a/crates/emberwave-ffi/examples/ios/EmberwaveBench/VideoToolbox.swift +++ b/crates/emberwave-ffi/examples/ios/EmberwaveBench/VideoToolbox.swift @@ -301,13 +301,13 @@ func makeHEVCStream( /// second, handing each picture to `sink`: the paced decode of `timeHEVC`, /// with a display loop beside it. func hevcPaced( - _ stream: HEVCStream, _ plan: Plan, device: MTLDevice, sink: PictureSink? + _ stream: HEVCStream, _ plan: Plan, rate: Double, device: MTLDevice, sink: PictureSink? ) throws -> (stats: Stats, hardware: Bool?) { let decoder = try settingUp { try HEVCDecoder(stream.format, kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, device) } defer { decoder.invalidate() } - let stats = try decoder.time(stream.frames, plan, paced: true, sink: sink) + let stats = try decoder.time(stream.frames, plan, paced: true, rate: rate, sink: sink) return (stats, decoder.hardware) } @@ -770,10 +770,11 @@ private final class HEVCDecoder { /// the decoder can take after any frame, and the others, each timed; one /// after the other, or with `paced` as a stream's frames arrive, the /// warm-up too. - func time(_ frames: [[UInt8]], _ plan: Plan, paced: Bool = false, sink: PictureSink? = nil) - throws -> Stats - { - var clock = FrameClock() + func time( + _ frames: [[UInt8]], _ plan: Plan, paced: Bool = false, rate: Double = Double(streamFrameRate), + sink: PictureSink? = nil + ) throws -> Stats { + var clock = FrameClock(rate: rate) let warm = now() var index = 0 while index < plan.warmupFrames || now() - warm < plan.warmupMs { diff --git a/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/DisplayLoop.swift b/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/DisplayLoop.swift index 98a3e06..ebdf678 100644 --- a/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/DisplayLoop.swift +++ b/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/DisplayLoop.swift @@ -68,12 +68,15 @@ struct DisplayOutcome { /// the frames that were drawn at all, those timed only. let toScreen: [Double] /// Frames offered by the decode, those timed only, and how many of them the - /// loop never drew because a newer one had come before its next refresh. + /// loop never drew because a newer one had come before its next refresh, + /// and how many it drew that the display never presented. let offered: Int let skipped: Int - /// Refreshes drawn; the time between the display link's calls, the median - /// and the number of calls that came more than half an interval later, - /// a sign of a loop that cannot keep up. + let notPresented: Int + /// From the first timed frame on: refreshes drawn; the time between the + /// display link's calls, the median, and the number of calls that came + /// more than half an interval later. The calls' times carry the link + /// thread's own wake-up jitter, so a few late ones say little. let refreshes: Int let interval: Double let late: Int @@ -116,7 +119,9 @@ final class DisplayLoop: NSObject { private var sequence = 0 private var drawn = -1 private var offeredTimed = 0 - private var skipped = 0 + private var drawnTimed = 0 + private var notPresented = 0 + private var measuring = false private var refreshes = 0 private var intervals: [Double] = [] private var ticks = 0 @@ -192,6 +197,12 @@ final class DisplayLoop: NSObject { /// another input, never takes it. var isOnScreen: Bool { window.occlusionState.contains(.visible) } + /// What the window is doing, for a failure's message. + var windowState: String { + "\(screenName): window visible \(window.isVisible), on screen \(isOnScreen), " + + "on screen named \(window.screen?.localizedName ?? "none"), app active \(NSApp.isActive)" + } + /// The picture size the screen is drawn at, for the report. var drawableSize: CGSize { layer.drawableSize } @@ -204,6 +215,16 @@ final class DisplayLoop: NSObject { latest = Offered( sequence: sequence, arrived: arrived, layout: layout, textures: textures, timed: timed) if timed { + // The statistics of the display link start with the timed frames, + // not with the window or the warm-up. + if !measuring { + measuring = true + refreshes = 0 + intervals = [] + ticks = 0 + noPicture = 0 + noDrawable = 0 + } offeredTimed += 1 } lock.unlock() @@ -219,7 +240,9 @@ final class DisplayLoop: NSObject { sequence = 0 drawn = -1 offeredTimed = 0 - skipped = 0 + drawnTimed = 0 + notPresented = 0 + measuring = false refreshes = 0 intervals = [] ticks = 0 @@ -230,13 +253,7 @@ final class DisplayLoop: NSObject { lock.unlock() divider = max(1, Int((Double(maximumRate) / Double(rate)).rounded())) counter = 0 - NSApp.activate() - window.orderFrontRegardless() - // The window server takes a moment to put it on the screen. - let shown = Date(timeIntervalSinceNow: 2) - while !window.occlusionState.contains(.visible), Date() < shown { - RunLoop.main.run(until: Date(timeIntervalSinceNow: 0.05)) - } + showWindow() var link: CVDisplayLink? CVDisplayLinkCreateWithCGDisplay(displayID, &link) if let link { @@ -254,9 +271,30 @@ final class DisplayLoop: NSObject { return maximumRate / divider } + /// Puts the window on the screen without drawing in it, as the rows without + /// a display loop have it, so that only the drawing differs between them + /// and the others: an app without a window is throttled by the system + /// after some seconds. Call on the main thread. + func showWindow() { + NSApp.activate() + window.orderFrontRegardless() + // The window server takes a moment to put it on the screen. + let shown = Date(timeIntervalSinceNow: 2) + while !window.occlusionState.contains(.visible), Date() < shown { + RunLoop.main.run(until: Date(timeIntervalSinceNow: 0.05)) + } + } + + /// Hides the window again. Call on the main thread. + func hideWindow() { + window.orderOut(nil) + } + /// Stops drawing, hides the window and says what the loop saw, once the /// last pictures have been presented. Call on the main thread. func stop() -> DisplayOutcome { + // The last frame is drawn at the next refresh. + RunLoop.main.run(until: Date(timeIntervalSinceNow: 0.05)) if let link { CVDisplayLinkStop(link) } @@ -270,9 +308,9 @@ final class DisplayLoop: NSObject { let median = intervals.sorted()[safe: intervals.count / 2] ?? 0 let late = intervals.filter { $0 > median * 1.5 }.count return DisplayOutcome( - toScreen: toScreen, offered: offeredTimed, skipped: skipped, refreshes: refreshes, - interval: median, late: late, ticks: ticks, noPicture: noPicture, - noDrawable: noDrawable) + toScreen: toScreen, offered: offeredTimed, skipped: offeredTimed - drawnTimed, + notPresented: notPresented, refreshes: refreshes, interval: median, late: late, + ticks: ticks, noPicture: noPicture, noDrawable: noDrawable) } private func tick() { @@ -322,8 +360,8 @@ final class DisplayLoop: NSObject { lock.lock() refreshes += 1 if fresh { - if drawn >= 0, frame.timed { - skipped += frame.sequence - drawn - 1 + if frame.timed { + drawnTimed += 1 } drawn = frame.sequence } @@ -331,8 +369,14 @@ final class DisplayLoop: NSObject { if fresh, frame.timed { let arrived = frame.arrived drawable.addPresentedHandler { [weak self] drawable in - // 0 for a drawable that was never presented. - guard drawable.presentedTime > 0, let self else { + guard let self else { + return + } + // 0 for a drawable that was never presented, or was skipped. + guard drawable.presentedTime > 0 else { + self.lock.lock() + self.notPresented += 1 + self.lock.unlock() return } let latency = drawable.presentedTime * 1000 - arrived diff --git a/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/DisplayRun.swift b/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/DisplayRun.swift index 3bd785f..9db8fbe 100644 --- a/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/DisplayRun.swift +++ b/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/DisplayRun.swift @@ -10,6 +10,7 @@ import AppKit import Emberwave import Foundation +import IOKit.pwr_mgt import Metal extension Emberwave { @@ -119,7 +120,7 @@ extension Emberwave { return end - start } - var clock = FrameClock() + var clock = FrameClock(rate: displayStreamRate) let warm = now() var index = 0 while index < plan.warmupFrames || now() - warm < plan.warmupMs { @@ -141,6 +142,11 @@ extension Emberwave { /// about every two seconds. private let displayStreamRate = 59.5 +/// Frames in one beat between that stream and a 60 Hz loop: 2 s, 119 frames. A +/// run times whole beats, so that where the frames fall against the refreshes +/// averages out the same in every run. +let displayBeatFrames = 119 + /// One way of decoding to measure with and without a display loop. private struct Way { let name: String @@ -150,7 +156,16 @@ private struct Way { /// Runs the whole measurement and returns its report. Call off the main /// thread, which has to run the application's loop. func runDisplayBenchmark(_ plan: Plan, progress: (String) -> Void) throws -> String { - FrameClock.rate = displayStreamRate + // The displays have to stay on, and be on: an unattended Mac sleeps them, + // and then nothing reaches a screen. + let activity = ProcessInfo.processInfo.beginActivity( + options: [.idleDisplaySleepDisabled, .idleSystemSleepDisabled, .userInitiated], + reason: "emberwave display-loop benchmark") + defer { ProcessInfo.processInfo.endActivity(activity) } + var wake: IOPMAssertionID = 0 + IOPMAssertionDeclareUserActivity( + "emberwave display-loop benchmark" as CFString, kIOPMUserActiveLocal, &wake) + defer { IOPMAssertionRelease(wake) } let emberwave = try Emberwave() guard let device = emberwave.device else { throw EmberwaveFailure(description: "no Metal device") @@ -204,9 +219,13 @@ func runDisplayBenchmark(_ plan: Plan, progress: (String) -> Void) throws -> Str "Per frame (ms): mean / median / 99th percentile. decode: from the frame's arrival until " + "its picture is in textures; to screen: from the frame's arrival until the display " + "presented it (presentedTime of its drawable, the first refresh that drew it). " - + "skipped: timed frames no refresh drew, because a newer one came first. " - + "refreshes: the median time between the display link's calls, and how many calls came " - + "more than half an interval late.", + + "not drawn: timed frames no refresh drew, because a newer one came first (the last " + + "frame has a refresh to be drawn at); not presented: drawn, but the display never " + + "presented the drawable. display link: the median time between its calls, and how " + + "many came more than half an interval late, from the first timed frame on; the " + + "calls' times carry the link thread's own jitter, so a few late ones say little. " + + "The frames are whole beats of the stream's drift against a 60 Hz loop. Without a " + + "loop, the window is there, black, with nothing drawn in it.", ] for size in pictureSizes { progress("\(size.name): making \(plan.pictures) test pictures") @@ -214,7 +233,14 @@ func runDisplayBenchmark(_ plan: Plan, progress: (String) -> Void) throws -> Str let frames = try emberwave.encodeFrames(size, pictures) let rate = hevcRates(size)[0] progress("\(size.name): encoding HEVC at \(rate.bitsPerSecond / 1_000_000) Mbit/s") - let stream = try? makeHEVCStream(size, rate, pictures, plan) + // A Mac without HEVC, as a virtual machine may be, is not a failure; anything + // else is. + let stream: HEVCStream? + do { + stream = try makeHEVCStream(size, rate, pictures, plan) + } catch let failure as MediaFailure where failure.unavailable { + stream = nil + } if stream == nil { lines.append("\(size.name): HEVC not measured, VideoToolbox could not make the stream") } @@ -229,12 +255,23 @@ func runDisplayBenchmark(_ plan: Plan, progress: (String) -> Void) throws -> Str if let stream { ways.append( Way(name: "HEVC into NV12 textures") { - try hevcPaced(stream, plan, device: device, sink: $0).stats + try hevcPaced(stream, plan, rate: displayStreamRate, device: device, sink: $0).stats }) } lines.append("\(size.name), thermal state \(thermalState()):") for refresh in [nil] + loops.map({ Optional($0.rate) }) { lines.append(refresh.map { " display loop at \($0) Hz:" } ?? " no display loop:") + // Without a loop the window is there, black, and nothing draws in it. + let idle = refresh == nil ? loops.last?.loop : nil + if let idle { + DispatchQueue.main.sync { idle.showWindow() } + progress("idle window: " + DispatchQueue.main.sync { idle.windowState }) + } + defer { + if let idle { + DispatchQueue.main.sync { idle.hideWindow() } + } + } for way in ways { progress("\(size.name): \(way.name), \(refresh.map { "display \($0) Hz" } ?? "no display")") guard let refresh, let loop = loops.first(where: { $0.rate == refresh })?.loop else { @@ -242,6 +279,13 @@ func runDisplayBenchmark(_ plan: Plan, progress: (String) -> Void) throws -> Str continue } _ = DispatchQueue.main.sync { loop.start(rate: refresh) } + guard DispatchQueue.main.sync(execute: { loop.isOnScreen }) else { + let state = DispatchQueue.main.sync { loop.windowState } + _ = DispatchQueue.main.sync { loop.stop() } + throw EmberwaveFailure( + description: "the display loop's window never reached the screen (is it asleep?): " + + state) + } let decode: Stats do { decode = try way.run(loop.offer) @@ -256,9 +300,11 @@ func runDisplayBenchmark(_ plan: Plan, progress: (String) -> Void) throws -> Str } else { text += "; to screen \(Stats(seen.toScreen, total: seen.toScreen.reduce(0, +)))" } - text += "; skipped \(seen.skipped) of \(seen.offered); refreshes every " - + String(format: "%.2f ms (%.0f Hz), late %d of %d", seen.interval, 1000 / max(seen.interval, 0.001), seen.late, seen.refreshes) - text += "; ticks \(seen.ticks), no picture \(seen.noPicture), no drawable \(seen.noDrawable)" + text += "; not drawn \(seen.skipped) of \(seen.offered), not presented \(seen.notPresented)" + text += String( + format: "; display link every %.2f ms (%.0f Hz), %d of %d calls late", + seen.interval, 1000 / max(seen.interval, 0.001), seen.late, seen.ticks) + text += "; no picture \(seen.noPicture), no drawable \(seen.noDrawable)" lines.append(row(way.name, text, indent: 4)) } } diff --git a/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/main.swift b/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/main.swift index d9e1295..01e87fe 100644 --- a/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/main.swift +++ b/crates/emberwave-ffi/examples/ios/EmberwaveBenchMac/main.swift @@ -8,7 +8,12 @@ import AppKit import Foundation -let plan = CommandLine.arguments.contains("-autorun") || CommandLine.arguments.contains("-short") ? Plan.short : Plan() +var plan = CommandLine.arguments.contains("-autorun") || CommandLine.arguments.contains("-short") ? Plan.short : Plan() +// The display-loop run times whole beats between its stream and a 60 Hz loop: +// see `displayStreamRate`. +if CommandLine.arguments.contains("-display") { + plan.frames = plan.frames == Plan.short.frames ? displayBeatFrames : 3 * displayBeatFrames +} // -display: decodes beside a display loop over a screen (issue #81), which // needs the application's run loop on the main thread, so the work goes to // another one. It covers a screen for a few minutes. diff --git a/scripts/bench_app_on_mac.sh b/scripts/bench_app_on_mac.sh index 8d2d701..0ac532a 100755 --- a/scripts/bench_app_on_mac.sh +++ b/scripts/bench_app_on_mac.sh @@ -11,9 +11,11 @@ # -autorun the short run of CI: every size briefly, and the three texture # checks; fails unless the report is complete and they pass # -display instead of the report above, decodes at 60 frames a second beside -# a display loop (issue #81): a borderless window covers the screen -# with the highest refresh rate while it runs, a few minutes, and -# draws the last picture at 60 and 120 Hz. Leave the Mac alone. +# a display loop (issue #81): a borderless window covers a screen +# while it runs, a few minutes, and draws the last picture at 60 +# and at 120 Hz: on the slowest screen that has the rate, and on +# the built-in screen when that one takes no window or has more. +# It keeps the displays awake. Leave the Mac alone. -short is brief. # # Needs macOS with Xcode, and the XCFramework with its library for the Mac, # which scripts/build_xcframework.sh makes. A full run takes a few minutes: