From e412b679f0f3694930060fd6e833da7bc45f8081 Mon Sep 17 00:00:00 2001 From: rpone Date: Sun, 25 Oct 2026 00:02:19 +0800 Subject: [PATCH 01/16] feat(html-local): add shared-texture GPU video export Enable opt-in Electron capture through electronPath, reuse deterministic frame preparation and source sampling, and encode ordered shared textures with WebCodecs before FFmpeg muxing. Close request-owned HTTP connections and file streams after capture. Validated type checking, 82 existing tests, alpha fixtures with explicit browser paths, 600-frame MG sequencing, source sampling, cancellation, limits, fractional rates, and packed-code export on Apple Silicon. Windows and Linux hardware validation remains pending. --- packages/provider-html-local/README.md | 147 +++++++----- packages/provider-html-local/package.json | 3 + .../provider-html-local/src/activation.ts | 7 +- .../src/capture-process.ts | 7 +- .../provider-html-local/src/capture-worker.ts | 2 + packages/provider-html-local/src/capture.ts | 107 +++++---- packages/provider-html-local/src/electron.ts | 25 ++ .../provider-html-local/src/gpu-capture.ts | 219 ++++++++++++++++++ .../provider-html-local/src/gpu-encoder.cjs | 90 +++++++ .../provider-html-local/src/gpu-worker.cjs | 28 +++ .../provider-html-local/src/opaque-capture.ts | 57 +++-- packages/provider-html-local/src/options.ts | 10 +- packages/provider-html-local/src/output.ts | 3 +- packages/provider-html-local/src/program.ts | 6 + packages/provider-html-local/src/provider.ts | 2 +- packages/provider-html-local/src/render.ts | 18 +- packages/provider-html-local/src/server.ts | 16 +- pnpm-lock.yaml | 91 ++++++++ 18 files changed, 707 insertions(+), 131 deletions(-) create mode 100644 packages/provider-html-local/src/electron.ts create mode 100644 packages/provider-html-local/src/gpu-capture.ts create mode 100644 packages/provider-html-local/src/gpu-encoder.cjs create mode 100644 packages/provider-html-local/src/gpu-worker.cjs diff --git a/packages/provider-html-local/README.md b/packages/provider-html-local/README.md index dc0efdf2e..dd5cb7455 100644 --- a/packages/provider-html-local/README.md +++ b/packages/provider-html-local/README.md @@ -5,78 +5,110 @@ Trusted local implementation of: - `@hypit/html-program@1#rasterize-visual` → `TimelineVisual`; - `@hypit/html-program@1#rasterize-frames` → ordered PNG BlobRefs. -The Provider consumes the typed `HtmlProgram` directly. It serves the staged -program to independent Puppeteer/Chrome processes, applies absolute integer -frames through `window.__hypitFrameProgram`, decodes compiler-declared source -video frames with FFmpeg, captures PNG pixels through Chrome DevTools Protocol, -and streams ordered frames into one H.264 encoder. It has no external HTML video -engine or producer dependency. +The Provider stages a typed `HtmlProgram`, applies absolute integer frames through +`window.__hypitFrameProgram`, and supplies compiler-declared video samples from a +bounded FFmpeg decode store. It supports two video export configurations: -Filesystem staging is private to this local Provider. `@hypit/html-program` -retains only the portable program and its pure materialization function; it -does not expose a Node project/staging subpath for hypothetical executors. -Inspection of staged Compositable Surface files and probing the selected media -tools are private here for the same reason: they validate inputs used by this -executor and do not establish a shared media-execution ABI. +| Configuration | Capture and encoding | +| --- | --- | +| `chromePath`, or managed Chrome selection | Puppeteer PNG capture and ordered FFmpeg H.264 encoding | +| `electronPath` | Electron shared GPU textures, Canvas cropping, WebCodecs H.264 encoding, FFmpeg MP4 muxing | -## Runtime boundary +PNG frame requests use the selected Chrome browser. Filesystem staging, source +probing and decoded-frame leases belong to this Provider. -The Endpoint facet is `rasterize`. Runtime configuration owns deployment policy: +## Runtime configuration -- `workers`: fixed browser count or `auto`; +The Endpoint facet is `rasterize`. Runtime configuration owns execution policy: + +- `workers`: fixed worker count or `auto`; - `maxWorkers`: automatic ceiling; - `defaultConcurrency`: whole requests admitted in this Endpoint pool; -- `browserCapacity`: Chrome slots reserved across admitted requests; +- `browserCapacity`: worker slots reserved across admitted requests; - `chromePath`, or exact `browserVersion` plus `browserCacheDirectory`; +- `electronPath`: an installed Electron 45+ distribution executable, enabling GPU video export; - `ffmpegPath` and `ffprobePath`; - optional process/stage deadlines and byte budgets. -The package recommendation is stored as `hypit.htmlBrowser.version`. Managed -Chrome defaults to `~/.cache/hypit/html-rasterizer/chrome`; its ManagedProgram -state is stored below `.hypit-html-browser-program`. Browser installation occurs -only through explicit Runtime preparation: +For GPU export, `auto` uses the admitted worker ceiling, limited by the requested +frame span. PNG capture adjusts its active workers using measured throughput. +GPU export requires `browserGpu: "auto"` or `"hardware"`, even canvas dimensions, +shared-texture support and a supported WebCodecs H.264 configuration. Unsupported +configurations fail with a diagnostic. + +The Chrome recommendation is stored as `hypit.htmlBrowser.version`. Managed Chrome +defaults to `~/.cache/hypit/html-rasterizer/chrome`; its ManagedProgram state is +stored below `.hypit-html-browser-program`. Browser installation occurs through +explicit Runtime preparation: ```sh hypit programs prepare --runtime ./hypit.runtime.json --endpoint html.local ``` -Inspection and execution never search environment variables or silently choose a -different system browser. `chromePath` selects a user-managed executable; -`browserVersion` selects one exact managed Chrome for Testing version. The two -selection modes are mutually exclusive. - -## Frame selection and parallelism - -A continuous request retains original absolute frame identities. A sparse request -accepts strictly increasing, non-contiguous frames. Workers are independent -browser processes and may start at any requested frame; they do not wait for an -earlier worker to traverse the program. +`chromePath` selects a user-managed Chrome executable; `browserVersion` selects +one exact managed Chrome for Testing version. These selections are mutually +exclusive. `electronPath` is configured separately and is checked during Runtime +inspection. Relative executable paths in Runtime configuration resolve against +the Runtime data root. -The compiler's typed `frameSources` plan determines exact source-video samples, -including reverse, hold, loop and rational-rate mappings. The Provider probes -needed sources, decodes bounded windows lazily, shares decoded frames only within -the current execution, and leases them under `maxDecodedSourceBytes`. Chrome does -not independently decode those same videos. Arbitrary author HTML is not parsed -to infer source-time semantics. +Example `html.local` Endpoint configuration: -Completed PNGs enter a bounded ordered sink. FFmpeg is the necessary local -serialization point for one MP4 stream, not a scheduler or shared metadata -service. Capture of later frames can continue while earlier frames are encoded. -Sparse frame output writes only requested PNGs and is bounded by -`maxRenderedBytes`. - -## Lifecycle and failure - -Each request owns its temporary directory, localhost server, Chrome processes, -decoded-frame store and encoder. Nothing is shared across requests. Cancellation -or failure closes page sessions, source leases, processes and resource I/O. -Process-tree cleanup is best effort after abnormal exits; hard OS termination -requires containment supplied by the deployment. +```json +{ + "electronPath": "/absolute/path/to/electron", + "workers": 8, + "quality": "standard" +} +``` -The executor waits for page frame listeners, declared fonts, dynamic image -decodes and compositor settlement before accepting a screenshot. Readiness is -scoped to compiler-published capture roots when available; real global stylesheet -edits intentionally fall back to whole-document inspection. +On macOS the executable is inside `Electron.app/Contents/MacOS/Electron`; on +Windows it is `electron.exe`; on Linux it is `electron`. Electron's shared-texture +API is experimental. The integration has been exercised on macOS / Apple Silicon; +Windows and Linux require validation on their target GPU and driver combinations. +Linux additionally needs the display environment required by Electron. + +## Frames and encoding + +A continuous request retains original absolute frame identities. A sparse PNG +request accepts strictly increasing, non-contiguous frames. Workers can start at +any requested frame. The compiler's `frameSources` plan determines reverse, hold, +loop and rational-rate video sampling. Decoded samples are leased under +`maxDecodedSourceBytes`; their images replace compiler-owned native video pixels. + +The GPU path renders each Program in a fixed-size same-origin iframe. A separate +32-bit frame identifier below the authored viewport identifies the accepted +texture. Only its 8 KiB identifier region is read back to the CPU. Canvas crops +that region before creating the encoder's `VideoFrame`. Each worker retains at +most one frame waiting for ordered encoding. `maxPendingFrameBytes` bounds pending +compressed output, with one chunk allowed to exceed the budget. Scene surfaces +remain alive through the encoder flush. + +WebCodecs uses H.264 High with `quality` latency and a hardware preference. +At 1920 × 1080 / 30 fps, `quality` selects target bitrates of 6 Mbps (`draft`), +12 Mbps (`standard`), or 24 Mbps (`high`). Targets scale with pixel count and frame +rate, with a 64 Kbps minimum. Actual hardware encoder availability is determined +by Chromium. FFmpeg assigns the requested rational frame time base and muxes the +encoded stream. PNG video export uses x264 CRF 28, 23, or 18 respectively. +These settings do not imply identical rate control or picture quality. + +For PNG video capture, `maxPendingFrameBytes` bounds the ordered PNG queue. +`maxRenderedBytes` limits the final MP4 or total requested PNG bytes; GPU capture +also checks compressed bytes as they arrive. Output validation decodes every frame +and verifies dimensions, codec, exact frame rate and frame count before storage. + +## Lifecycle and page readiness + +Each request owns its temporary directory, localhost server, browser processes, +decoded-frame store and encoder. Cancellation and failure stop that execution and +release its resources. Process-tree cleanup is best effort after abnormal exits; +hard OS termination requires containment supplied by the deployment. + +Frame preparation waits for page frame listeners, declared fonts and dynamic +image decodes. GPU capture additionally waits for two animation-frame callbacks +before updating its texture identifier. Authored asynchronous drawing must finish +inside the Program's frame completion hooks. Child pages rendered in another +renderer process are rejected by GPU capture; the iframe container makes +`window.top` refer to the capture shell. ## Direct use @@ -85,10 +117,9 @@ import { rasterizeHtmlProgram } from "@hypit/provider-html-local"; const visual = await rasterizeHtmlProgram( { program, range: { startFrame: 240, endFrameExclusive: 360 } }, - { resources, workers: 4 }, + { resources, electronPath: "/absolute/path/to/electron", workers: 8 }, ); ``` -Omit `range` for the complete program. Audio is not part of this result; -`@hypit/media-local` and the author assembly prepare audio and mux the -final artifact. +Omit `range` for the complete program. Audio is prepared and muxed by +`@hypit/media-local` and the author assembly. diff --git a/packages/provider-html-local/package.json b/packages/provider-html-local/package.json index cfb8838f4..4d0770119 100644 --- a/packages/provider-html-local/package.json +++ b/packages/provider-html-local/package.json @@ -9,6 +9,8 @@ }, "files": [ "dist", + "src/gpu-worker.cjs", + "src/gpu-encoder.cjs", "README.md" ], "hypit": { @@ -43,6 +45,7 @@ "@hypit/script": "workspace:*", "@hypit/timeline": "workspace:*", "@hypit/visual-track": "workspace:*", + "electron": "45.0.0", "sharp": "0.35.4" } } diff --git a/packages/provider-html-local/src/activation.ts b/packages/provider-html-local/src/activation.ts index 543414fe5..ea3a3499b 100644 --- a/packages/provider-html-local/src/activation.ts +++ b/packages/provider-html-local/src/activation.ts @@ -22,7 +22,7 @@ const localHTMLRuntimeAdapter = createRuntimeEndpointAdapterFacet({ if (context.pool === undefined) throw new Error("local HTML renderer Provider Pool is required"); const config = runtimeConfigObject(context.config, "local HTML renderer"); runtimeConfigExact(config, [ - "nodePath", "chromePath", "browserVersion", "browserCacheDirectory", "browserDownloadBaseUrl", "ffprobePath", "ffmpegPath", "workers", "maxWorkers", "quality", "browserGpu", + "nodePath", "chromePath", "electronPath", "browserVersion", "browserCacheDirectory", "browserDownloadBaseUrl", "ffprobePath", "ffmpegPath", "workers", "maxWorkers", "quality", "browserGpu", "defaultConcurrency", "browserCapacity", "initializationTimeoutMs", "frameTimeoutMs", "processTimeoutMs", "maxProcessOutputBytes", "artifactStagingConcurrency", "maxPendingFrameBytes", "maxDecodedSourceBytes", "maxRenderedBytes", ], "local HTML renderer"); runtimeConfigString(config.nodePath, "HTML renderer nodePath"); @@ -41,6 +41,8 @@ const localHTMLRuntimeAdapter = createRuntimeEndpointAdapterFacet({ } const configuredNode = runtimeConfigString(config.nodePath, "HTML renderer nodePath"); const configuredChrome = runtimeConfigString(config.chromePath, "HTML renderer chromePath"); + const configuredElectron = runtimeConfigString(config.electronPath, "HTML renderer electronPath"); + const electronPath = configuredElectron === undefined ? undefined : resolve(context.dataRoot, configuredElectron); const browserVersion = runtimeConfigString(config.browserVersion, "HTML renderer browserVersion"); const browserDownloadBaseUrl = runtimeConfigString(config.browserDownloadBaseUrl, "HTML renderer browserDownloadBaseUrl"); const configuredCache = runtimeConfigString(config.browserCacheDirectory, "HTML renderer browserCacheDirectory"); @@ -48,6 +50,7 @@ const localHTMLRuntimeAdapter = createRuntimeEndpointAdapterFacet({ chromePath: resolve(context.dataRoot, configuredChrome), }) }); const browser = { + ...(electronPath === undefined ? {} : { electronPath }), ...(browserVersion === undefined ? {} : { browserVersion }), ...(browserDownloadBaseUrl === undefined ? {} : { browserDownloadBaseUrl }), ...(chromePath === undefined ? {} : { chromePath }), @@ -100,6 +103,8 @@ const localHTMLRuntimeAdapter = createRuntimeEndpointAdapterFacet({ ffmpegPath, }), diagnose: async () => [ + ...(electronPath === undefined ? [] : [{ severity: "info" as const, code: "HTML_GPU_EXPORT", subject: context.instance, + message: `Shared-texture video export using Electron: ${electronPath}` }]), { severity: "info", code: "HTML_BROWSER_SELECTION", subject: context.instance, message: `${chromePath === undefined ? `Managed Chrome Headless Shell ${selectedBrowserVersion(browser)} (${browserVersion === undefined ? "Provider recommendation" : "Profile version"})` : "Profile browser"}: ${browserExecutablePath(browser)}` }, ...await diagnoseRuntimeExecutable({ root: context.dataRoot, configured: configuredFfmpeg, fallback: "ffmpeg", subject: "FFmpeg" }), diff --git a/packages/provider-html-local/src/capture-process.ts b/packages/provider-html-local/src/capture-process.ts index af9acbcfa..a7c8eede6 100644 --- a/packages/provider-html-local/src/capture-process.ts +++ b/packages/provider-html-local/src/capture-process.ts @@ -4,6 +4,7 @@ import { fileURLToPath } from "node:url"; import { killRenderTree } from "./process-tree.js"; import type { CaptureInput } from "./capture.js"; import type { HtmlRasterProgress } from "./render.js"; +import { electronEnvironment } from "./electron.js"; const cleanupMs = 5_000; @@ -17,11 +18,15 @@ export async function runCaptureProcess( ): Promise { signal.throwIfAborted(); await new Promise((resolve, reject) => { - const child = spawn(process.execPath, [ + const gpu = input.frames === undefined && input.config.electronPath !== undefined; + const child = spawn(gpu ? input.config.electronPath! : process.execPath, gpu ? [ + fileURLToPath(new URL("../src/gpu-worker.cjs", import.meta.url)), input.directory, entry.href, + ] : [ "--import", import.meta.resolve("tsx"), "--import", new URL("./capture-exit.js", import.meta.url).href, fileURLToPath(entry), ], { + ...(gpu ? { env: electronEnvironment() } : {}), detached: process.platform !== "win32", windowsHide: true, stdio: ["ignore", "pipe", "pipe", "ipc"], }); diff --git a/packages/provider-html-local/src/capture-worker.ts b/packages/provider-html-local/src/capture-worker.ts index d1d52dafc..0b5949a9d 100644 --- a/packages/provider-html-local/src/capture-worker.ts +++ b/packages/provider-html-local/src/capture-worker.ts @@ -1,6 +1,7 @@ import { releaseCaptureExitCleanup } from "./capture-exit.js"; import { captureStagedVisual } from "./capture.js"; import type { CaptureInput } from "./capture.js"; +import { createRequire } from "node:module"; const controller = new AbortController(); const message = (error: unknown) => error instanceof Error ? error.message : String(error); @@ -21,6 +22,7 @@ const finish = (value: { type: "completed" } | { type: "failed"; error: string } process.send?.(value, undefined, undefined, (error) => { if (error) { console.error(message(error)); process.exitCode = 1; } if (process.connected) process.disconnect(); + if (process.versions.electron !== undefined) createRequire(import.meta.url)("electron").app.exit(error ? 1 : 0); }); }; controller.signal.addEventListener("abort", () => { diff --git a/packages/provider-html-local/src/capture.ts b/packages/provider-html-local/src/capture.ts index 449378e54..7fa539a8d 100644 --- a/packages/provider-html-local/src/capture.ts +++ b/packages/provider-html-local/src/capture.ts @@ -5,7 +5,7 @@ import type { Browser } from "puppeteer-core"; import type { HtmlProgram } from "@hypit/html-program"; import type { MediaFrameRange } from "@hypit/media"; import type { HtmlRasterProgress, resolveExecutionOptions } from "./render.js"; -import { assert, mediaExecutablePath, openProcessInput } from "./process.js"; +import { assert, mediaExecutablePath, openProcessInput, runProcess } from "./process.js"; import { verifyOutput } from "./output.js"; import { distributeFrameRange, requestedFrameRanges, sourceFrameAt, sourceWindows, videoSlots } from "./sampling.js"; import type { VideoSlot } from "./sampling.js"; @@ -17,6 +17,8 @@ import { decodeSourceFrameWindow, probeSourceVideo } from "./source-frame-decode import { SourceFrameStore } from "./source-frame-store.js"; import type { SourceFrameWindow } from "./source-frame-store.js"; import { serveHtmlProgram } from "./server.js"; +import { createGpuCapture, gpuCapturePages } from "./gpu-capture.js"; +import type { SourceFrameUpdate } from "./opaque-capture.js"; export type CaptureInput = { readonly document: HtmlProgram; @@ -40,10 +42,11 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort const at = (index: number) => input.frames?.[index] ?? range.startFrame + index; const fps = { num: document.frameRate.numerator, den: document.frameRate.denominator }; const limit = renderWorkerLimit(config, frameCount, fps.num / fps.den); - const concurrency = new CaptureConcurrency(limit, config.workers === "auto"); - // Short contiguous batches retain sequential capture while allowing free browsers - // to help with expensive passages. This queue exists only inside this render. - const batchCount = Math.max(limit, Math.ceil(frameCount / Math.max(1, Math.round(fps.num / fps.den)))); + const useGpu = input.frames === undefined && config.electronPath !== undefined; + const concurrency = new CaptureConcurrency(limit, config.workers === "auto" && !useGpu); + // GPU workers claim individual ordinals so waiting textures do not serialize + // whole batches. PNG workers retain contiguous batches for decode reuse. + const batchCount = useGpu ? frameCount : Math.max(limit, Math.ceil(frameCount / Math.max(1, Math.round(fps.num / fps.den)))); const batches = distributeFrameRange({ startFrame: 0, endFrameExclusive: frameCount }, batchCount); let nextBatch = 0; const started = performance.now(); @@ -78,6 +81,7 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort } }; let server: Awaited> | undefined; + let gpu: Awaited> | undefined; let encoderCompletion: Promise | undefined; let sourceFrameStore: SourceFrameStore | undefined; // Closing pages interrupts an in-flight capture as well as the next loop iteration. @@ -136,12 +140,17 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort } return payloads; }; - server = await serveHtmlProgram(work); + const gpuPages = useGpu ? gpuCapturePages(document.canvas) : undefined; + server = await serveHtmlProgram(work, gpuPages?.pages); + if (gpuPages !== undefined) { + gpu = await stage("GPU encoder initialization", config.initializationTimeoutMs, + () => createGpuCapture(input, server!.url, gpuPages, controller)); + } const serverUrl = server.url; const outputFrames = join(work, "frames"); const output = join(work, "visual.mp4"); const crf = { draft: 28, standard: 23, high: 18 }[config.quality]; - const encoder = input.frames === undefined ? openProcessInput({ executable: ffmpegPath, + const encoder = input.frames === undefined && !useGpu ? openProcessInput({ executable: ffmpegPath, argv: ["-v", "error", "-y", "-f", "image2pipe", "-framerate", `${fps.num}/${fps.den}`, "-vcodec", "png", "-i", "pipe:0", "-frames:v", String(frameCount), "-an", "-c:v", "libx264", "-crf", String(crf), "-preset", config.quality === "draft" ? "veryfast" : "medium", @@ -154,8 +163,8 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort frameCount, maxPendingBytes: config.maxPendingFrameBytes, signal, writer: { write: encoder.write, close: async () => { await encoder.close(); } }, }); - if (encoder === undefined) await mkdir(outputFrames); - else { + if (input.frames !== undefined) await mkdir(outputFrames); + if (encoder !== undefined) { encoderCompletion = encoder.completed; void encoder.completed.catch((error) => { orderedFrames!.fail(error); @@ -184,30 +193,40 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort let setupMs = 0; try { signal.throwIfAborted(); - session = await puppeteer.launch({ executablePath: config.chromePath, headless: true, - defaultViewport: { width: document.canvas.width, height: document.canvas.height, deviceScaleFactor: 1 }, - args: ["--no-sandbox", "--disable-dev-shm-usage", - ...(config.browserGpu === "software" ? ["--disable-gpu"] : []), - ] }); - sessions.add(session); - signal.throwIfAborted(); - const page = await session.newPage(); - if (config.initializationTimeoutMs !== undefined) { - page.setDefaultTimeout(config.initializationTimeoutMs); - page.setDefaultNavigationTimeout(config.initializationTimeoutMs); - } - await page.evaluateOnNewDocument(input.frameSelectionPrelude); - const browserPid = session.process()?.pid; - onProgress?.({ phase: "worker-initializing", worker, range, browserPid, elapsedMs: elapsedMs() }); - await stage(`worker ${worker} initialization`, config.initializationTimeoutMs, async () => { - await page.goto(serverUrl, { waitUntil: "load" }); - await page.evaluate(async () => { - const program = (window as unknown as { __hypitFrameProgram?: { prepare(): Promise } }).__hypitFrameProgram; - if (program === undefined) throw new Error("HTML Program page ABI is missing"); - await program.prepare(); + let browserPid: number | undefined; + let captureFrame: (frame: number, sources: readonly SourceFrameUpdate[], ordinal: number, prepared: () => void) + => Promise>>>>; + if (gpu !== undefined) { + onProgress({ phase: "worker-initializing", worker, range, browserPid: undefined, elapsedMs: elapsedMs() }); + const gpuWorker = await stage(`worker ${worker} initialization`, config.initializationTimeoutMs, () => gpu!.createWorker()); + browserPid = gpuWorker.browserPid; + captureFrame = gpuWorker.capture; + } else { + session = await puppeteer.launch({ executablePath: config.chromePath, headless: true, + defaultViewport: { width: document.canvas.width, height: document.canvas.height, deviceScaleFactor: 1 }, + args: ["--no-sandbox", "--disable-dev-shm-usage", + ...(config.browserGpu === "software" ? ["--disable-gpu"] : []), + ] }); + sessions.add(session); + signal.throwIfAborted(); + const page = await session.newPage(); + if (config.initializationTimeoutMs !== undefined) { + page.setDefaultTimeout(config.initializationTimeoutMs); + page.setDefaultNavigationTimeout(config.initializationTimeoutMs); + } + await page.evaluateOnNewDocument(input.frameSelectionPrelude); + browserPid = session.process()?.pid; + onProgress?.({ phase: "worker-initializing", worker, range, browserPid, elapsedMs: elapsedMs() }); + await stage(`worker ${worker} initialization`, config.initializationTimeoutMs, async () => { + await page.goto(serverUrl, { waitUntil: "load" }); + await page.evaluate(async () => { + const program = (window as unknown as { __hypitFrameProgram?: { prepare(): Promise } }).__hypitFrameProgram; + if (program === undefined) throw new Error("HTML Program page ABI is missing"); + await program.prepare(); + }); }); - }); - const captureFrame = await createOpaqueFrameCapture(page, document.canvas); + captureFrame = await createOpaqueFrameCapture(page, document.canvas); + } setupMs = performance.now() - launched; startupMs = Math.max(startupMs, setupMs); ready = true; @@ -229,7 +248,8 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort const required = sourceDemand(frameRanges(index, index + 1)); // A preference is the unconsumed suffix, not the whole batch: frames // behind this Worker must become ordinary eviction candidates. - const preferred = sourceDemand(frameRanges(index, batch.endFrameExclusive)); + const preferredEnd = useGpu ? Math.min(frameCount, index + Math.max(1, Math.round(fps.num / fps.den))) : batch.endFrameExclusive; + const preferred = sourceDemand(frameRanges(index, preferredEnd)); const sourceAcquireStarted = performance.now(); const sourceLease = await sourceFrameStore!.acquire(required, preferred); sourceAcquireWorkerMs += performance.now() - sourceAcquireStarted; @@ -239,7 +259,7 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort const sourceUpdates = [...payloads].map(([id, payload]) => ({ id, url: new URL(relative(work, payload.framePath).split(sep).map(encodeURIComponent).join("/"), serverUrl).href })); captured = await stage(`worker ${worker} frame ${frame}`, config.frameTimeoutMs, - () => captureFrame(frame, sourceUpdates)); + () => captureFrame(frame, sourceUpdates, index, () => sourceLease.release())); } finally { // Output submission may block behind an earlier frame. Never // retain source leases across that independent backpressure. @@ -252,9 +272,9 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort timing.seekMs += captured.seekMs; timing.prepareMs += captured.prepareMs; timing.screenshotMs += captured.screenshotMs; - if (orderedFrames === undefined) { + if (input.frames !== undefined) { await writeFile(join(outputFrames, `${String(index).padStart(9, "0")}.png`), captured.buffer); - } else { + } else if (orderedFrames !== undefined) { const outputSubmitStarted = performance.now(); await orderedFrames.submit(index, captured.buffer); outputSubmitWorkerMs += performance.now() - outputSubmitStarted; @@ -274,7 +294,7 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort } } console.log(`Capture worker ${worker}: ${completed} frames; startup ${Math.round(setupMs)} ms; ` - + `seek ${Math.round(timing.seekMs)} ms; prepare ${Math.round(timing.prepareMs)} ms; PNG ${Math.round(timing.screenshotMs)} ms`); + + `seek ${Math.round(timing.seekMs)} ms; prepare ${Math.round(timing.prepareMs)} ms; ${useGpu ? "texture and encode" : "PNG"} ${Math.round(timing.screenshotMs)} ms`); onProgress({ phase: "worker-complete", worker, completed, browserPid, elapsedMs: elapsedMs() }); } catch (error) { controller.abort(error); @@ -290,7 +310,7 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort } }; const initialWorkers = concurrency.target; - console.log(`Capture: opaque fast PNG; ${config.workers === "auto" ? "auto" : "fixed"} workers ${initialWorkers}, limit ${limit}`); + console.log(`Capture: ${useGpu ? "shared GPU texture" : "opaque fast PNG"}; ${config.workers === "auto" ? "auto" : "fixed"} workers ${initialWorkers}, limit ${limit}`); for (let i = 0; i < initialWorkers; i++) launch(); // New workers can join while earlier jobs run; collect every launched job. let awaited = 0; @@ -311,7 +331,15 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort + `source acquire ${Math.round(sourceAcquireWorkerMs)} worker-ms; output submit ${Math.round(outputSubmitWorkerMs)} worker-ms`); if (input.frames !== undefined) return outputFrames; onProgress({ phase: "encoding", elapsedMs: elapsedMs() }); - await orderedFrames!.close(); + if (gpu !== undefined) { + const raw = await gpu.finish(); + // The requested frame clock defines every packet duration, including the last. + await runProcess({ executable: ffmpegPath, + argv: ["-v", "error", "-y", "-r", `${fps.num}/${fps.den}`, "-i", raw, + "-an", "-c:v", "copy", "-bsf:v", `setts=pts=N:dts=N:duration=1:time_base=${fps.den}/${fps.num}`, + "-movflags", "+faststart", output], + timeoutMs: config.processTimeoutMs, maxOutputBytes: config.maxProcessOutputBytes, signal }); + } else await orderedFrames!.close(); const outputStat = await stat(output); assert(outputStat.size > 0 && outputStat.size <= config.maxRenderedBytes, "HTML renderer output is empty or exceeds its byte limit"); await verifyOutput({ path: output, document: { ...document, frameCount }, ffprobePath, @@ -326,6 +354,7 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort await Promise.allSettled([...sessions].map(close)); if (sourceFrameStore !== undefined) await sourceFrameStore.close(); if (encoderCompletion !== undefined) await Promise.allSettled([encoderCompletion]); + if (gpu !== undefined) await gpu.close(); if (server !== undefined) await server.close(); } } diff --git a/packages/provider-html-local/src/electron.ts b/packages/provider-html-local/src/electron.ts new file mode 100644 index 000000000..52f68ee86 --- /dev/null +++ b/packages/provider-html-local/src/electron.ts @@ -0,0 +1,25 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; +import { access, stat } from "node:fs/promises"; +import { constants } from "node:fs"; + +export function electronEnvironment(): NodeJS.ProcessEnv { + const env = { ...process.env }; + delete env.ELECTRON_RUN_AS_NODE; + delete env.NODE_OPTIONS; + return env; +} + +export async function requireElectronExecutable(path: string): Promise { + try { + if (!(await stat(path)).isFile()) throw new Error("Electron path is not a file"); + await access(path, process.platform === "win32" ? constants.F_OK : constants.X_OK); + const { stdout } = await promisify(execFile)(path, ["--version"], { + timeout: 15_000, windowsHide: true, env: electronEnvironment(), + }); + const version = /^v(\d+)\.\d+\.\d+(?:-\S+)?$/mu.exec(stdout.trim()); + if (version === null || Number(version[1]) < 45) throw new Error(`Expected Electron 45+, received ${stdout.trim()}`); + } catch (error) { + throw new Error(`HTML renderer Electron is unavailable at ${path}: ${error instanceof Error ? error.message : String(error)}. Configure electronPath with an installed Electron 45+ executable.`, { cause: error }); + } +} diff --git a/packages/provider-html-local/src/gpu-capture.ts b/packages/provider-html-local/src/gpu-capture.ts new file mode 100644 index 000000000..959ee33e0 --- /dev/null +++ b/packages/provider-html-local/src/gpu-capture.ts @@ -0,0 +1,219 @@ +import { createRequire } from "node:module"; +import { createWriteStream } from "node:fs"; +import { finished } from "node:stream/promises"; +import { randomUUID } from "node:crypto"; +import { join } from "node:path"; +import { fileURLToPath } from "node:url"; +import type { BrowserWindow, IpcMainEvent, OffscreenSharedTexture } from "electron"; +import type { Page } from "puppeteer-core"; +import type { CaptureInput } from "./capture.js"; +import { createFramePreparer } from "./opaque-capture.js"; +import type { SourceFrameUpdate } from "./opaque-capture.js"; +import { assert } from "./process.js"; + +export function gpuCapturePages({ width, height }: { width: number; height: number }) { + const shell = `/__hypit_capture_${randomUUID()}.html`; + const encoder = `/__hypit_encoder_${randomUUID()}.html`; + const html = ``; + return { shell, encoder, pages: new Map([[shell, html], [encoder, "Hypit encoder"]]) }; +} + +type PendingFrame = { token: number; ordinal: number; resolve(): void; reject(error: unknown): void }; +type Scene = { window: BrowserWindow; pending?: PendingFrame | undefined; + latest?: OffscreenSharedTexture | undefined; busy: boolean }; + +/** One request owns its windows, texture references and ordered H.264 stream. */ +export async function createGpuCapture(input: CaptureInput, origin: string, + pages: ReturnType, controller: AbortController) { + const { BrowserWindow, ipcMain, sharedTexture } = createRequire(import.meta.url)("electron") as typeof import("electron"); + const { config, document } = input; + const { width, height } = document.canvas; + const frameCount = input.range.endFrameExclusive - input.range.startFrame; + const fps = document.frameRate.numerator / document.frameRate.denominator; + assert(width % 2 === 0 && height % 2 === 0, "Shared-texture H.264 export requires even canvas dimensions"); + assert(frameCount < 2 ** 32, "Shared-texture export exceeds the frame identifier range"); + assert(sharedTexture?.importSharedTexture !== undefined, "This Electron build does not provide sharedTexture"); + const signal = controller.signal; + const raw = join(input.directory, "encoded.h264"); + const writer = createWriteStream(raw); + const writerClosed = finished(writer).catch(() => {}); + const scenes = new Set(); + let closing = false, bytesWritten = 0, chunks = 0, lastTimestamp = -1; + const encoding = new BrowserWindow({ show: false, width: 300, height: 200, webPreferences: { + preload: fileURLToPath(new URL("../src/gpu-encoder.cjs", import.meta.url)), sandbox: false, backgroundThrottling: false, + } }); + const fail = (error: unknown) => { if (!closing) controller.abort(error instanceof Error ? error : new Error(String(error))); }; + writer.on("error", fail); + const trusted = (event: IpcMainEvent) => event.sender === encoding.webContents; + function processTexture(scene: Scene) { + if (closing || scene.busy || scene.pending === undefined || scene.latest === undefined) return; + const texture = scene.latest; + scene.latest = undefined; + scene.busy = true; + let imported: ReturnType; + try { + imported = sharedTexture.importSharedTexture({ textureInfo: texture.textureInfo, allReferencesReleased: () => texture.release() }); + } catch (error) { texture.release(); fail(error); return; } + void sharedTexture.sendSharedTexture({ frame: encoding.webContents.mainFrame, importedSharedTexture: imported }, + { token: scene.pending.token, ordinal: scene.pending.ordinal }).finally(() => imported.release()).catch(fail); + } + const observed = (event: IpcMainEvent, token: number, accepted: boolean) => { + if (!trusted(event) || closing) return; + const scene = [...scenes].find(item => item.pending?.token === token); + if (scene === undefined) return; + scene.busy = false; + if (accepted) { + const pending = scene.pending!; + scene.pending = undefined; + scene.latest?.release(); scene.latest = undefined; + pending.resolve(); + } + processTexture(scene); + }; + const captureError = (event: IpcMainEvent, error: string) => { if (trusted(event)) fail(new Error(error)); }; + ipcMain.on("hypit-observed", observed); + ipcMain.on("hypit-capture-error", captureError); + ipcMain.handle("hypit-encoded", async (event, bytes: Uint8Array, timestamp: number) => { + assert(event.sender === encoding.webContents, "Unexpected encoder sender"); + signal.throwIfAborted(); + assert(timestamp > lastTimestamp && chunks < frameCount, "Encoder emitted an unexpected frame timestamp"); + lastTimestamp = timestamp; chunks++; bytesWritten += bytes.byteLength; + assert(bytesWritten <= config.maxRenderedBytes, "HTML renderer output exceeds its byte limit"); + await new Promise((resolve, reject) => writer.write(Buffer.from(bytes), error => error ? reject(error) : resolve())); + }); + const closeScene = (scene: Scene) => { + scene.pending?.reject(signal.reason ?? new Error("GPU capture closed")); scene.pending = undefined; + scene.latest?.release(); scene.latest = undefined; + if (!scene.window.isDestroyed()) scene.window.destroy(); + scenes.delete(scene); + }; + const close = async () => { + if (!closing) { + closing = true; + signal.removeEventListener("abort", aborted); + for (const scene of scenes) closeScene(scene); + if (!encoding.isDestroyed()) encoding.destroy(); + ipcMain.off("hypit-observed", observed); + ipcMain.off("hypit-capture-error", captureError); + ipcMain.removeHandler("hypit-encoded"); + writer.destroy(); + } + await writerClosed; + }; + const aborted = () => { void close(); }; + signal.addEventListener("abort", aborted, { once: true }); + encoding.webContents.on("render-process-gone", (_event, detail) => fail(new Error(`GPU encoder exited: ${detail.reason}`))); + encoding.webContents.setWindowOpenHandler(() => ({ action: "deny" })); + + const waitMessage = (channel: string) => new Promise((resolve, reject) => { + const done = (event: IpcMainEvent) => { if (trusted(event)) { cleanup(); resolve(); } }; + const abort = () => { cleanup(); reject(signal.reason); }; + const cleanup = () => { ipcMain.off(channel, done); signal.removeEventListener("abort", abort); }; + ipcMain.on(channel, done); signal.addEventListener("abort", abort, { once: true }); + if (signal.aborted) abort(); + }); + try { + signal.throwIfAborted(); + const ready = waitMessage("hypit-encoder-ready"); + void ready.catch(() => {}); + await encoding.loadURL(new URL(pages.encoder, origin).href); + await ready; + const configured = waitMessage("hypit-configured"); + void configured.catch(() => {}); + const bitrate = Math.max(64_000, Math.round({ draft: 6_000_000, standard: 12_000_000, high: 24_000_000 }[config.quality] + * width * height / (1920 * 1080) * fps / 30)); + encoding.webContents.send("hypit-configure", { width, height, fps, bitrate, maxPendingBytes: config.maxPendingFrameBytes }); + await configured; + } catch (error) { controller.abort(error); await close(); throw error; } + + return { + async createWorker() { + signal.throwIfAborted(); + const window = new BrowserWindow({ show: false, width: Math.max(width, 256), height: height + 8, + useContentSize: true, backgroundColor: "#000000", webPreferences: { backgroundThrottling: false, + offscreen: { useSharedTexture: true, sharedTexturePixelFormat: "argb", deviceScaleFactor: 1 } } }); + const scene: Scene = { window, busy: false }; + scenes.add(scene); + window.webContents.setFrameRate(240); + window.webContents.setWindowOpenHandler(() => ({ action: "deny" })); + window.webContents.on("render-process-gone", (_event, detail) => fail(new Error(`GPU scene exited: ${detail.reason}`))); + window.webContents.on("paint", event => { + const texture = event.texture; + if (texture === undefined) return; + if (closing || texture.textureInfo.widgetType !== "frame") { texture.release(); return; } + scene.latest?.release(); scene.latest = texture; + processTexture(scene); + }); + const sameRenderer = () => { + const top = window.webContents.mainFrame; + assert(top.framesInSubtree.every(frame => frame.processId === top.processId), + "Shared-texture export requires child pages in the same renderer process"); + }; + await window.loadURL("about:blank"); + window.webContents.debugger.attach("1.3"); + await window.webContents.debugger.sendCommand("Page.addScriptToEvaluateOnNewDocument", { source: input.frameSelectionPrelude }); + await window.loadURL(new URL(pages.shell, origin).href); + const child = window.webContents.mainFrame.frames.find(frame => frame.url === new URL("index.html", origin).href); + assert(child !== undefined, "GPU capture shell did not load the HTML Program"); + sameRenderer(); + await child.executeJavaScript("__hypitFrameProgram.prepare()"); + const viewport = await child.executeJavaScript("({width:innerWidth,height:innerHeight})") as { width: number; height: number }; + assert(viewport.width === width && viewport.height === height, "GPU capture changed the HTML Program viewport"); + const page: Pick = { + evaluate: ((fn: (...args: unknown[]) => unknown, ...args: unknown[]) => + child.executeJavaScript(`(${fn.toString()})(...${JSON.stringify(args)})`)) as Page["evaluate"], + }; + const prepare = await createFramePreparer(page); + return { + browserPid: window.webContents.getOSProcessId(), + async capture(frame: number, sources: readonly SourceFrameUpdate[], ordinal: number, prepared: () => void) { + signal.throwIfAborted(); sameRenderer(); + const timing = await prepare(frame, sources); + // Decoded source files can be evicted after their images have loaded, + // before this texture waits for earlier output ordinals. + prepared(); + const started = performance.now(); + await child.executeJavaScript("new Promise(resolve=>requestAnimationFrame(()=>requestAnimationFrame(resolve)))"); + sameRenderer(); signal.throwIfAborted(); + const token = ordinal + 1; + const observed = new Promise((resolve, reject) => { scene.pending = { token, ordinal, resolve, reject }; }); + void observed.catch(() => {}); + let repaint: ReturnType | undefined; + try { + await child.executeJavaScript(`parent.__hypitCaptureCommit(${token})`); + processTexture(scene); + repaint = setInterval(() => { + if (!closing && !scene.busy && scene.pending?.token === token) { + void child.executeJavaScript(`parent.__hypitCaptureCommit(${token})`).catch(fail); + } + }, 100); + await observed; + } finally { clearInterval(repaint); } + return { buffer: Buffer.alloc(0), ...timing, screenshotMs: performance.now() - started }; + }, + }; + }, + async finish() { + signal.throwIfAborted(); + const accepted = await encoding.webContents.executeJavaScript("window.__hypitFinishEncoding()"); + assert(accepted === frameCount && chunks === frameCount, "GPU encoder did not complete every requested frame"); + await new Promise((resolve, reject) => writer.end((error?: Error | null) => error ? reject(error) : resolve())); + signal.throwIfAborted(); + return raw; + }, + close, + }; +} diff --git a/packages/provider-html-local/src/gpu-encoder.cjs b/packages/provider-html-local/src/gpu-encoder.cjs new file mode 100644 index 000000000..3626f0e55 --- /dev/null +++ b/packages/provider-html-local/src/gpu-encoder.cjs @@ -0,0 +1,90 @@ +const { ipcRenderer, sharedTexture, contextBridge } = require("electron"); + +let encoder, options, canvas, context, failure; +let nextOrdinal = 0, pendingBytes = 0; +const turns = new Map(), output = new Set(), capacity = new Set(); +const notify = () => { for (const wake of capacity) wake(); capacity.clear(); }; +function check() { if (failure) throw failure; } +function fail(error) { + if (failure) return; + failure = error; + for (const turn of turns.values()) turn.reject(error); + turns.clear(); notify(); + ipcRenderer.send("hypit-capture-error", String(error.stack ?? error)); +} + +ipcRenderer.on("hypit-configure", async (_event, input) => { + try { + options = input; + const config = { codec: "avc1.640034", width: input.width, height: input.height, bitrate: input.bitrate, + framerate: input.fps, latencyMode: "quality", hardwareAcceleration: "prefer-hardware", avc: { format: "annexb" } }; + if (!(await VideoEncoder.isConfigSupported(config)).supported) { + throw Error("The selected Electron/GPU does not support the requested H.264 encoding configuration"); + } + canvas = new OffscreenCanvas(input.width, input.height); + context = canvas.getContext("2d", { alpha: false, colorSpace: "srgb" }); + if (!context) throw Error("GPU encoder canvas is unavailable"); + context.imageSmoothingEnabled = false; + encoder = new VideoEncoder({ error: fail, output(chunk) { + const bytes = new Uint8Array(chunk.byteLength); + chunk.copyTo(bytes); pendingBytes += bytes.length; + const sent = ipcRenderer.invoke("hypit-encoded", bytes, chunk.timestamp).catch(fail).finally(() => { + pendingBytes -= bytes.length; output.delete(sent); notify(); + }); + output.add(sent); + } }); + encoder.configure(config); + ipcRenderer.send("hypit-configured"); + } catch (error) { fail(error); } +}); + +sharedTexture.setSharedTextureReceiver(async ({ importedSharedTexture: imported }, request) => { + let source, frame; + try { + check(); + source = imported.getVideoFrame(); + if (!["BGRA", "BGRX", "RGBA", "RGBX"].includes(source.format)) { + throw Error(`Unsupported shared texture pixel format: ${source.format}`); + } + if (source.displayWidth !== Math.max(options.width, 256) || source.displayHeight !== options.height + 8) { + ipcRenderer.send("hypit-observed", request.token, false); return; + } + // Read only the renderer-owned identifier, then crop it from the GPU image. + const rect = { x: 0, y: options.height, width: 256, height: 8 }; + const bytes = new Uint8Array(source.allocationSize({ rect })); + const [plane] = await source.copyTo(bytes, { rect }); + let token = 0, valid = true; + for (let bit = 0; bit < 32; bit++) { + const x = (bit * 8 + 4) * 4; + const top = bytes[plane.offset + 2 * plane.stride + x], bottom = bytes[plane.offset + 6 * plane.stride + x]; + if (top > 224 && bottom < 32) token += 2 ** bit; + else if (!(top < 32 && bottom > 224)) valid = false; + } + if (!valid || token !== request.token) { + ipcRenderer.send("hypit-observed", request.token, false); return; + } + if (request.ordinal !== nextOrdinal) { + await new Promise((resolve, reject) => turns.set(request.ordinal, { resolve, reject })); + } + check(); + while (pendingBytes > options.maxPendingBytes) { await new Promise(resolve => capacity.add(resolve)); check(); } + context.drawImage(source, 0, 0, options.width, options.height, 0, 0, options.width, options.height); + const timestamp = Math.round(request.ordinal * 1e6 / options.fps); + const end = Math.round((request.ordinal + 1) * 1e6 / options.fps); + frame = new VideoFrame(canvas, { timestamp, duration: end - timestamp }); + encoder.encode(frame, { keyFrame: request.ordinal % Math.max(1, Math.round(options.fps * 2)) === 0 }); + frame.close(); frame = undefined; + while (encoder.encodeQueueSize > 8) { + await new Promise(resolve => encoder.addEventListener("dequeue", resolve, { once: true })); check(); + } + nextOrdinal++; + turns.get(nextOrdinal)?.resolve(); turns.delete(nextOrdinal); + ipcRenderer.send("hypit-observed", request.token, true); + } catch (error) { fail(error); } + finally { frame?.close(); source?.close(); imported.release(); } +}); + +contextBridge.exposeInMainWorld("__hypitFinishEncoding", async () => { + check(); await encoder.flush(); await Promise.all(output); check(); encoder.close(); return nextOrdinal; +}); +window.addEventListener("DOMContentLoaded", () => ipcRenderer.send("hypit-encoder-ready")); diff --git a/packages/provider-html-local/src/gpu-worker.cjs b/packages/provider-html-local/src/gpu-worker.cjs new file mode 100644 index 000000000..a9fc83c91 --- /dev/null +++ b/packages/provider-html-local/src/gpu-worker.cjs @@ -0,0 +1,28 @@ +const { app } = require("electron"); +const path = require("node:path"); + +app.setPath("userData", path.join(process.argv[2], "electron")); +app.on("window-all-closed", () => {}); +app.commandLine.appendSwitch("disable-frame-rate-limit"); +app.commandLine.appendSwitch("disable-gpu-vsync"); +app.commandLine.appendSwitch("force-color-profile", "srgb"); +if (process.platform === "darwin") { + app.commandLine.appendSwitch("enable-features", "VTVideoEncodeAcceleratorOpaqueRgbSharedImageEncode"); +} + +// The owner can send its request before the ESM capture module has loaded. +const pending = []; +const queue = value => pending.push(value); +process.on("message", queue); +app.whenReady().then(async () => { + app.dock?.hide(); + const { register } = await import("tsx/esm/api"); + register(); + await import(process.argv[3]); + process.off("message", queue); + for (const value of pending) process.emit("message", value); +}).catch(error => { + console.error(error); + if (process.connected) process.send({ type: "failed", error: String(error) }); + else app.exit(1); +}); diff --git a/packages/provider-html-local/src/opaque-capture.ts b/packages/provider-html-local/src/opaque-capture.ts index 8a534e01e..5a19df56b 100644 --- a/packages/provider-html-local/src/opaque-capture.ts +++ b/packages/provider-html-local/src/opaque-capture.ts @@ -5,22 +5,22 @@ type RenderWindow = Window & { __hypitFrameProgram: { prepare(): Promise; applyFrame(frame: number): Promise }; __hypitHtmlVisualError?: string; __hypitCaptureRoots?: readonly Element[]; + __hypitPrepareImages(): Promise; }; export type SourceFrameUpdate = { readonly id: string; readonly url: string }; -/** - * Capture one already-composited page state as an opaque PNG. The Provider owns - * the page ABI, exact source-frame overlays and CDP session directly; there is - * no intermediate rendering engine or patched installed dependency. - */ -export async function createOpaqueFrameCapture(page: Page, canvas: { readonly width: number; readonly height: number }) { - const cdp = await page.createCDPSession(); - // MP4 has no alpha channel. The authored Canvas paints over this final matte. - await cdp.send("Emulation.setDefaultBackgroundColorOverride", { color: { r: 0, g: 0, b: 0, a: 1 } }); +/** Install page-owned resource readiness and prepare deterministic frame states. */ +export async function createFramePreparer(page: Pick) { // Image readiness belongs to this page. A seek can select a new image through // attributes, a CSS class or a pseudo-element after initial page readiness. await page.evaluate(() => { + // Compiler-declared video samples are supplied by the decoded-frame store. + // Hide every source element so inactive native videos cannot cover its images. + for (const video of Array.from(document.querySelectorAll("video[data-hypit-source-frame]"))) { + video.dataset.hypitAuthoredVisibility = video.style.visibility; + video.style.visibility = "hidden"; + } const decoded = new Map>(); const dirtyRoots = new Set(); const animatedRoots = new Set(); @@ -46,7 +46,15 @@ export async function createOpaqueFrameCapture(page: Page, canvas: { readonly wi background(value: string, pending: Promise[]) { for (const match of value.matchAll(/url\(\s*(?:"((?:\\.|[^"\\])*)"|'((?:\\.|[^'\\])*)'|([^\s"'()]+))\s*\)/gu)) { const url = (match[1] ?? match[2] ?? match[3] ?? "").trim(); - if (url) pending.push(this.load(url)); + if (url) { + const resource = new URL(url, document.baseURI); + const page = new URL(document.URL); + const fragment = resource.hash; + resource.hash = page.hash = ""; + // Local SVG paint servers are resolved by this document's renderer. + if (fragment && resource.href === page.href) continue; + pending.push(this.load(url)); + } } }, style(style: CSSStyleDeclaration, pending: Promise[]) { @@ -55,6 +63,9 @@ export async function createOpaqueFrameCapture(page: Page, canvas: { readonly wi } }, inlineSignature(value: string): string { + // Ordinary transform/opacity writes need no CSS parser. Escaped names + // still go through CSSOM so escaped resource functions are recognized. + if (!/\\|url\(|image-set\(|cross-fade\(|var\(/iu.test(value)) return ""; const style = document.createElement("span").style; style.cssText = value; const candidates = [style.backgroundImage, style.maskImage, style.borderImageSource, style.listStyleImage, style.content]; @@ -239,7 +250,7 @@ export async function createOpaqueFrameCapture(page: Page, canvas: { readonly wi }); return async (frame: number, sources: readonly SourceFrameUpdate[] = []) => { const start = performance.now(); - await page.evaluate(async (input) => { + const prepareMs = await page.evaluate(async (input) => { const w = window as unknown as RenderWindow; await w.__hypitFrameProgram.applyFrame(input.frame); const active = new Set(input.sources.map(source => source.id)); @@ -268,20 +279,30 @@ export async function createOpaqueFrameCapture(page: Page, canvas: { readonly wi if (image.src !== source.url) image.src = source.url; await image.decode(); })); - }, { frame, sources }); - const sought = performance.now(); - await page.evaluate(async () => { - const w = window as unknown as RenderWindow & { __hypitPrepareImages: () => Promise }; + const prepareStarted = performance.now(); await w.__hypitPrepareImages(); if (w.__hypitHtmlVisualError !== undefined) throw new Error(w.__hypitHtmlVisualError); - }); + return performance.now() - prepareStarted; + }, { frame, sources }); const prepared = performance.now(); + return { seekMs: prepared - start - prepareMs, prepareMs }; + }; +} + +/** Capture the prepared page state as an opaque PNG. */ +export async function createOpaqueFrameCapture(page: Page, canvas: { readonly width: number; readonly height: number }) { + const cdp = await page.createCDPSession(); + // MP4 has no alpha channel. The authored Canvas paints over this final matte. + await cdp.send("Emulation.setDefaultBackgroundColorOverride", { color: { r: 0, g: 0, b: 0, a: 1 } }); + const prepare = await createFramePreparer(page); + return async (frame: number, sources: readonly SourceFrameUpdate[] = []) => { + const timing = await prepare(frame, sources); + const started = performance.now(); const result = await cdp.send("Page.captureScreenshot", { format: "png", optimizeForSpeed: true, fromSurface: true, captureBeyondViewport: false, clip: { x: 0, y: 0, width: canvas.width, height: canvas.height, scale: 1 }, }); - return { buffer: Buffer.from(result.data, "base64"), - seekMs: sought - start, prepareMs: prepared - sought, screenshotMs: performance.now() - prepared }; + return { buffer: Buffer.from(result.data, "base64"), ...timing, screenshotMs: performance.now() - started }; }; } diff --git a/packages/provider-html-local/src/options.ts b/packages/provider-html-local/src/options.ts index 300fb2f9f..46b09f406 100644 --- a/packages/provider-html-local/src/options.ts +++ b/packages/provider-html-local/src/options.ts @@ -3,15 +3,17 @@ export type HtmlQuality = "draft" | "standard" | "high"; export type HtmlBrowserGpu = "auto" | "software" | "hardware"; export type HtmlExecutionOptions = import("./browser.js").BrowserOptions & { + /** Electron 45+ executable for shared-texture video export. PNG requests use Chrome. */ + readonly electronPath?: string; readonly ffprobePath?: string; - /** Selected FFmpeg command for both source extraction and final H.264 encoding. */ + /** FFmpeg command for source extraction, PNG video encoding and GPU stream muxing. */ readonly ffmpegPath?: string; - /** Parallel Chrome workers inside one render. This is separate from Provider request concurrency. */ + /** Parallel capture workers inside one render, separate from Provider request concurrency. */ readonly workers?: HtmlWorkers; /** Upper bound for auto's browser reservation; explicit workers remain fixed. */ readonly maxWorkers?: number; readonly quality?: HtmlQuality; - /** Chrome's rasterizer, default hardware. Use software without a usable GPU. */ + /** Browser rasterizer, default hardware. Use software for PNG capture without a usable GPU. */ readonly browserGpu?: HtmlBrowserGpu; readonly processTimeoutMs?: number; readonly initializationTimeoutMs?: number; @@ -19,7 +21,7 @@ export type HtmlExecutionOptions = import("./browser.js").BrowserOptions & { readonly maxProcessOutputBytes?: number; /** Artifact staging lifecycles in flight inside one render. */ readonly artifactStagingConcurrency?: number; - /** Bytes of completed, out-of-order PNGs retained while an MP4 waits for its next frame. */ + /** Pending PNG bytes, or compressed bytes awaiting writes during GPU export. */ readonly maxPendingFrameBytes?: number; /** Bytes of decoded source PNGs resident in one render's reusable working set. */ readonly maxDecodedSourceBytes?: number; diff --git a/packages/provider-html-local/src/output.ts b/packages/provider-html-local/src/output.ts index 294d9286c..4a2b5b1ae 100644 --- a/packages/provider-html-local/src/output.ts +++ b/packages/provider-html-local/src/output.ts @@ -67,7 +67,8 @@ export async function verifyOutput(args: { assert(stream.width === args.document.canvas.width && stream.height === args.document.canvas.height, "Rendered visual canvas differs from its document"); const rate = parseRational(stream.avg_frame_rate ?? stream.r_frame_rate, "Rendered visual frame rate"); - assert(equalsRational(rate, args.document.frameRate), "Rendered visual frame rate differs from its document"); + assert(equalsRational(rate, args.document.frameRate), + `Rendered visual frame rate ${rate.numerator}/${rate.denominator} differs from its document ${args.document.frameRate.numerator}/${args.document.frameRate.denominator}`); assert(outputFrameCount(stream) === args.document.frameCount, "Rendered visual frame count differs from its document"); } diff --git a/packages/provider-html-local/src/program.ts b/packages/provider-html-local/src/program.ts index 02094de4d..2cad68d0f 100644 --- a/packages/provider-html-local/src/program.ts +++ b/packages/provider-html-local/src/program.ts @@ -5,6 +5,7 @@ import type { ManagedProgram, ManagedProgramState } from "@hypit/runtime-local/e import { browserCacheDirectory, browserDownloadBaseUrl, browserDownloadUrl, browserExecutablePath, configuredBrowserPath, requireBrowserExecutable, selectedBrowserVersion } from "./browser.js"; import type { BrowserOptions } from "./browser.js"; import { runProcess } from "./process.js"; +import { requireElectronExecutable } from "./electron.js"; type MediaToolVersion = | { readonly state: "ready"; readonly version: string } @@ -47,6 +48,7 @@ export function localHtmlBrowserProgram( readonly nodePath: string; readonly ffprobePath: string; readonly ffmpegPath?: string; + readonly electronPath?: string; }, ): ManagedProgram { const version = selectedBrowserVersion(input); @@ -72,6 +74,10 @@ export function localHtmlBrowserProgram( }, } : {}), async probe(): Promise { + if (input.electronPath !== undefined) { + try { await requireElectronExecutable(input.electronPath); } + catch (error) { return { state: "down", detail: error instanceof Error ? error.message : String(error) }; } + } const browser = await probeBrowser(); if (browser.state !== "ready") return browser; return await probeHtmlMediaTools(input.ffprobePath, input.ffmpegPath); diff --git a/packages/provider-html-local/src/provider.ts b/packages/provider-html-local/src/provider.ts index 1f4418902..4a88fa487 100644 --- a/packages/provider-html-local/src/provider.ts +++ b/packages/provider-html-local/src/provider.ts @@ -16,7 +16,7 @@ export type CreateLocalHtmlProviderOptions = HtmlExecutionOptions & { readonly nodePath?: string; /** Whole render requests admitted concurrently; independent of frame workers. */ readonly defaultConcurrency?: number; - /** Shared Chrome slots across Need executions using this pool. */ + /** Shared capture worker slots across Need executions using this pool. */ readonly browserCapacity?: number; }; diff --git a/packages/provider-html-local/src/render.ts b/packages/provider-html-local/src/render.ts index e9d6604f2..2ad7bfb61 100644 --- a/packages/provider-html-local/src/render.ts +++ b/packages/provider-html-local/src/render.ts @@ -1,7 +1,7 @@ import { createReadStream } from "node:fs"; import { mkdtemp, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; -import { join } from "node:path"; +import { join, resolve } from "node:path"; import type { EndpointInvocationContext } from "@hypit/hypit/endpoint"; import { sealTimelineVisual } from "@hypit/media"; import type { MediaFrameRange, TimelineVisual } from "@hypit/media"; @@ -17,6 +17,7 @@ import { autoWorkerLimit, renderWorkerLimit } from "./concurrency.js"; import { requestedFrameRanges } from "./sampling.js"; import { browserCacheDirectory, browserExecutablePath, configuredBrowserPath, requireBrowserExecutable, selectedBrowserVersion } from "./browser.js"; import { htmlArtifactRelativePath, stageHtmlProgram } from "./stage.js"; +import { requireElectronExecutable } from "./electron.js"; export type HtmlRasterProgress = | { readonly phase: "staging" | "encoding" | "storing"; readonly elapsedMs: number } @@ -44,7 +45,10 @@ export function resolveExecutionOptions(options: HtmlExecutionOptions) { assert(["draft", "standard", "high"].includes(quality), "HTML renderer quality is invalid"); const browserGpu = options.browserGpu ?? "hardware"; assert(["auto", "software", "hardware"].includes(browserGpu), "HTML renderer browserGpu is invalid"); + assert(options.electronPath === undefined || options.electronPath.trim().length > 0, "HTML renderer electronPath must not be empty"); + assert(options.electronPath === undefined || browserGpu !== "software", "Shared-texture export requires browserGpu auto or hardware"); return { + electronPath: options.electronPath === undefined ? undefined : resolve(options.electronPath), chromePath: configuredBrowserPath(options), browserVersion: selectedBrowserVersion(options), browserCacheDirectory: browserCacheDirectory(options), @@ -85,8 +89,10 @@ export async function rasterizeHtmlFrames(request: HtmlFrameRequest, options: Ht } async function capture(request: HtmlRasterRequest | HtmlFrameRequest, options: HtmlRasterOptions): Promise { - const config = { ...resolveExecutionOptions(options), chromePath: browserExecutablePath(options) }; const frames = "frames" in request ? request.frames : undefined; + const execution = resolveExecutionOptions(options); + const config = { ...execution, + chromePath: frames === undefined && execution.electronPath !== undefined ? "" : browserExecutablePath(options) }; const document = request.program; const range = frames === undefined ? (request as HtmlRasterRequest).range ?? { startFrame: 0, endFrameExclusive: document.frameCount } @@ -115,12 +121,14 @@ async function capture(request: HtmlRasterRequest | HtmlFrameRequest, options: H }; try { signal.throwIfAborted(); - await requireBrowserExecutable(config.chromePath, config.browserVersion); + const gpu = frames === undefined && config.electronPath !== undefined; + if (gpu) await requireElectronExecutable(config.electronPath!); + else await requireBrowserExecutable(config.chromePath, config.browserVersion); await options.onDiagnostic?.({ level: "info", message: - `Capture ${frameCount} frames; workers ${config.workers} (limit ${renderWorkerLimit(config, frameCount, document.frameRate.numerator / document.frameRate.denominator)}); opaque fast PNG; GPU ${config.browserGpu}; browser ${config.chromePath}` + `Capture ${frameCount} frames; workers ${config.workers} (limit ${renderWorkerLimit(config, frameCount, document.frameRate.numerator / document.frameRate.denominator)}); ${gpu ? "shared GPU texture" : "opaque fast PNG"}; GPU ${config.browserGpu}; browser ${gpu ? config.electronPath : config.chromePath}` + `; artifact staging concurrency ${config.artifactStagingConcurrency}` + `; decoded source budget ${config.maxDecodedSourceBytes} bytes` - + (frames === undefined ? `; quality ${config.quality}; ordered FFmpeg pipe; pending PNG budget ${config.maxPendingFrameBytes} bytes; encoder ${config.ffmpegPath}` : "; PNG output") }); + + (frames === undefined ? `; quality ${config.quality}; ${gpu ? "WebCodecs H.264 and FFmpeg mux" : "ordered FFmpeg pipe"}; pending frame budget ${config.maxPendingFrameBytes} bytes; FFmpeg ${config.ffmpegPath}` : "; PNG output") }); options.onProgress?.({ phase: "staging", elapsedMs: 0 }); work = await mkdtemp(join(tmpdir(), "hypit-html-local-")); const read: import("./stage.js").HtmlArtifactReader = async (artifact, readSignal) => { diff --git a/packages/provider-html-local/src/server.ts b/packages/provider-html-local/src/server.ts index 753edce4c..abbed0c0a 100644 --- a/packages/provider-html-local/src/server.ts +++ b/packages/provider-html-local/src/server.ts @@ -11,11 +11,15 @@ const mediaTypes: Readonly> = { }; /** Serve one staged program and its lazily decoded frames to local capture pages. */ -export async function serveHtmlProgram(root: string): Promise<{ readonly url: string; close(): Promise }> { +export async function serveHtmlProgram(root: string, pages: ReadonlyMap = new Map()): Promise<{ readonly url: string; close(): Promise }> { const base = resolve(root); const server = createServer(async (request, response) => { try { const pathname = decodeURIComponent(new URL(request.url ?? "/", "http://127.0.0.1").pathname); + const page = pages.get(pathname); + if (page !== undefined) { + response.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }).end(page); return; + } const path = resolve(base, pathname === "/" ? "index.html" : `.${pathname}`); if (path !== base && !path.startsWith(`${base}${sep}`)) { response.writeHead(403).end(); return; @@ -34,7 +38,9 @@ export async function serveHtmlProgram(root: string): Promise<{ readonly url: st "Accept-Ranges": "bytes", ...(range === null ? {} : { "Content-Range": `bytes ${start}-${end}/${info.size}` }), }); - createReadStream(path, { start, end }).on("error", () => response.destroy()).pipe(response); + const stream = createReadStream(path, { start, end }); + response.once("close", () => stream.destroy()); + stream.on("error", () => response.destroy()).pipe(response); } catch (error) { response.writeHead((error as NodeJS.ErrnoException).code === "ENOENT" ? 404 : 500).end(); } @@ -47,6 +53,10 @@ export async function serveHtmlProgram(root: string): Promise<{ readonly url: st if (address === null || typeof address === "string") throw new Error("HTML Program server did not bind a TCP port"); return { url: `http://127.0.0.1:${address.port}/`, - close: () => new Promise((resolveClose, reject) => server.close(error => error ? reject(error) : resolveClose())), + close: () => new Promise((resolveClose, reject) => { + server.close(error => error ? reject(error) : resolveClose()); + // Capture has finished; browser media requests can still hold keep-alive sockets. + server.closeAllConnections(); + }), }; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8abdd0f4b..119a5268f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1941,6 +1941,9 @@ importers: '@hypit/visual-track': specifier: workspace:* version: link:../visual-track + electron: + specifier: 45.0.0 + version: 45.0.0 sharp: specifier: 0.35.4 version: 0.35.4(@types/node@24.10.1) @@ -3144,6 +3147,14 @@ packages: search-insights: optional: true + '@electron-internal/extract-zip@1.0.6': + resolution: {integrity: sha512-/aQLgFDdCSnwChfUwqekSmE1q2dCTzhUy5saIHBb5vyYn5LzWaKPaIz1wM0lDMCZYBoIG4svJNvAJb6vkI8BuA==} + engines: {node: '>=22.12.0'} + + '@electron/get@5.1.0': + resolution: {integrity: sha512-3kSBtG8ObcTVfXanm5vVJ6UnBLEVmVsRk1M+vGqCuMBV+XLCbJYuWQful+yIy0GQDsSlK0kHEriEHn7SPk4EnA==} + engines: {node: '>=22.12.0'} + '@emnapi/runtime@1.11.3': resolution: {integrity: sha512-Xz4Tpyki7XyrpbUK1jR1AhdAdaXyhhY4lZ3neLodmhpuWfy2PAQN5B46sAiU4liOXGLkHypn/qU+jvfWSCYYLA==} @@ -4189,6 +4200,15 @@ packages: csstype@3.2.3: resolution: {integrity: sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==} + debug@4.4.3: + resolution: {integrity: sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==} + engines: {node: '>=6.0'} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + dequal@2.0.3: resolution: {integrity: sha512-0je+qPKHEMohvfRTCEo3CrPG6cAzAYgmzKyxRiYSSDkS6eGJdyVJm7WaYA5ECaAD9wLB2T4EEeymA5aFVcYXCA==} engines: {node: '>=6'} @@ -4232,6 +4252,11 @@ packages: resolution: {integrity: sha512-qI4JLRKnSzqFqr7hAlS5xQDusBCjKSEG4t4+7aNrIQMHBcsC2TGEhuyABJdYkgSewL57PNLYEiibY2iPKhKpaA==} engines: {node: '>=20.19.0'} + electron@45.0.0: + resolution: {integrity: sha512-yjz4uhMBcO6L5X7CWgRMobf2EMJPnb9j4zuf+nu/TU/6E4Sqfx7fn/a2eACo4qngjml/0JnsoZhxYroJLxLd2w==} + engines: {node: '>= 22.12.0'} + hasBin: true + emoji-regex-xs@1.0.0: resolution: {integrity: sha512-LRlerrMYoIDrT6jgpeZ2YYl/L8EulRTt5hQcYjy5AInh7HWXKimpqx68aknBFpGL2+/IcogTcaydJEgaTmOpDg==} @@ -4250,6 +4275,10 @@ packages: resolution: {integrity: sha512-zwfzJecQ/Uej6tusMqwAqU/6KL2XaB2VZ2Jg54Je6ahNBGNH6Ek6g3jjNCF0fG9EWQKGZNddNjU5F1ZQn/sBnA==} engines: {node: '>=20.19.0'} + env-paths@3.0.0: + resolution: {integrity: sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A==} + engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + esbuild@0.21.5: resolution: {integrity: sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw==} engines: {node: '>=12'} @@ -4295,6 +4324,9 @@ packages: get-tsconfig@4.14.0: resolution: {integrity: sha512-yTb+8DXzDREzgvYmh6s9vHsSVCHeC0G3PI5bEXNBHtmshPnO+S5O7qgLEOn0I5QvMy6kpZN8K1NKGyilLb93wA==} + graceful-fs@4.2.11: + resolution: {integrity: sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==} + hast-util-to-html@9.0.5: resolution: {integrity: sha512-OguPdidb+fbHQSU4Q4ZiLKnzWo8Wwsf5bZfbvu7//a9oTYoqD/fWpe96NuHkoS9h0ccGOTe0C4NGXdtS0iObOw==} @@ -4440,6 +4472,9 @@ packages: resolution: {integrity: sha512-snEhs+6G5Tjd4I7tLCDOaoln2RgE0bD19RzEKgvgK2hZ5VKy3MpLhLTZ2fWpXSTg4K2cyPwp+VHATFJhxfnOeA==} engines: {node: '>=18.0.0'} + ms@2.1.3: + resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} + nanoid@3.3.16: resolution: {integrity: sha512-bzlKTyNJ7+LdGIIwy8ijFpIqEQIvafahV7eYykJ8Cvh42EdJeODoJ6gUJXpQJvej1BddH8OqTXZNE/KfbWAu8Q==} engines: {node: ^10 || ^12 || ^13.7 || ^14 || >=15.0.1} @@ -4483,6 +4518,10 @@ packages: preact-render-to-string: optional: true + progress@2.0.3: + resolution: {integrity: sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA==} + engines: {node: '>=0.4.0'} + property-information@7.2.0: resolution: {integrity: sha512-IAtzIB6sUiWaJYrX9smp3V46pBGbBeLFRGdh25kg1334VcBlD8HzhPeNIWQH9zhGmo2itIe25EHt9dQP7G5hmg==} @@ -4568,6 +4607,10 @@ packages: resolution: {integrity: sha512-yDPMNjp4WyfYBkHnjIRLfca1i6KMyGCtsVgoKe/z1+6vukgaENdgGBZt+ZmKPc4gavvEZ5OgHfHdrazhgNyG7w==} engines: {node: '>=12'} + sumchecker@3.0.1: + resolution: {integrity: sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg==} + engines: {node: '>= 8.0'} + superjson@2.2.6: resolution: {integrity: sha512-H+ue8Zo4vJmV2nRjpx86P35lzwDT3nItnIsocgumgr0hHMQ+ZGq5vrERg9kJBo5AWGmxZDhzDo+WVIJqkB0cGA==} engines: {node: '>=16'} @@ -4604,6 +4647,10 @@ packages: undici-types@7.16.0: resolution: {integrity: sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw==} + undici@7.30.1: + resolution: {integrity: sha512-03KrMbjIFDWQR5RNPrSw3bd1r12JpyFQl9/yxjUbJtqkjmllqC5lFjZh6ujeuJCKr8EOQvRW872Oy/az+nLYxg==} + engines: {node: '>=20.18.1'} + unist-util-is@6.0.1: resolution: {integrity: sha512-LsiILbtBETkDz8I9p1dQ0uyRUWuaQzd/cuEeS1hoRSyW5E5XGmTzlwY1OrNzzakGowI9Dr/I8HVaw4hTtnxy8g==} @@ -4915,6 +4962,21 @@ snapshots: transitivePeerDependencies: - '@algolia/client-search' + '@electron-internal/extract-zip@1.0.6': {} + + '@electron/get@5.1.0': + dependencies: + debug: 4.4.3 + env-paths: 3.0.0 + graceful-fs: 4.2.11 + progress: 2.0.3 + semver: 7.8.5 + sumchecker: 3.0.1 + optionalDependencies: + undici: 7.30.1 + transitivePeerDependencies: + - supports-color + '@emnapi/runtime@1.11.3': dependencies: tslib: 2.8.1 @@ -5617,6 +5679,10 @@ snapshots: csstype@3.2.3: {} + debug@4.4.3: + dependencies: + ms: 2.1.3 + dequal@2.0.3: {} detect-libc@2.1.2: {} @@ -5663,6 +5729,14 @@ snapshots: domelementtype: 3.0.0 domhandler: 6.0.1 + electron@45.0.0: + dependencies: + '@electron-internal/extract-zip': 1.0.6 + '@electron/get': 5.1.0 + '@types/node': 24.10.1 + transitivePeerDependencies: + - supports-color + emoji-regex-xs@1.0.0: {} emoji-regex@10.6.0: {} @@ -5673,6 +5747,8 @@ snapshots: entities@8.0.0: {} + env-paths@3.0.0: {} + esbuild@0.21.5: optionalDependencies: '@esbuild/aix-ppc64': 0.21.5 @@ -5751,6 +5827,8 @@ snapshots: dependencies: resolve-pkg-maps: 1.0.0 + graceful-fs@4.2.11: {} + hast-util-to-html@9.0.5: dependencies: '@types/hast': 3.0.5 @@ -5906,6 +5984,8 @@ snapshots: modern-tar@0.8.5: {} + ms@2.1.3: {} + nanoid@3.3.16: {} nanoid@3.3.18: {} @@ -5940,6 +6020,8 @@ snapshots: preact@10.29.8: {} + progress@2.0.3: {} + property-information@7.2.0: {} puppeteer-core@25.10.0: @@ -6105,6 +6187,12 @@ snapshots: dependencies: ansi-regex: 6.2.2 + sumchecker@3.0.1: + dependencies: + debug: 4.4.3 + transitivePeerDependencies: + - supports-color + superjson@2.2.6: dependencies: copy-anything: 4.0.5 @@ -6136,6 +6224,9 @@ snapshots: undici-types@7.16.0: {} + undici@7.30.1: + optional: true + unist-util-is@6.0.1: dependencies: '@types/unist': 3.0.3 From a78652cf9ab3b4ce0f7a469bb3a499fd49fec47b Mon Sep 17 00:00:00 2001 From: rpone Date: Sun, 25 Oct 2026 01:07:36 +0800 Subject: [PATCH 02/16] perf(html-local): decode source video directly in GPU scenes Reuse browser video decoders and bounded sample caches for forward, reverse, held, and looping sources. Composite VideoSamples through Canvas layers before shared-texture encoding. Preserve source color metadata for planar alpha samples. Validated the 600-frame video project with independent timeline and source-frame barcodes, transparent media composition, tiny-cache reverse sampling, existing provider checks, and an unpacked package render. --- packages/provider-html-local/README.md | 34 ++-- packages/provider-html-local/package.json | 2 + .../provider-html-local/src/browser-video.mjs | 166 ++++++++++++++++++ packages/provider-html-local/src/capture.ts | 27 +-- .../provider-html-local/src/gpu-capture.ts | 18 +- .../provider-html-local/src/opaque-capture.ts | 11 +- packages/provider-html-local/src/server.ts | 3 +- pnpm-lock.yaml | 23 +++ 8 files changed, 253 insertions(+), 31 deletions(-) create mode 100644 packages/provider-html-local/src/browser-video.mjs diff --git a/packages/provider-html-local/README.md b/packages/provider-html-local/README.md index dd5cb7455..ba8c6e8b3 100644 --- a/packages/provider-html-local/README.md +++ b/packages/provider-html-local/README.md @@ -6,13 +6,13 @@ Trusted local implementation of: - `@hypit/html-program@1#rasterize-frames` → ordered PNG BlobRefs. The Provider stages a typed `HtmlProgram`, applies absolute integer frames through -`window.__hypitFrameProgram`, and supplies compiler-declared video samples from a -bounded FFmpeg decode store. It supports two video export configurations: +`window.__hypitFrameProgram`, and supplies compiler-declared video samples. +It supports two video export configurations: -| Configuration | Capture and encoding | -| --- | --- | -| `chromePath`, or managed Chrome selection | Puppeteer PNG capture and ordered FFmpeg H.264 encoding | -| `electronPath` | Electron shared GPU textures, Canvas cropping, WebCodecs H.264 encoding, FFmpeg MP4 muxing | +| Configuration | Source video | Capture and encoding | +| --- | --- | --- | +| `chromePath`, or managed Chrome selection | Bounded FFmpeg PNG decode store | Puppeteer PNG capture and ordered FFmpeg H.264 encoding | +| `electronPath` | Mediabunny demuxing, WebCodecs decoding, Canvas video layers | Electron shared GPU textures, Canvas cropping, WebCodecs H.264 encoding, FFmpeg MP4 muxing | PNG frame requests use the selected Chrome browser. Filesystem staging, source probing and decoded-frame leases belong to this Provider. @@ -33,8 +33,8 @@ The Endpoint facet is `rasterize`. Runtime configuration owns execution policy: For GPU export, `auto` uses the admitted worker ceiling, limited by the requested frame span. PNG capture adjusts its active workers using measured throughput. GPU export requires `browserGpu: "auto"` or `"hardware"`, even canvas dimensions, -shared-texture support and a supported WebCodecs H.264 configuration. Unsupported -configurations fail with a diagnostic. +shared-texture support, browser support for each source codec and a supported +WebCodecs H.264 configuration. Unsupported configurations fail with a diagnostic. The Chrome recommendation is stored as `hypit.htmlBrowser.version`. Managed Chrome defaults to `~/.cache/hypit/html-rasterizer/chrome`; its ManagedProgram state is @@ -72,8 +72,22 @@ Linux additionally needs the display environment required by Electron. A continuous request retains original absolute frame identities. A sparse PNG request accepts strictly increasing, non-contiguous frames. Workers can start at any requested frame. The compiler's `frameSources` plan determines reverse, hold, -loop and rational-rate video sampling. Decoded samples are leased under -`maxDecodedSourceBytes`; their images replace compiler-owned native video pixels. +loop and rational-rate video sampling. PNG capture leases decoded images under +`maxDecodedSourceBytes` and places them over compiler-owned video elements. + +GPU capture decodes source video inside each scene renderer. Forward sampling +reuses a decoder; reverse sampling decodes bounded forward intervals into a +sample cache. Holds and loops reuse cached samples. Canvas layers retain the +video elements' styles and placement. Inactive sources release their decoders. +`maxDecodedSourceBytes` is divided across workers and bounds retained sample +bytes, allowing one oversized sample per worker. Decoder queues, demuxer caches +and compositor surfaces require additional memory. Worker count also multiplies +source decoding; more workers can reduce throughput for video-heavy projects. + +Source color tags are preserved. Untagged SDR uses a BT.601 matrix and sRGB +transfer. Planar alpha samples retain source color metadata through an in-memory +copy before Canvas composition. Mediabunny 1.62.1 is distributed under +[MPL-2.0](https://github.com/Vanilagy/mediabunny/blob/main/LICENSE). The GPU path renders each Program in a fixed-size same-origin iframe. A separate 32-bit frame identifier below the authored viewport identifies the accepted diff --git a/packages/provider-html-local/package.json b/packages/provider-html-local/package.json index 4d0770119..e5b5df29b 100644 --- a/packages/provider-html-local/package.json +++ b/packages/provider-html-local/package.json @@ -11,6 +11,7 @@ "dist", "src/gpu-worker.cjs", "src/gpu-encoder.cjs", + "src/browser-video.mjs", "README.md" ], "hypit": { @@ -23,6 +24,7 @@ "@hypit/html-program": "workspace:^0.1.0", "@hypit/media": "workspace:^0.1.0", "@puppeteer/browsers": "3.2.2", + "mediabunny": "1.62.1", "puppeteer-core": "25.10.0", "tsx": "4.21.0" }, diff --git a/packages/provider-html-local/src/browser-video.mjs b/packages/provider-html-local/src/browser-video.mjs new file mode 100644 index 000000000..366398485 --- /dev/null +++ b/packages/provider-html-local/src/browser-video.mjs @@ -0,0 +1,166 @@ +/** Installed inside the scene renderer; decoded samples stay in the browser. */ +export async function installBrowserVideo(moduleUrl, maxBytes) { + const media = await import(moduleUrl); + const sources = new Map(); + const cache = new Map(); + let resident = 0, samplesRead = 0, decoderStarts = 0, cacheHits = 0, peak = 0; + const open = (url) => { + let pending = sources.get(url); + if (pending === undefined) { + pending = (async () => { + const input = new media.Input({ source: new media.UrlSource(url), formats: media.ALL_FORMATS }); + const track = await input.getPrimaryVideoTrack(); + if (track === null || !await track.canDecode()) { + input.dispose(); + throw new Error(`Browser video decoder does not support ${url}`); + } + // Untagged SDR follows the FFmpeg source path's BT.601 matrix and sRGB transfer. + const decoderConfig = track.getDecoderConfig.bind(track); + track.getDecoderConfig = async () => { + const config = await decoderConfig(); + if (!config) return config; + return { ...config, colorSpace: { + primaries: config.colorSpace?.primaries ?? "bt709", + transfer: config.colorSpace?.transfer ?? "iec61966-2-1", + matrix: config.colorSpace?.matrix ?? "smpte170m", + fullRange: config.colorSpace?.fullRange ?? false, + } }; + }; + return { input, sink: new media.VideoSampleSink(track, { hardwareAcceleration: "prefer-hardware" }), + colorSpace: (await track.getDecoderConfig())?.colorSpace, + firstTimestamp: await track.getFirstTimestamp(), width: track.displayWidth, height: track.displayHeight }; + })(); + sources.set(url, pending); + } + return pending; + }; + const retain = (key, sample, protectedKey) => { + const previous = cache.get(key); + if (previous !== undefined) { + sample.close(); + return; + } + const bytes = sample.allocationSize(); + while (resident + bytes > maxBytes) { + const victim = [...cache.keys()].find(key => key !== protectedKey); + if (victim === undefined) { + if (key !== protectedKey) { sample.close(); return; } + break; + } + const entry = cache.get(victim); + entry.sample.close(); + resident -= entry.bytes; + cache.delete(victim); + } + cache.set(key, { sample, bytes }); + resident += bytes; + peak = Math.max(peak, resident); + }; + const readSample = async (source, timestamp) => { + if (!source.iterator || timestamp < source.current?.timestamp || timestamp - source.current?.timestamp > 2) { + if (source.iterator) await source.iterator.return(); + source.current?.close(); source.next?.close(); + decoderStarts++; + source.iterator = source.sink.samples(timestamp); + source.current = (await source.iterator.next()).value; + source.next = (await source.iterator.next()).value; + if (source.current) samplesRead++; + if (source.next) samplesRead++; + } + while (source.next && source.next.timestamp <= timestamp) { + source.current?.close(); + source.current = source.next; + source.next = (await source.iterator.next()).value; + if (source.next) samplesRead++; + } + if (!source.current) throw new Error(`Browser video has no sample at ${timestamp}`); + const sample = source.current; + // Planar alpha samples need the source color tags before RGB conversion. + if (/^I\d{3}A/.test(sample.format)) { + const bytes = new Uint8Array(sample.allocationSize()); + const layout = await sample.copyTo(bytes); + return new media.VideoSample(bytes, { format: sample.format, codedWidth: sample.codedWidth, + codedHeight: sample.codedHeight, timestamp: sample.timestamp, duration: sample.duration, + rotation: sample.rotation, flip: sample.flip, visibleRect: sample.visibleRect, + displayWidth: sample.displayWidth, displayHeight: sample.displayHeight, + colorSpace: source.colorSpace, layout }); + } + return sample.clone(); + }; + const w = window; + w.__hypitVideoStats = () => ({ samplesRead, decoderStarts, cacheHits, peak }); + w.__hypitDrawVideoSources = async (updates) => { + const active = new Set(updates.map(update => update.id)); + const activeUrls = new Set(updates.map(update => update.url)); + for (const [url, pending] of sources) { + if (activeUrls.has(url)) continue; + const source = await pending; + source.current?.close(); source.next?.close(); + await source.iterator?.return(); + source.input.dispose(); sources.delete(url); + } + for (const canvas of Array.from(document.querySelectorAll("[data-hypit-source-frame-canvas]"))) { + if (!active.has(canvas.dataset.hypitSourceFrameCanvas) && canvas.style.display !== "none") { + canvas.style.display = "none"; + canvas.width = canvas.height = 1; + } + } + for (const update of updates) { + const selection = update.sample; + if (selection === undefined) + throw new Error("Browser video sampling metadata is missing"); + const source = await open(update.url); + const keyFor = (frame) => `${update.url}:${frame}`; + const key = keyFor(selection.frame); + if (!cache.has(key)) { + // Reverse playback decodes a bounded forward interval, then reuses its samples. + const count = selection.reverse ? Math.max(1, Math.min(64, + Math.floor(maxBytes / Math.max(1, activeUrls.size) / (source.sampleBytes ?? source.width * source.height * 4)))) : 1; + const start = selection.reverse ? Math.max(0, selection.frame - count + 1) : selection.frame; + const length = selection.reverse ? selection.frame - start + 1 : count; + const timestamps = Array.from({ length }, (_, index) => source.firstTimestamp + + (start + index + 0.5) * selection.fps.den / selection.fps.num); + for (let index = 0; index < timestamps.length; index++) { + const sample = await readSample(source, timestamps[index]); + source.sampleBytes = sample.allocationSize(); + retain(keyFor(start + index), sample, key); + } + } else cacheHits++; + const entry = cache.get(key); + if (entry === undefined) + throw new Error("Browser video cache lost a requested sample"); + cache.delete(key); + cache.set(key, entry); + const video = document.getElementById(update.id); + if (video === null) + throw new Error(`HTML Program source element ${update.id} is missing`); + let canvas = document.querySelector(`[data-hypit-source-frame-canvas="${CSS.escape(update.id)}"]`); + if (canvas === null) { + canvas = document.createElement("canvas"); + canvas.dataset.hypitSourceFrameCanvas = update.id; + video.insertAdjacentElement("afterend", canvas); + } + canvas.className = video.className; + canvas.style.cssText = video.style.cssText; + canvas.style.visibility = video.dataset.hypitAuthoredVisibility ?? video.style.visibility; + canvas.style.display = video.style.display; + if (canvas.width !== source.width) + canvas.width = source.width; + if (canvas.height !== source.height) + canvas.height = source.height; + video.style.visibility = "hidden"; + const context = canvas.getContext("2d"); + if (context === null) + throw new Error("Browser video canvas could not be created"); + context.clearRect(0, 0, canvas.width, canvas.height); + entry.sample.draw(context, 0, 0, canvas.width, canvas.height); + } + }; + window.addEventListener("pagehide", () => { + for (const entry of cache.values()) + entry.sample.close(); + cache.clear(); + for (const source of sources.values()) + void source.then(value => { value.current?.close(); value.next?.close(); void value.iterator?.return(); value.input.dispose(); }); + }, { once: true }); +} diff --git a/packages/provider-html-local/src/capture.ts b/packages/provider-html-local/src/capture.ts index 7fa539a8d..ec649447f 100644 --- a/packages/provider-html-local/src/capture.ts +++ b/packages/provider-html-local/src/capture.ts @@ -104,7 +104,7 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort metadata: Awaited> }>(); // Chrome needs dimensions before initialization. Pixel frames themselves // are decoded lazily for the screenshots that lease them. - for (const source of plannedSources) { + for (const source of useGpu ? [] : plannedSources) { const path = resolve(work, source.src); assert(path.startsWith(`${work}${sep}`), "HTML renderer source is outside the staged project"); const metadata = await probeSourceVideo({ executable: ffprobePath, path, @@ -113,7 +113,7 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort sources.set(source.src, { path, fps: source.fps, metadata }); } let extraction = 0; - sourceFrameStore = new SourceFrameStore({ maxBytes: config.maxDecodedSourceBytes, signal, + if (!useGpu) sourceFrameStore = new SourceFrameStore({ maxBytes: config.maxDecodedSourceBytes, signal, decode: (src, window) => { const source = sources.get(src); assert(source !== undefined, `HTML renderer source ${src} is outside the requested render`); @@ -140,11 +140,11 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort } return payloads; }; - const gpuPages = useGpu ? gpuCapturePages(document.canvas) : undefined; + const gpuPages = useGpu ? await gpuCapturePages(document.canvas) : undefined; server = await serveHtmlProgram(work, gpuPages?.pages); if (gpuPages !== undefined) { gpu = await stage("GPU encoder initialization", config.initializationTimeoutMs, - () => createGpuCapture(input, server!.url, gpuPages, controller)); + () => createGpuCapture(input, server!.url, gpuPages, controller, limit)); } const serverUrl = server.url; const outputFrames = join(work, "frames"); @@ -194,6 +194,7 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort try { signal.throwIfAborted(); let browserPid: number | undefined; + let videoStats: (() => Promise) | undefined; let captureFrame: (frame: number, sources: readonly SourceFrameUpdate[], ordinal: number, prepared: () => void) => Promise>>>>; if (gpu !== undefined) { @@ -201,6 +202,7 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort const gpuWorker = await stage(`worker ${worker} initialization`, config.initializationTimeoutMs, () => gpu!.createWorker()); browserPid = gpuWorker.browserPid; captureFrame = gpuWorker.capture; + videoStats = gpuWorker.stats; } else { session = await puppeteer.launch({ executablePath: config.chromePath, headless: true, defaultViewport: { width: document.canvas.width, height: document.canvas.height, deviceScaleFactor: 1 }, @@ -251,19 +253,21 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort const preferredEnd = useGpu ? Math.min(frameCount, index + Math.max(1, Math.round(fps.num / fps.den))) : batch.endFrameExclusive; const preferred = sourceDemand(frameRanges(index, preferredEnd)); const sourceAcquireStarted = performance.now(); - const sourceLease = await sourceFrameStore!.acquire(required, preferred); + const sourceLease = useGpu ? undefined : await sourceFrameStore!.acquire(required, preferred); sourceAcquireWorkerMs += performance.now() - sourceAcquireStarted; let captured: Awaited>; try { - const payloads = activeFramePayloads(frame); - const sourceUpdates = [...payloads].map(([id, payload]) => ({ id, - url: new URL(relative(work, payload.framePath).split(sep).map(encodeURIComponent).join("/"), serverUrl).href })); + const sourceUpdates: SourceFrameUpdate[] = useGpu + ? slotIndex.at(frame).map(slot => ({ id: slot.id, url: new URL(slot.src, serverUrl).href, + sample: { frame: sourceFrameAt(slot, frame), fps: slot.sourceFps, reverse: slot.sourceRate.numerator < 0n } })) + : [...activeFramePayloads(frame)].map(([id, payload]) => ({ id, + url: new URL(relative(work, payload.framePath).split(sep).map(encodeURIComponent).join("/"), serverUrl).href })); captured = await stage(`worker ${worker} frame ${frame}`, config.frameTimeoutMs, - () => captureFrame(frame, sourceUpdates, index, () => sourceLease.release())); + () => captureFrame(frame, sourceUpdates, index, () => sourceLease?.release())); } finally { // Output submission may block behind an earlier frame. Never // retain source leases across that independent backpressure. - sourceLease.release(); + sourceLease?.release(); } if (input.frames !== undefined) { capturedBytes += captured.buffer.byteLength; @@ -295,6 +299,7 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort } console.log(`Capture worker ${worker}: ${completed} frames; startup ${Math.round(setupMs)} ms; ` + `seek ${Math.round(timing.seekMs)} ms; prepare ${Math.round(timing.prepareMs)} ms; ${useGpu ? "texture and encode" : "PNG"} ${Math.round(timing.screenshotMs)} ms`); + if (videoStats !== undefined) console.log(`Capture worker ${worker} video: ${JSON.stringify(await videoStats())}`); onProgress({ phase: "worker-complete", worker, completed, browserPid, elapsedMs: elapsedMs() }); } catch (error) { controller.abort(error); @@ -324,7 +329,7 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort if (failure !== undefined) throw failure; signal.throwIfAborted(); await Promise.all(closing.values()); - console.log(`Capture source working set: ${sourceTotal} unique frames demanded; decoded ${sourceFrameStore.decodedFrames} frames; ` + if (sourceFrameStore !== undefined) console.log(`Capture source working set: ${sourceTotal} unique frames demanded; decoded ${sourceFrameStore.decodedFrames} frames; ` + `redecoded ${Math.max(0, sourceFrameStore.decodedFrames - sourceTotal)}; decoder runs ${extraction}; ` + `decoded bytes ${sourceFrameStore.decodedBytes}; largest frame ${sourceFrameStore.largestDecodedFrameBytes} bytes; ` + `peak ${sourceFrameStore.peakResidentBytes} bytes; budget ${config.maxDecodedSourceBytes} bytes; ` diff --git a/packages/provider-html-local/src/gpu-capture.ts b/packages/provider-html-local/src/gpu-capture.ts index 959ee33e0..5f5564de9 100644 --- a/packages/provider-html-local/src/gpu-capture.ts +++ b/packages/provider-html-local/src/gpu-capture.ts @@ -1,5 +1,6 @@ import { createRequire } from "node:module"; import { createWriteStream } from "node:fs"; +import { readFile } from "node:fs/promises"; import { finished } from "node:stream/promises"; import { randomUUID } from "node:crypto"; import { join } from "node:path"; @@ -11,9 +12,13 @@ import { createFramePreparer } from "./opaque-capture.js"; import type { SourceFrameUpdate } from "./opaque-capture.js"; import { assert } from "./process.js"; -export function gpuCapturePages({ width, height }: { width: number; height: number }) { +export async function gpuCapturePages({ width, height }: { width: number; height: number }) { const shell = `/__hypit_capture_${randomUUID()}.html`; const encoder = `/__hypit_encoder_${randomUUID()}.html`; + const decoder = `/__hypit_decoder_${randomUUID()}.mjs`; + const video = `/__hypit_video_${randomUUID()}.mjs`; + const bundle = await readFile(new URL("../../bundles/mediabunny.min.mjs", import.meta.resolve("mediabunny")), "utf8"); + const videoModule = await readFile(new URL("../src/browser-video.mjs", import.meta.url), "utf8"); const html = `') }; +return Object.fromEntries(Object.entries({ + video: video.program, mg, mg4k, staggered: tiled(8), aligned: tiled(8,1920,1080,600,true), + accuracy: tiled(8,1920,1080,600,false,true), uhd: tiled(4,3840,2160,300), +}).map(([name,program]) => [name, { program, blobs: name.startsWith('mg') ? motion.blobs : blobs }])); +} + +function decode(file, filter, width) { + const result = spawnSync('ffmpeg', ['-v','error','-i',file,'-vf',filter,'-pix_fmt','gray','-f','rawvideo','pipe:1'], + { maxBuffer: 32*1024*1024, timeout: 120000 }); + if (result.status !== 0) throw Error('Identifier decode failed: ' + (result.error ?? result.stderr)); + if (result.stdout.length % width) throw Error('Truncated identifier decode'); + return result.stdout; +} + +// The supplied benchmark projects encode source frame IDs in the bottom-left of each source, +// and timeline IDs in the output. Coordinates match the 1080p fixtures used in PR #437. +export function verifyIdentifiers(file, kind, count) { + const errors = [], checks = []; + const record = (frame, channel, actual, expected) => { + if (actual !== expected) { if (errors.length < 20) errors.push({frame,channel,actual,expected}); } + return actual !== expected ? 1 : 0; + }; + let mismatches = 0, observations = 0; + if (kind === 'accuracy') { + const branches = Array.from({length:9},(_,i)=>`[s${i}]crop=320:8:${i<8?(i%3)*640:1600}:${i<8?Math.floor(i/3)*360+350:1068}[c${i}]`); + const graph = `[0:v]split=9${branches.map((_,i)=>`[s${i}]`).join('')};${branches.join(';')};${branches.map((_,i)=>`[c${i}]`).join('')}vstack=inputs=9,format=gray[out]`; + const r = spawnSync('ffmpeg',['-v','error','-i',file,'-filter_complex',graph,'-map','[out]','-f','rawvideo','pipe:1'],{maxBuffer:32*1024*1024,timeout:120000}); + if (r.status !== 0) throw Error('Layer identifier decode failed: '+(r.error ?? r.stderr)); + const frames = r.stdout.length/(320*72); + if (frames !== count) throw Error(`Identifier frame count ${frames}, expected ${count}`); + for(let f=0;fa-b); if(values[Math.floor(values.length/2)]>128)actual|=1<f], + ['source-a','crop=214:16:64:920',10,f=>f<150?f:f<300?449-f:f<450?120:60+Math.floor((f-450)*1.5)%60], + ['source-b','crop=80:6:1376:484',10,f=>f%360], + ] : [['timeline','crop=160:8:0:1072',10,f=>f]]; + for(const [channel,crop,bits,expected] of channels) { + const raw=decode(file,crop+`,scale=${bits}:1:flags=area`,bits), frames=raw.length/bits; + if(frames!==count)throw Error(`Identifier frame count ${frames}, expected ${count}`); + for(let f=0;f128)actual|=1< Date: Sun, 25 Oct 2026 05:08:37 +0800 Subject: [PATCH 11/16] docs(html-local): use recorded MG export duration --- scripts/benchmark-html-gpu-results.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/benchmark-html-gpu-results.md b/scripts/benchmark-html-gpu-results.md index 17cd5d7a4..d137b9c7b 100644 --- a/scripts/benchmark-html-gpu-results.md +++ b/scripts/benchmark-html-gpu-results.md @@ -13,7 +13,7 @@ Eight staggered/reversed 1080p video layers, 600 frames. Export seconds include The default configuration reduced the measured median from 22.287 to 16.097 seconds, approximately 28%. Three repetitions completed for both configurations; the planned five-repetition comparison remains incomplete. The 3 GiB median differs from the default by about 0.08 seconds in these observations. These results support retaining the current provisional defaults and do not establish an optimum. -Default configuration frame-ID checks passed: 5,400 timeline/source observations for eight layers, 1,800 observations for the two-video project and 600 timeline observations for complex MG. The ordinary video export took 12.13 seconds; the automatic-worker MG export took 18.62 seconds. Completed jobs left no observed descendants. The operator-cancelled job is recorded separately. +Default configuration frame-ID checks passed: 5,400 timeline/source observations for eight layers, 1,800 observations for the two-video project and 600 timeline observations for complex MG. The ordinary video export took 12.13 seconds; the automatic-worker MG export took 18.61 seconds. Completed jobs left no observed descendants. The operator-cancelled job is recorded separately. Memory is sampled every 250 ms across the export host and its descendants, including Electron processes that create separate process groups. RSS and macOS physical footprint are distinct from retained sample allocation and exclusive GPU memory. Only the corrected process-tree sampling run is summarized here. From feefcbb933a04fe4e513b19d3d9d265ee4788768 Mon Sep 17 00:00:00 2001 From: rpone Date: Mon, 26 Oct 2026 20:13:56 +0800 Subject: [PATCH 12/16] test(html-local): align budget expectations with configured defaults --- packages/provider-html-local/test/range.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/provider-html-local/test/range.test.ts b/packages/provider-html-local/test/range.test.ts index c89caa310..d847e2115 100644 --- a/packages/provider-html-local/test/range.test.ts +++ b/packages/provider-html-local/test/range.test.ts @@ -68,8 +68,8 @@ test("the render deadline is global while stage deadlines are explicit deploymen assert.equal(defaults.initializationTimeoutMs, undefined); assert.equal(defaults.frameTimeoutMs, undefined); assert.equal(defaults.artifactStagingConcurrency, 4); - assert.equal(defaults.maxPendingFrameBytes, 256 * 1024 * 1024); - assert.equal(defaults.maxDecodedSourceBytes, 1024 * 1024 * 1024); + assert.equal(defaults.maxPendingFrameBytes, 128 * 1024 * 1024); + assert.equal(defaults.maxDecodedSourceBytes, 2304 * 1024 * 1024); const explicit = resolveExecutionOptions({ initializationTimeoutMs: 31_000, frameTimeoutMs: 16_000, artifactStagingConcurrency: 3, maxPendingFrameBytes: 12_345, maxDecodedSourceBytes: 54_321 }); From ca0e9cbc2bea42a3256c3a784ea14fb6f153768d Mon Sep 17 00:00:00 2001 From: rpone Date: Mon, 26 Oct 2026 20:36:41 +0800 Subject: [PATCH 13/16] fix(html-local): separate benchmark fixtures and complete worker comparisons --- scripts/benchmark-html-gpu.md | 14 +++- scripts/benchmark-html-gpu.mjs | 22 ++---- scripts/html-gpu-benchmark-report.mjs | 102 ++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 19 deletions(-) create mode 100644 scripts/html-gpu-benchmark-report.mjs diff --git a/scripts/benchmark-html-gpu.md b/scripts/benchmark-html-gpu.md index 51fff284d..fd3c7db41 100644 --- a/scripts/benchmark-html-gpu.md +++ b/scripts/benchmark-html-gpu.md @@ -25,8 +25,8 @@ The script runs the complete sequence without further input: 1. Warm up and verify default video, eight-layer and MG frame identifiers. 2. Compare four source/queue configurations in five interleaved repetitions. 3. Compare 128, 256 and 512 MiB pending queues in five interleaved repetitions. -4. Screen MG worker counts, then repeat the two fastest candidates to five - observations each. Compare two/four workers for staggered video. +4. Screen MG worker counts 1 through 8 and automatic selection, then repeat + the two fastest candidates and the automatic baseline to five observations each. Compare two/four workers for staggered video. 5. Export aligned video, 4K video composition and scaled 4K MG with defaults. 6. Verify the fastest complete source/queue candidate's layer identifiers. 7. Verify two/four concurrent exports, cancellation and renderer termination @@ -51,7 +51,7 @@ same command resumes remaining jobs when source and input fingerprints match. Completed failures stay in the report; use a fresh output directory for a new experiment. Concurrent runners sharing an output directory are rejected. -Each job has a five-minute deadline followed by bounded process-group cleanup. +Each job has a five-minute deadline followed by bounded process-tree cleanup. Signals interrupt the current job and leave remaining work available for resume. Before each job, the runner requires at least 2 GiB of free output-volume space. Exports used for independent verification are saved; timing-only exports stay in @@ -60,3 +60,11 @@ A frame-ID preflight failure stops the experiment before parameter comparisons. The script produces local reports. PR descriptions and repository changes are published separately after reviewing those reports. + +To regenerate reports from a saved experiment without rendering: + +```sh +node scripts/html-gpu-benchmark-report.mjs dist/html-gpu-benchmark +``` + +The report groups each fixture separately and records resumed experiment sessions. diff --git a/scripts/benchmark-html-gpu.mjs b/scripts/benchmark-html-gpu.mjs index 464635565..f23159a76 100644 --- a/scripts/benchmark-html-gpu.mjs +++ b/scripts/benchmark-html-gpu.mjs @@ -7,6 +7,7 @@ import { parseArgs } from 'node:util'; import { createHash } from 'node:crypto'; import { cpus, totalmem, release } from 'node:os'; import { setTimeout as delay } from 'node:timers/promises'; +import { writeBenchmarkReport } from './html-gpu-benchmark-report.mjs'; const root = resolve(dirname(fileURLToPath(import.meta.url)), '..'); process.chdir(root); @@ -90,7 +91,7 @@ if(args.job){ const { loadProject }=await import('./html-gpu-benchmark-fixtures.mjs'); const fingerprint=createHash('sha256'); fingerprint.update(command('git',['rev-parse','HEAD']));fingerprint.update(command('git',['diff','HEAD','--'])); - for(const file of [script,resolve(root,'scripts/html-gpu-benchmark-fixtures.mjs')])fingerprint.update(await readFile(file)); + for(const file of [script,resolve(root,'scripts/html-gpu-benchmark-fixtures.mjs'),resolve(root,'scripts/html-gpu-benchmark-report.mjs')])fingerprint.update(await readFile(file)); for(const project of [videoProject,mgProject]){const data=await loadProject(project);fingerprint.update(JSON.stringify(data.program));for(const [artifact,bytes]of data.blobs){fingerprint.update(artifact.resource);fingerprint.update(bytes);}} fingerprint.update(String(reps));const identity=fingerprint.digest('hex'),stateFile=resolve(directory,'state.json'); let state;try{state=JSON.parse(await readFile(stateFile,'utf8'));}catch(e){if(e.code!=='ENOENT')throw e;} @@ -120,19 +121,7 @@ if(args.job){ function stats(){return{at:new Date().toISOString(),swap:command('sysctl',['vm.swapusage']),thermal:command('pmset',['-g','therm'])};} const median=xs=>{xs=[...xs].sort((a,b)=>a-b);return xs.length%2?xs[Math.floor(xs.length/2)]:(xs[xs.length/2-1]+xs[xs.length/2])/2;}; async function summarize(){ - const groups={}; - for(const result of state.results)for(const item of result.exports??[])if(item.group){ - const key=item.group+' / '+JSON.stringify(item.config);(groups[key]??=[]).push({...item,passed:result.passed,peakRssMiB:result.peakRssMiB,peakFootprintMiB:result.peakFootprintMiB});} - const rows=Object.entries(groups).map(([key,all])=>{const good=all.filter(x=>x.passed),times=good.map(x=>x.seconds);return{key,completed:good.length,failed:all.length-good.length, - medianSeconds:times.length?median(times):null,minSeconds:times.length?Math.min(...times):null,maxSeconds:times.length?Math.max(...times):null, - medianPeakRssMiB:good.length?median(good.map(x=>x.peakRssMiB)):null,medianPeakFootprintMiB:good.length&&readUsage?median(good.map(x=>x.peakFootprintMiB)):null, - decoderStarts:good.map(x=>x.videoStats?.reduce((s,v)=>s+v.decoderStarts,0)), - };}); - const summary={status:state.status??'running',head:state.head,host:state.host,repetitions:reps,completedJobs:state.results.length, - failedJobs:state.results.filter(r=>!r.passed).map(r=>r.id),rows}; - await json(resolve(directory,'summary.json'),summary); - const f=n=>n==null?'n/a':n.toFixed(3); - await writeFile(resolve(directory,'REPORT.md'),`# HTML GPU benchmark\n\nStatus: ${summary.status}. Head: ${state.head}. Repetitions: ${reps}.\n\nExport seconds include Provider preparation, capture, encode, mux, full decode validation and storage. Independent frame-ID checks run afterward. Memory is sampled every 250 ms across the isolated export host and its descendants during export, including resource setup during the export phase. Normal desktop activity remains uncontrolled. RSS and macOS physical footprint are distinct from retained sample allocation and exclusive GPU memory. Each job uses a fresh Node process; managed Electron download caches persist.\n\n| Group and configuration | Completed | Failed | Median seconds | Range, seconds | Median peak RSS, MiB | Median peak physical footprint, MiB |\n| --- | --- | --- | --- | --- | --- | --- |\n${rows.map(r=>`| ${r.key} | ${r.completed} | ${r.failed} | ${f(r.medianSeconds)} | ${f(r.minSeconds)} to ${f(r.maxSeconds)} | ${f(r.medianPeakRssMiB)} | ${f(r.medianPeakFootprintMiB)} |`).join('\n')}\n\nFailed jobs: ${summary.failedJobs.join(', ')||'none'}. Full diagnostics, frame-ID observations, resource samples, environment snapshots and process cleanup results are in state.json and jobs/. The 4K cases use 1080p sources or scaled MG with authored Canvas raster resolutions.\n`); + await writeBenchmarkReport(directory,state); await json(stateFile,state); } async function run(id,items){ @@ -168,7 +157,7 @@ if(args.job){ } const item=(id,doc,config,group,verify)=>({id,doc,config,group,verify}); const repeat=async(group,doc,configs)=>{for(let r=0;r({workers:4,maxPendingFrameBytes:n*MiB}))); - const workerOptions=[1,2,3,4,6,8,'auto']; + const workerOptions=[1,2,3,4,5,6,7,8,'auto']; for(const workers of workerOptions)await run('workers-screen-'+workers,[item('workers-screen-'+workers,'mg',{workers},'workers')]); const ranked=state.results.flatMap(r=>r.passed?(r.exports??[]):[]).filter(r=>r.group==='workers'&&r.id.startsWith('workers-screen-')).sort((a,b)=>a.seconds-b.seconds); const finalists=[];for(const row of ranked)if(!finalists.some(c=>c.workers===row.config.workers)&&finalists.length<2)finalists.push(row.config); + if(!finalists.some(config=>config.workers==='auto'))finalists.push({workers:'auto'}); state.workerFinalists=finalists;await summarize(); for(let r=1;r a - b); + if (!values.length) return null; + const middle = Math.floor(values.length / 2); + return values.length % 2 ? values[middle] : (values[middle - 1] + values[middle]) / 2; +} + +export function summarizeBenchmark(state) { + const groups = new Map(); + for (const result of state.results) for (const item of result.exports ?? []) { + if (!item.group) continue; + const key = `${item.group} / ${item.doc} / ${JSON.stringify(item.config)}`; + const group = groups.get(key) ?? { key, group: item.group, fixture: item.doc, config: item.config, all: [] }; + group.all.push({ ...item, passed: result.passed, rss: result.peakRssMiB, footprint: result.peakFootprintMiB }); + groups.set(key, group); + } + const rows = [...groups.values()].map(({ all, ...group }) => { + const good = all.filter(item => item.passed); + const times = good.map(item => item.seconds); + return { ...group, completed: good.length, failed: all.length - good.length, + frameCounts: [...new Set(good.map(item => item.frames))], + medianSeconds: median(times), minSeconds: times.length ? Math.min(...times) : null, + maxSeconds: times.length ? Math.max(...times) : null, + medianPeakRssMiB: median(good.map(item => item.rss)), + medianPeakFootprintMiB: median(good.map(item => item.footprint)), + decoderStarts: good.map(item => item.videoStats?.reduce((sum, value) => sum + value.decoderStarts, 0)), + samplesRead: good.map(item => item.videoStats?.reduce((sum, value) => sum + value.samplesRead, 0)), + retainedSamplePeakBytes: good.map(item => item.videoStats?.reduce((sum, value) => sum + value.peak, 0)), + }; + }); + const checked = state.results.flatMap(result => (result.exports ?? []) + .filter(item => item.verification).map(item => ({ job: result.id, item: item.id, passed: result.passed, ...item.verification }))); + return { status: state.status ?? 'running', head: state.head, host: state.host, repetitions: state.repetitions, + startedAt: state.startedAt, finishedAt: state.finishedAt, resumptions: state.resumptions ?? [], + completedJobs: state.results.length, failedJobs: state.results.filter(result => !result.passed).map(result => result.id), + successfulExports: state.results.filter(result => result.passed).flatMap(result => result.exports ?? []) + .filter(item => !item.error).length, + expectedFailures: state.results.filter(result => result.passed).flatMap(result => result.exports ?? []) + .filter(item => item.action && item.error).length, + verifiedObservations: checked.reduce((sum, item) => sum + (item.observations ?? 0), 0), + verificationFailures: checked.filter(item => item.error || item.mismatches || !item.passed), checked, rows, + additionalCases: state.results.filter(result => !(result.exports ?? []).some(item => item.group)) + .map(result => ({ id: result.id, passed: result.passed, seconds: result.seconds, + exports: result.exports?.length ?? 0, peakRssMiB: result.peakRssMiB, peakFootprintMiB: result.peakFootprintMiB })), + leaks: state.results.flatMap(result => (result.leaked ?? []).map(process => ({ job: result.id, ...process }))), + }; +} + +export function benchmarkMarkdown(summary) { + const n = value => Number.isFinite(value) ? value.toFixed(3) : 'n/a'; + const host = summary.host; + const lines = ['# HTML GPU export measurements', '', + `Status: ${summary.status}. Renderer baseline: \`${summary.head}\`. Target repetitions: ${summary.repetitions}.`, '', + `Host: ${host.cpu}, ${(host.totalMemoryMiB / 1024).toFixed(0)} GiB RAM, ${host.platform} ${host.release}, Node ${host.node}.`, '', + `Completed jobs: ${summary.completedJobs}. Successful exports: ${summary.successfulExports}. Expected injected failures: ${summary.expectedFailures}. Failed jobs: ${summary.failedJobs.length}.`, '', + '## Measurement settings', '', + 'Export seconds include Provider preparation, capture, encoding, muxing, full output decode validation and storage. Independent frame-ID checks run afterward. Each job starts a fresh Node process; the prepared Electron cache persists. Normal desktop activity remains uncontrolled.', '', + 'Memory is sampled every 250 ms across the export host and its descendants, including Electron processes with independent process groups. RSS, macOS physical footprint and retained sample allocation are distinct quantities. Exclusive device memory is not measured.', '', + 'The four-layer 4K video case composes 1080p sources. The scaled 4K MG case retains authored Canvas raster resolutions. Timing-only jobs keep output in memory; independent verification jobs also save MP4 files.', '', + '## Timing and memory results', '', + '| Group | Fixture | Configuration | Frames per export | Completed | Failed | Median seconds | Range, seconds | Median peak RSS, MiB | Median peak physical footprint, MiB |', + '| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |']; + for (const row of summary.rows) lines.push(`| ${row.group} | ${row.fixture} | ${JSON.stringify(row.config)} | ${row.frameCounts.join(" / ")} | ${row.completed} | ${row.failed} | ${n(row.medianSeconds)} | ${n(row.minSeconds)} to ${n(row.maxSeconds)} | ${n(row.medianPeakRssMiB)} | ${n(row.medianPeakFootprintMiB)} |`); + lines.push('', 'Each fixture is summarized separately. Worker finalists were selected from one screening observation; their initial observations are subject to selection bias. Source and queue configurations are interleaved. These measurements describe the tested fixtures and do not establish a universal optimum.', '', + '## Frame identifiers and cleanup', '', + `Independent frame-ID observations: ${summary.verifiedObservations}. Failed verification records: ${summary.verificationFailures.length}. Observed processes remaining after completed jobs: ${summary.leaks.length}.`, '', + '| Job | Export | Observations | Mismatches | Verification error |', '| --- | --- | --- | --- | --- |'); + for (const item of summary.checked) lines.push(`| ${item.job} | ${item.item} | ${item.observations ?? 'n/a'} | ${item.mismatches ?? 'n/a'} | ${item.error ?? ''} |`); + lines.push('', '## Additional cases', '', + '| Job | Passed | Exports | Batch seconds | Peak RSS, MiB | Peak physical footprint, MiB |', + '| --- | --- | --- | --- | --- | --- |'); + for (const item of summary.additionalCases) lines.push(`| ${item.id} | ${item.passed} | ${item.exports} | ${n(item.seconds)} | ${n(item.peakRssMiB)} | ${n(item.peakFootprintMiB)} |`); + lines.push('', 'Injected cancellation and renderer termination pass when the intended export fails, its action was triggered, peers complete correctly and cleanup succeeds. Recovery is measured in a subsequent job.', '', '## Experiment sessions', '', + `Started: ${summary.startedAt}. Finished: ${summary.finishedAt ?? 'in progress'}.`); + for (const session of summary.resumptions) lines.push('', `Resumed: ${session.at}, checkout \`${session.head}\`. ${session.reason}`); + if (summary.resumptions.length) lines.push('', 'Completed observations span interrupted and resumed sessions. Medians and ranges retain both sessions; desktop load may differ between them.'); + lines.push('', `Failed jobs: ${summary.failedJobs.join(', ') || 'none'}.`, '', + 'Raw case inputs, logs, results, environment snapshots and memory samples are stored in state.json and jobs/ beside the generated report. Runner usage is documented in benchmark-html-gpu.md.', ''); + return lines.join('\n'); +} + +export async function writeBenchmarkReport(directory, state) { + const summary = summarizeBenchmark(state); + for (const [name, contents] of [ + ['summary.json', JSON.stringify(summary, null, 2)], ['REPORT.md', benchmarkMarkdown(summary)], + ]) { + await writeFile(resolve(directory, name + '.tmp'), contents); + await rename(resolve(directory, name + '.tmp'), resolve(directory, name)); + } + return summary; +} + +if (process.argv[1] && import.meta.url === pathToFileURL(resolve(process.argv[1])).href) { + const directory = resolve(process.argv[2] ?? 'dist/html-gpu-benchmark'); + const state = JSON.parse(await readFile(resolve(directory, 'state.json'), 'utf8')); + await writeBenchmarkReport(directory, state); + console.log(resolve(directory, 'REPORT.md')); +} From 39753c213e513c61f03fe7e6163ad86fe7ab24fe Mon Sep 17 00:00:00 2001 From: rpone Date: Mon, 26 Oct 2026 20:49:11 +0800 Subject: [PATCH 14/16] chore(html-local): complete GPU benchmark results and source-based resume --- scripts/benchmark-html-gpu-results.md | 126 +++++++++++++++++++++++--- scripts/benchmark-html-gpu.md | 3 +- scripts/benchmark-html-gpu.mjs | 42 ++++++++- 3 files changed, 151 insertions(+), 20 deletions(-) diff --git a/scripts/benchmark-html-gpu-results.md b/scripts/benchmark-html-gpu-results.md index d137b9c7b..f49f5ddae 100644 --- a/scripts/benchmark-html-gpu-results.md +++ b/scripts/benchmark-html-gpu-results.md @@ -1,22 +1,120 @@ -# Partial automated HTML GPU measurements +# HTML GPU export measurements -Measured on Apple M1 Pro / 32 GiB / macOS 27.0, with production renderer code at `608b5959`. The automated run was stopped at the user's request after 17 completed jobs. All 17 passed. The interrupted fourth repetition of the previous configuration is excluded. +Status: completed. Renderer baseline: `608b595988a5c0d15a3f8880a1066d977fee59c4`. Target repetitions: 5. -Eight staggered/reversed 1080p video layers, 600 frames. Export seconds include Provider preparation, capture, encoding, muxing, full output decode validation and storage. Fresh isolated Node processes are used per job. Normal desktop applications remained open. +Host: Apple M1 Pro, 32 GiB RAM, darwin 27.0.0, Node v26.0.0. -| Source / pending budget | Completed repetitions | Median seconds | Range, seconds | Median peak RSS, MiB | Median peak physical footprint, MiB | Decoder starts per export | -| --- | --- | --- | --- | --- | --- | --- | -| 1 GiB / 256 MiB | 3 | 22.287 | 20.046 to 23.114 | 1053 | 833 | 53 | -| 2.25 GiB / 128 MiB (defaults) | 3 | 16.097 | 15.490 to 17.835 | 1000 | 826 | 14 | -| 3 GiB / 128 MiB | 3 | 16.016 | 15.709 to 20.835 | 1000 | 830 | 14 | -| 4 GiB / 128 MiB | 4 | 17.254 | 15.686 to 25.486 | 994 | 827 | 14 | +Completed jobs: 76. Successful exports: 80. Expected injected failures: 2. Failed jobs: 0. -The default configuration reduced the measured median from 22.287 to 16.097 seconds, approximately 28%. Three repetitions completed for both configurations; the planned five-repetition comparison remains incomplete. The 3 GiB median differs from the default by about 0.08 seconds in these observations. These results support retaining the current provisional defaults and do not establish an optimum. +## Current defaults and interpretation -Default configuration frame-ID checks passed: 5,400 timeline/source observations for eight layers, 1,800 observations for the two-video project and 600 timeline observations for complex MG. The ordinary video export took 12.13 seconds; the automatic-worker MG export took 18.61 seconds. Completed jobs left no observed descendants. The operator-cancelled job is recorded separately. +Retain a 2.25 GiB source-sample budget and a 128 MiB pending-frame budget. The five-observation staggered-video median decreased from 23.114 to 17.835 seconds, a 22.8% reduction. Decoder iterator starts decreased from 53 to 14. The 3/4 GiB cases also used 14 starts. Logical sample budgets do not establish total process or GPU memory usage. -Memory is sampled every 250 ms across the export host and its descendants, including Electron processes that create separate process groups. RSS and macOS physical footprint are distinct from retained sample allocation and exclusive GPU memory. Only the corrected process-tree sampling run is summarized here. +For 1080p MG with four workers, 128/256/512 MiB pending queues produced five-observation medians of 16.044/16.259/16.150 seconds. Retain automatic worker selection: its five-observation MG median was 16.690 seconds, versus 20.377 for four workers and 18.265 for six. Other explicit worker counts have single screening observations. The timing ranges overlap, and sessions include uncontrolled desktop load; these measurements do not establish a universal optimum. -The remaining queue repetitions, worker screening/finalists, 4K control exports, fastest-candidate frame-ID checks, low-budget cases, concurrent exports and fault/recovery cases were not reached in this run. Earlier manual results for those areas remain documented separately in PR #437. +The automatic video strategy remains one scene. The two/four-worker staggered-video screening exports took 28.974/48.136 seconds. The 4K video control exported 300 frames; the scaled 4K MG control exported 600 frames. Their results are summarized separately below. -The runner and fixture adapter passed Node syntax checks and the completed jobs above. The complete automatic sequence remains unvalidated. Raw job inputs, logs, results, memory samples and interrupted-job diagnostics are retained locally under `dist/autonomous-gpu-benchmark-v2/`. Runner usage is documented in `benchmark-html-gpu.md`. +## Repository validation + +Repository type checking passed. The initial local full suite completed with 1,121 passes, two process-test timeouts and 24 skips. Those two tests passed in an isolated sequential run together with the Provider suite: 64 passed, 11 skipped. The cause of the initial timeouts is unconfirmed. Both Windows and Linux full CI checks, independent npm package verification and the other CI checks passed at `feefcbb9`; subsequent harness/report commits receive their own CI run. + +The existing default-value assertions were updated to the selected budgets. Report grouping now includes the fixture name. Resume identities cover actual production source, harness, inputs and host rather than documentation-only commit changes. Legacy state migrated after comparing the original source, harness, fixture fingerprint and host. Resuming the completed experiment retained all 76 jobs and started no exports. + +The complete dataset contains 76 passed jobs, 80 successful exports, two expected injected failures and 31,200 independently checked frame identifiers with zero mismatches. No completed job left observed descendants. Earlier Windows/Linux package checks do not establish GPU hardware compatibility; the GPU measurements here cover Apple Silicon macOS. + +## Measurement settings + +Export seconds include Provider preparation, capture, encoding, muxing, full output decode validation and storage. Independent frame-ID checks run afterward. Each job starts a fresh Node process; the prepared Electron cache persists. Normal desktop activity remains uncontrolled. + +Memory is sampled every 250 ms across the export host and its descendants, including Electron processes with independent process groups. RSS, macOS physical footprint and retained sample allocation are distinct quantities. Exclusive device memory is not measured. + +The four-layer 4K video case composes 1080p sources. The scaled 4K MG case retains authored Canvas raster resolutions. Timing-only jobs keep output in memory; independent verification jobs also save MP4 files. + +## Timing and memory results + +| Group | Fixture | Configuration | Frames per export | Completed | Failed | Median seconds | Range, seconds | Median peak RSS, MiB | Median peak physical footprint, MiB | +| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | +| defaults | video | {} | 600 | 1 | 0 | 12.132 | 12.132 to 12.132 | 961.953 | 658.547 | +| source | staggered | {"workers":1,"maxDecodedSourceBytes":1073741824,"maxPendingFrameBytes":268435456} | 600 | 5 | 0 | 23.114 | 20.046 to 29.098 | 1049.422 | 832.172 | +| source | staggered | {} | 600 | 5 | 0 | 17.835 | 15.490 to 23.682 | 995.719 | 825.501 | +| source | staggered | {"workers":1,"maxDecodedSourceBytes":3221225472,"maxPendingFrameBytes":134217728} | 600 | 5 | 0 | 20.835 | 15.709 to 37.089 | 929.641 | 821.501 | +| source | staggered | {"workers":1,"maxDecodedSourceBytes":4294967296,"maxPendingFrameBytes":134217728} | 600 | 5 | 0 | 18.651 | 15.686 to 25.486 | 993.078 | 825.703 | +| queue | mg | {"workers":4,"maxPendingFrameBytes":134217728} | 600 | 5 | 0 | 16.044 | 14.705 to 21.481 | 1765.703 | 1840.161 | +| queue | mg | {"workers":4,"maxPendingFrameBytes":268435456} | 600 | 5 | 0 | 16.259 | 15.376 to 21.019 | 1719.000 | 1892.521 | +| queue | mg | {"workers":4,"maxPendingFrameBytes":536870912} | 600 | 5 | 0 | 16.150 | 14.894 to 16.829 | 1778.578 | 1849.005 | +| workers | mg | {"workers":1} | 600 | 1 | 0 | 26.855 | 26.855 to 26.855 | 896.109 | 849.235 | +| workers | mg | {"workers":2} | 600 | 1 | 0 | 21.917 | 21.917 to 21.917 | 1300.406 | 1238.658 | +| workers | mg | {"workers":3} | 600 | 1 | 0 | 16.295 | 16.295 to 16.295 | 1530.266 | 1540.020 | +| workers | mg | {"workers":4} | 600 | 5 | 0 | 20.377 | 14.456 to 24.258 | 1583.844 | 1816.255 | +| workers | mg | {"workers":6} | 600 | 5 | 0 | 18.265 | 15.083 to 21.226 | 2029.563 | 2402.337 | +| workers | mg | {"workers":8} | 600 | 1 | 0 | 17.077 | 17.077 to 17.077 | 2284.375 | 2902.840 | +| workers | mg | {"workers":"auto"} | 600 | 5 | 0 | 16.690 | 16.103 to 18.608 | 2738.469 | 2914.778 | +| video-workers | staggered | {"workers":2} | 600 | 1 | 0 | 28.974 | 28.974 to 28.974 | 986.266 | 1297.597 | +| video-workers | staggered | {"workers":4} | 600 | 1 | 0 | 48.136 | 48.136 to 48.136 | 1473.688 | 1913.459 | +| defaults | aligned | {} | 600 | 1 | 0 | 13.349 | 13.349 to 13.349 | 962.688 | 815.828 | +| defaults | mg4k | {} | 600 | 1 | 0 | 46.725 | 46.725 to 46.725 | 2715.266 | 5292.106 | +| defaults | uhd | {} | 300 | 1 | 0 | 19.586 | 19.586 to 19.586 | 988.422 | 1170.298 | +| workers | mg | {"workers":5} | 600 | 1 | 0 | 16.053 | 16.053 to 16.053 | 2085.453 | 2171.366 | +| workers | mg | {"workers":7} | 600 | 1 | 0 | 15.687 | 15.687 to 15.687 | 2512.344 | 2696.010 | + +Each fixture is summarized separately. Worker finalists were selected from one screening observation; their initial observations are subject to selection bias. Source and queue configurations are interleaved. These measurements describe the tested fixtures and do not establish a universal optimum. + +## Frame identifiers and cleanup + +Independent frame-ID observations: 31200. Failed verification records: 0. Observed processes remaining after completed jobs: 0. + +| Job | Export | Observations | Mismatches | Verification error | +| --- | --- | --- | --- | --- | +| accuracy-defaults | accuracy-defaults | 5400 | 0 | | +| defaults-video | defaults-video | 1800 | 0 | | +| mg-defaults | mg-defaults | 600 | 0 | | +| accuracy-workers-6 | accuracy-workers-6 | 600 | 0 | | +| accuracy-workers-4 | accuracy-workers-4 | 600 | 0 | | +| accuracy-queue-fastest | accuracy-queue-fastest | 600 | 0 | | +| accuracy-low-cache | accuracy-low-cache | 1080 | 0 | | +| accuracy-small-queue | accuracy-small-queue | 120 | 0 | | +| accuracy-fastest | accuracy-fastest | 5400 | 0 | | +| concurrent-2 | concurrent-2-0 | 1800 | 0 | | +| concurrent-2 | concurrent-2-1 | 1800 | 0 | | +| concurrent-4 | concurrent-4-0 | 1800 | 0 | | +| concurrent-4 | concurrent-4-1 | 1800 | 0 | | +| concurrent-4 | concurrent-4-2 | 1800 | 0 | | +| concurrent-4 | concurrent-4-3 | 1800 | 0 | | +| concurrent-faults | fault-peer | 1800 | 0 | | +| recovery | recovery | 1800 | 0 | | +| accuracy-workers-auto | accuracy-workers-auto | 600 | 0 | | + +## Additional cases + +| Job | Passed | Exports | Batch seconds | Peak RSS, MiB | Peak physical footprint, MiB | +| --- | --- | --- | --- | --- | --- | +| warmup | true | 1 | 3.500 | 834.484 | 560.438 | +| accuracy-defaults | true | 1 | 16.045 | 988.125 | 800.938 | +| mg-defaults | true | 1 | 18.630 | 2742.813 | 2866.371 | +| accuracy-workers-6 | true | 1 | 18.996 | 1952.984 | 2395.212 | +| accuracy-workers-4 | true | 1 | 21.000 | 1598.844 | 1855.787 | +| accuracy-queue-fastest | true | 1 | 19.164 | 1818.563 | 1799.818 | +| accuracy-low-cache | true | 1 | 37.242 | 873.203 | 730.188 | +| accuracy-small-queue | true | 1 | 9.752 | 1445.641 | 1150.067 | +| accuracy-fastest | true | 1 | 18.080 | 881.594 | 821.688 | +| concurrent-2 | true | 2 | 28.753 | 1443.922 | 1085.193 | +| concurrent-4 | true | 4 | 30.303 | 2702.125 | 2379.132 | +| concurrent-faults | true | 3 | 17.159 | 1637.203 | 827.317 | +| recovery | true | 1 | 14.624 | 959.219 | 635.063 | +| accuracy-workers-auto | true | 1 | 15.274 | 2764.234 | 2947.278 | + +Injected cancellation and renderer termination pass when the intended export fails, its action was triggered, peers complete correctly and cleanup succeeds. Recovery is measured in a subsequent job. + +## Experiment sessions + +Started: 2026-10-24T21:01:42.289Z. Finished: 2026-10-26T12:45:24.615Z. + +Resumed: 2026-10-26T12:13:57.835Z, checkout `feefcbb933a04fe4e513b19d3d9d265ee4788768`. Committed benchmark scripts and corrected stale default assertions; renderer/dependency trees and original input fingerprint verified unchanged. + +Resumed: 2026-10-26T12:37:05.806Z, checkout `ca0e9cbc2bea42a3256c3a784ea14fb6f153768d`. Extend worker screening to every integer from 1 through 8 and repeat the automatic baseline; correct report grouping. Production code and fixture inputs verified unchanged. + +Completed observations span interrupted and resumed sessions. Medians and ranges retain both sessions; desktop load may differ between them. + +Failed jobs: none. + +Raw case inputs, logs, results, environment snapshots and memory samples are stored in state.json and jobs/ beside the generated report. Runner usage is documented in benchmark-html-gpu.md. diff --git a/scripts/benchmark-html-gpu.md b/scripts/benchmark-html-gpu.md index fd3c7db41..fc37524f9 100644 --- a/scripts/benchmark-html-gpu.md +++ b/scripts/benchmark-html-gpu.md @@ -47,7 +47,8 @@ failed cases. Compare medians and ranges within the measured fixtures. Every job writes its logs, input, result and memory samples under `jobs/`. `state.json`, `summary.json` and `REPORT.md` update after each job. Rerunning the -same command resumes remaining jobs when source and input fingerprints match. +same command resumes remaining jobs when source, input and host fingerprints match. +Documentation and test-only commits preserve the source-content identity. Completed failures stay in the report; use a fresh output directory for a new experiment. Concurrent runners sharing an output directory are rejected. diff --git a/scripts/benchmark-html-gpu.mjs b/scripts/benchmark-html-gpu.mjs index f23159a76..2fa3bf15a 100644 --- a/scripts/benchmark-html-gpu.mjs +++ b/scripts/benchmark-html-gpu.mjs @@ -90,15 +90,47 @@ if(args.job){ command('ffmpeg',['-version']);command('ffprobe',['-version']); const { loadProject }=await import('./html-gpu-benchmark-fixtures.mjs'); const fingerprint=createHash('sha256'); - fingerprint.update(command('git',['rev-parse','HEAD']));fingerprint.update(command('git',['diff','HEAD','--'])); - for(const file of [script,resolve(root,'scripts/html-gpu-benchmark-fixtures.mjs'),resolve(root,'scripts/html-gpu-benchmark-report.mjs')])fingerprint.update(await readFile(file)); - for(const project of [videoProject,mgProject]){const data=await loadProject(project);fingerprint.update(JSON.stringify(data.program));for(const [artifact,bytes]of data.blobs){fingerprint.update(artifact.resource);fingerprint.update(bytes);}} - fingerprint.update(String(reps));const identity=fingerprint.digest('hex'),stateFile=resolve(directory,'state.json'); + fingerprint.update(JSON.stringify({scheme:3,node:process.version,platform:process.platform,arch:process.arch, + cpu:cpus()[0]?.model,release:release(),memory:totalmem()})); + const isSource=file=>!/(?:^|\/)(?:test|dist|node_modules)(?:\/|$)/.test(file)&&/\.(?:[cm]?[jt]s|json|yaml)$/.test(file); + const sourceList=spawnSync('git',['ls-files','--cached','--others','--exclude-standard','-z','--', + 'packages','package.json','pnpm-lock.yaml','tsconfig.json'],{encoding:'utf8',maxBuffer:4*MiB}); + if(sourceList.status!==0)throw Error('Cannot enumerate benchmark source files: '+sourceList.stderr); + const sources=[...new Set(sourceList.stdout.split('\0').filter(file=>file&&isSource(file)))].sort(); + const harness=['scripts/benchmark-html-gpu.mjs','scripts/html-gpu-benchmark-fixtures.mjs','scripts/html-gpu-benchmark-report.mjs']; + for(const file of [...sources,...harness]){fingerprint.update(file+'\0');fingerprint.update(await readFile(resolve(root,file)));fingerprint.update('\0');} + const inputs=[]; + for(const project of [videoProject,mgProject]){const data=await loadProject(project);inputs.push(JSON.stringify(data.program));for(const [artifact,bytes]of data.blobs)inputs.push(artifact.resource,bytes);} + inputs.push(String(reps));for(const input of inputs){fingerprint.update(input);fingerprint.update('\0');} + const identity=fingerprint.digest('hex'),stateFile=resolve(directory,'state.json'); let state;try{state=JSON.parse(await readFile(stateFile,'utf8'));}catch(e){if(e.code!=='ENOENT')throw e;} - if(state&&state.identity!==identity)throw Error('Source, fixtures or repetition count changed. Select a fresh --out directory.'); + if(state&&state.identityScheme!==3){ + const legacyHead=state.resumptions?.at(-1)?.head??state.head; + const oldFile=file=>{const result=spawnSync('git',['show',legacyHead+':'+file],{maxBuffer:4*MiB}); + if(result.status!==0)throw Error('Cannot recover the legacy benchmark harness; select a fresh --out directory.');return result.stdout;}; + const legacyHarness=harness.map(oldFile),oldRunner=legacyHarness[0].toString(); + const stripIdentity=text=>{const a=text.indexOf(" const fingerprint=createHash('sha256');"),b=text.indexOf('\n state??={',a)+1; + if(a<0||b<1)throw Error('Unrecognized benchmark identity format.');return text.slice(0,a)+''+text.slice(b).replace(' state.identityScheme=3;\n','');}; + const changed=command('git',['diff','--name-only',legacyHead,'--','packages','package.json','pnpm-lock.yaml','tsconfig.json']) + .split('\n').filter(file=>file&&isSource(file)); + const untracked=spawnSync('git',['ls-files','--others','--exclude-standard','-z','--','packages'],{encoding:'utf8'}); + if(untracked.status!==0||untracked.stdout.split('\0').some(file=>file&&isSource(file)))throw Error('Untracked production source prevents legacy resume.'); + const sameHost=state.host.platform===process.platform&&state.host.release===release()&&state.host.node===process.version + &&state.host.cpu===cpus()[0]?.model&&state.host.totalMemoryMiB===totalmem()/MiB; + if(changed.length||!sameHost||stripIdentity(oldRunner)!==stripIdentity(await readFile(script,'utf8')) + ||!legacyHarness[1].equals(await readFile(resolve(root,harness[1])))||!legacyHarness[2].equals(await readFile(resolve(root,harness[2])))) + throw Error('Legacy benchmark source or host changed; select a fresh --out directory.'); + const legacy=createHash('sha256').update(legacyHead);for(const bytes of [...legacyHarness,...inputs])legacy.update(bytes); + if(legacy.digest('hex')!==state.identity)throw Error('Legacy script and fixture fingerprint did not match.'); + state.identityMigration={at:new Date().toISOString(),legacyHead,previous:state.identity, + reason:'Verified identical production source, measurement harness, fixtures and host before migrating to a source-content identity.'}; + state.identity=identity; + } + if(state&&state.identity!==identity)throw Error('Source, fixtures, host or repetition count changed. Select a fresh --out directory.'); state??={identity,head:command('git',['rev-parse','HEAD']),dirty:!!command('git',['status','--porcelain']), startedAt:new Date().toISOString(),host:{platform:process.platform,release:release(),cpu:cpus()[0]?.model,totalMemoryMiB:totalmem()/MiB,node:process.version}, videoProject,mgProject,repetitions:reps,results:[]}; + state.identityScheme=3; let readUsage; try{const {default:koffi}=await import('koffi');readUsage=koffi.load('/usr/lib/libproc.dylib').func('int proc_pid_rusage(int pid, int flavor, void *buffer)');}catch{} function processes(rootPid, observed) { From 2ed24dbd07f2b8a344620fd0813fe9eabe865c1f Mon Sep 17 00:00:00 2001 From: rpone Date: Mon, 26 Oct 2026 21:12:30 +0800 Subject: [PATCH 15/16] ci: use the Ubuntu archive for minimal FFmpeg installation --- .github/workflows/ci.yml | 9 +++++++-- .github/workflows/npm-package.yml | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1a6e315cb..59fb7dee2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -66,8 +66,13 @@ jobs: - name: Install ffmpeg on Linux if: runner.os == 'Linux' run: | - sudo apt-get update - sudo apt-get install -y ffmpeg + for source in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do + if [ -f "$source" ]; then + sudo sed -i -E 's|https?://azure\.archive\.ubuntu\.com/ubuntu|https://archive.ubuntu.com/ubuntu|g' "$source" + fi + done + sudo apt-get -o Acquire::Retries=3 update + sudo apt-get -o Acquire::Retries=3 install -y --no-install-recommends ffmpeg - name: Install ffmpeg on Windows if: runner.os == 'Windows' diff --git a/.github/workflows/npm-package.yml b/.github/workflows/npm-package.yml index b2f9a5cbb..3ca9481e6 100644 --- a/.github/workflows/npm-package.yml +++ b/.github/workflows/npm-package.yml @@ -66,8 +66,13 @@ jobs: - name: Prepare the local media toolchain on Linux if: runner.os == 'Linux' run: | - sudo apt-get update - sudo apt-get install -y ffmpeg + for source in /etc/apt/sources.list /etc/apt/sources.list.d/*.list /etc/apt/sources.list.d/*.sources; do + if [ -f "$source" ]; then + sudo sed -i -E 's|https?://azure\.archive\.ubuntu\.com/ubuntu|https://archive.ubuntu.com/ubuntu|g' "$source" + fi + done + sudo apt-get -o Acquire::Retries=3 update + sudo apt-get -o Acquire::Retries=3 install -y --no-install-recommends ffmpeg - name: Prepare the local media toolchain on Windows if: runner.os == 'Windows' run: choco install ffmpeg --yes --no-progress From c080acd92907837437ef59a3da01ce10aa14561e Mon Sep 17 00:00:00 2001 From: rpone Date: Wed, 28 Oct 2026 23:54:32 +0800 Subject: [PATCH 16/16] fix(html-local): preserve GPU export frame rates across FFmpeg versions --- packages/provider-html-local/src/capture.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/provider-html-local/src/capture.ts b/packages/provider-html-local/src/capture.ts index 48e14e640..c2011a22d 100644 --- a/packages/provider-html-local/src/capture.ts +++ b/packages/provider-html-local/src/capture.ts @@ -341,10 +341,11 @@ export async function captureStagedVisual(input: CaptureInput, controller: Abort if (gpu !== undefined) { const raw = await gpu.finish(); await gpu.close(); - // The requested frame clock defines every packet duration, including the last. + // Use the same integer clock for input packets, setts and the MP4 track. + // Each frame occupies fps.den ticks, including the final packet. await runProcess({ executable: ffmpegPath, - argv: ["-v", "error", "-y", "-r", `${fps.num}/${fps.den}`, "-i", raw, - "-an", "-c:v", "copy", "-bsf:v", `setts=pts=N:dts=N:duration=1:time_base=${fps.den}/${fps.num}`, + argv: ["-v", "error", "-y", "-r", String(fps.num), "-i", raw, + "-an", "-c:v", "copy", "-bsf:v", `setts=pts=N*${fps.den}:dts=N*${fps.den}:duration=${fps.den}:time_base=1/${fps.num}`, "-video_track_timescale", String(fps.num), "-movflags", "+faststart", output], timeoutMs: config.processTimeoutMs, maxOutputBytes: config.maxProcessOutputBytes, signal }); } else await orderedFrames!.close();