From 5c18e10a327dbb34dcaa7e0cdf3d9086906e33f8 Mon Sep 17 00:00:00 2001 From: BIackFIame <77388790+BIackFIame@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:11:16 +0000 Subject: [PATCH 1/7] fix(orchestration): a restarted Windows pipe host keeps its address, and a crashing one is not restarted forever The browser gateway restarts a failed Windows pipe host on the pipe name the first host published, so agent helpers that already hold the address and their reconnect token come back. The native host takes an optional --pipe-name argument and accepts only names in the form it generates itself; the transport passes it and fails if the ready frame names a different pipe. Both gateways now keep their three-retry budget while a replacement host dies within a minute of starting, so a host that crashes right after start-up is given up on instead of being restarted every few seconds; a host that served longer than that earns a fresh budget. The orchestration gateway keeps its one-run leases: a restarted host serves only newly launched orchestrators. --- native/windows-agent-pipe-host/src/main.cc | 32 +++++++++++--- .../services/agent-browser/AgentGateway.ts | 20 +++++++-- .../agent-browser/OrchestrationGateway.ts | 13 +++++- .../agent-browser/WindowsPipeHostTransport.ts | 26 ++++++++++-- tests/agent-browser-protocol-gateway.test.mjs | 37 ++++++++++++++++ tests/orchestration-gateway.test.mjs | 42 +++++++++++++++++++ tests/windows-pipe-host-transport.test.mjs | 42 +++++++++++++++++++ 7 files changed, 200 insertions(+), 12 deletions(-) diff --git a/native/windows-agent-pipe-host/src/main.cc b/native/windows-agent-pipe-host/src/main.cc index 2693ae9f..a1b3a803 100644 --- a/native/windows-agent-pipe-host/src/main.cc +++ b/native/windows-agent-pipe-host/src/main.cc @@ -713,12 +713,33 @@ bool SelfTest() { return server_ok && client_ok.load(); } -bool ParseParentPid(int argc, wchar_t** argv, DWORD* parent_pid) { - if (argc != 3 || std::wcscmp(argv[1], L"--parent-pid") != 0) return false; +// A restarted host may reuse the name its failed predecessor published, so clients that already hold it +// (and their reconnect tokens) can come back. Only names this host itself would generate are accepted. +bool IsGeneratedPipeName(const wchar_t* value) { + constexpr wchar_t kPrefix[] = L"\\\\.\\pipe\\canvastty-agent-"; + const std::size_t prefix_length = std::wcslen(kPrefix); + if (std::wcsncmp(value, kPrefix, prefix_length) != 0) return false; + const wchar_t* suffix = value + prefix_length; + if (std::wcslen(suffix) != 32) return false; + for (const wchar_t* cursor = suffix; *cursor != L'\0'; ++cursor) { + const bool digit = *cursor >= L'0' && *cursor <= L'9'; + const bool lower_hex = *cursor >= L'a' && *cursor <= L'f'; + if (!digit && !lower_hex) return false; + } + return true; +} + +bool ParseArguments(int argc, wchar_t** argv, DWORD* parent_pid, std::wstring* pipe_name) { + if ((argc != 3 && argc != 5) || std::wcscmp(argv[1], L"--parent-pid") != 0) return false; wchar_t* end = nullptr; const unsigned long value = std::wcstoul(argv[2], &end, 10); if (end == argv[2] || *end != L'\0' || value == 0 || value > MAXDWORD) return false; *parent_pid = static_cast(value); + pipe_name->clear(); + if (argc == 5) { + if (std::wcscmp(argv[3], L"--pipe-name") != 0 || !IsGeneratedPipeName(argv[4])) return false; + pipe_name->assign(argv[4]); + } return true; } @@ -743,8 +764,9 @@ int wmain(int argc, wchar_t** argv) { } DWORD parent_pid = 0; - if (!ParseParentPid(argc, argv, &parent_pid)) { - SendTextFrame(FrameType::kFatal, "Expected --parent-pid ."); + std::wstring requested_pipe_name; + if (!ParseArguments(argc, argv, &parent_pid, &requested_pipe_name)) { + SendTextFrame(FrameType::kFatal, "Expected --parent-pid [--pipe-name ]."); return 13; } HANDLE parent = OpenProcess(SYNCHRONIZE, FALSE, parent_pid); @@ -758,7 +780,7 @@ int wmain(int argc, wchar_t** argv) { CloseHandle(parent); return 15; } - const std::wstring pipe_name = RandomPipeName(); + const std::wstring pipe_name = requested_pipe_name.empty() ? RandomPipeName() : requested_pipe_name; if (pipe_name.empty()) { SendTextFrame(FrameType::kFatal, "Secure pipe-name generation failed."); CloseHandle(parent); diff --git a/src/main/services/agent-browser/AgentGateway.ts b/src/main/services/agent-browser/AgentGateway.ts index 9ecc0a5a..2f08d799 100644 --- a/src/main/services/agent-browser/AgentGateway.ts +++ b/src/main/services/agent-browser/AgentGateway.ts @@ -43,6 +43,8 @@ import { const DEFAULT_CAPABILITY_TTL_MS = 60_000; const MAX_TRANSPORT_RESTART_ATTEMPTS = 3; const TRANSPORT_RESTART_BASE_DELAY_MS = 500; +/** A host that dies within this long of starting counts toward MAX_TRANSPORT_RESTART_ATTEMPTS; a longer run resets it. */ +const TRANSPORT_FAST_FAILURE_WINDOW_MS = 60_000; export const WINDOWS_AGENT_GATEWAY_UNAVAILABLE = "Agent browser access on Windows requires the packaged current-user-only named-pipe host."; @@ -112,6 +114,9 @@ export class AgentGateway { /** Bumped by close(): a Unix bring-up still creating or opening its socket then knows it was closed. */ private closeGeneration = 0; private restartTimer: NodeJS.Timeout | undefined; + /** The pipe name the first host published; a replacement listens on it again so helpers can reconnect. */ + private windowsPipeName: string | null = null; + private transportStartedAt: number | null = null; private restartAttempts = 0; private restartToken = 0; private recovering = false; @@ -166,10 +171,12 @@ export class AgentGateway { const settle = () => { if (this.startPromise === starting) this.startPromise = null; }; + const recovering = this.recovering; void starting.then(() => { settle(); this.recovering = false; - this.restartAttempts = 0; + // A replacement host counts as recovered only once it outlives the fast-failure window (see the fatal handler). + if (!recovering) this.restartAttempts = 0; }, settle); return starting; } @@ -179,7 +186,8 @@ export class AgentGateway { const transport = this.windowsPipeHostFactory({ hostPath: this.windowsHostPath, platform: this.platform, - parentPid: process.pid + parentPid: process.pid, + ...(this.windowsPipeName ? { pipeName: this.windowsPipeName } : {}) }); this.windowsTransport = transport; transport.on("fatal", () => this.handleTransportFatal(transport)); @@ -192,6 +200,8 @@ export class AgentGateway { throw new Error("Windows agent pipe host was superseded during startup."); } this.endpoint = endpoint; + this.windowsPipeName = endpoint; + this.transportStartedAt = this.now(); this.expiryTimer = setInterval(() => this.expireConnections(), 1_000); this.expiryTimer.unref(); return endpoint; @@ -335,6 +345,8 @@ export class AgentGateway { this.server = null; this.windowsTransport = null; this.endpoint = null; + this.windowsPipeName = null; + this.transportStartedAt = null; this.ownedRuntimeDirectory = null; if (server) await closeServer(server); if (windowsTransport) await windowsTransport.close(); @@ -344,7 +356,10 @@ export class AgentGateway { private handleTransportFatal(transport: WindowsPipeHostTransport): void { // A transport that was already replaced must not disturb its successor. if (this.windowsTransport !== transport) return; + const ranFor = this.transportStartedAt === null ? 0 : this.now() - this.transportStartedAt; + if (ranFor >= TRANSPORT_FAST_FAILURE_WINDOW_MS) this.restartAttempts = 0; this.windowsTransport = null; + this.transportStartedAt = null; this.endpoint = null; clearInterval(this.expiryTimer); this.expiryTimer = undefined; @@ -387,7 +402,6 @@ export class AgentGateway { return; } this.recovering = false; - this.restartAttempts = 0; } catch { if (token === this.restartToken && this.enabled) this.scheduleTransportRestart(); } diff --git a/src/main/services/agent-browser/OrchestrationGateway.ts b/src/main/services/agent-browser/OrchestrationGateway.ts index 407cd1c3..5a8d6094 100644 --- a/src/main/services/agent-browser/OrchestrationGateway.ts +++ b/src/main/services/agent-browser/OrchestrationGateway.ts @@ -41,6 +41,8 @@ import { const CAPABILITY_TTL_MS = 60_000; const MAX_TRANSPORT_RESTART_ATTEMPTS = 3; const TRANSPORT_RESTART_BASE_DELAY_MS = 500; +/** A replacement host that dies within this long of starting still counts toward MAX_TRANSPORT_RESTART_ATTEMPTS. */ +const TRANSPORT_FAST_FAILURE_WINDOW_MS = 60_000; interface CapabilityLease { connectionId: string; @@ -100,6 +102,8 @@ export class OrchestrationGateway { private generation = 0; private restartTimer: NodeJS.Timeout | null = null; private restartAttempts = 0; + /** When the current Windows pipe host started listening. */ + private transportStartedAt: number | null = null; private restartToken = 0; private recovering = false; /** The start caller (or recovery attempt) that currently owns a shared in-flight start. */ @@ -204,6 +208,7 @@ export class OrchestrationGateway { throw new Error("The Windows orchestration pipe host failed during startup."); } this.socketEndpoint = endpoint; + this.transportStartedAt = this.now(); } else { // Unix domain sockets cap at ~104 path bytes (macOS); fall back to a short // current-user directory exactly like the browser gateway does. @@ -253,12 +258,18 @@ export class OrchestrationGateway { this.socketEndpoint = null; this.ownedRuntimeDirectory = null; this.running = false; + this.transportStartedAt = null; } private handleTransportFatal(transport: WindowsPipeHostTransport, generation: number): void { // A late event from an old host, or one superseded by explicit stop(), cannot affect a successor. if (this.windowsTransport !== transport || generation !== this.generation) return; const wasRunning = this.running; + // A host that served for a while earns a fresh retry budget; one that dies right after starting does not, so a + // host that keeps crashing on start-up is not restarted forever. + const ranFor = this.transportStartedAt === null ? 0 : this.now() - this.transportStartedAt; + if (ranFor >= TRANSPORT_FAST_FAILURE_WINDOW_MS) this.restartAttempts = 0; + this.transportStartedAt = null; this.windowsTransport = null; this.socketEndpoint = null; this.running = false; @@ -317,8 +328,8 @@ export class OrchestrationGateway { if (this.recovering && this.enabled) this.scheduleTransportRestart(); return; } + // The retry budget is kept until the replacement outlives TRANSPORT_FAST_FAILURE_WINDOW_MS (see the fatal handler). this.recovering = false; - this.restartAttempts = 0; } catch { if (intent === this.startIntent && token === this.restartToken && this.enabled) this.scheduleTransportRestart(); } diff --git a/src/main/services/agent-browser/WindowsPipeHostTransport.ts b/src/main/services/agent-browser/WindowsPipeHostTransport.ts index 50ba164b..e2ddec30 100644 --- a/src/main/services/agent-browser/WindowsPipeHostTransport.ts +++ b/src/main/services/agent-browser/WindowsPipeHostTransport.ts @@ -43,6 +43,11 @@ export interface WindowsPipeHostTransportOptions { startupTimeoutMs?: number; platform?: NodeJS.Platform; spawnHost?: typeof spawn; + /** + * The endpoint a previous host of the same gateway published. A restarted host listens on it again, so clients + * that already hold the address (and their reconnect tokens) can come back; omitted, the host picks a fresh name. + */ + pipeName?: string; } interface RelayFrame { @@ -60,7 +65,7 @@ interface RelayFrame { */ export class WindowsPipeHostTransport extends EventEmitter { private readonly options: Required> - & Pick; + & Pick; private readonly sockets = new Map(); private decoder = new RelayFrameDecoder(); private child: ChildProcessWithoutNullStreams | null = null; @@ -77,8 +82,12 @@ export class WindowsPipeHostTransport extends EventEmitter { parentPid: options.parentPid ?? process.pid, startupTimeoutMs: options.startupTimeoutMs ?? DEFAULT_STARTUP_TIMEOUT_MS, platform: options.platform ?? process.platform, - spawnHost: options.spawnHost + spawnHost: options.spawnHost, + pipeName: options.pipeName }; + if (options.pipeName !== undefined && !isGeneratedPipeName(options.pipeName)) { + throw new Error("Windows agent pipe host name is not one the host generates."); + } } get address(): string { @@ -111,7 +120,10 @@ export class WindowsPipeHostTransport extends EventEmitter { const spawnHost = this.options.spawnHost ?? spawn; const child = spawnHost( this.options.hostPath, - ["--parent-pid", String(this.options.parentPid)], + [ + "--parent-pid", String(this.options.parentPid), + ...(this.options.pipeName ? ["--pipe-name", this.options.pipeName] : []) + ], { stdio: ["pipe", "pipe", "pipe"], windowsHide: true, @@ -150,6 +162,9 @@ export class WindowsPipeHostTransport extends EventEmitter { for (const frame of this.decoder.push(chunk)) { if (frame.type === HOST_TO_PARENT.ready && !settled) { const endpoint = parseReadyEndpoint(frame); + if (this.options.pipeName && endpoint !== this.options.pipeName) { + throw new Error("Windows pipe host listened on a different endpoint than requested."); + } this.endpoint = endpoint; this.started = true; settled = true; @@ -423,6 +438,11 @@ function parseReadyEndpoint(frame: RelayFrame): string { return endpoint; } +/** The names the native host generates: the fixed prefix and 16 random bytes in lowercase hex. */ +function isGeneratedPipeName(value: string): boolean { + return /^\\\\\.\\pipe\\canvastty-agent-[0-9a-f]{32}$/u.test(value); +} + function safeHostMessage(payload: Buffer): string { return payload.toString("utf8").replace(/[\u0000-\u001f\u007f]/g, " ").slice(0, 1_024) || "Windows agent pipe host reported a fatal error."; diff --git a/tests/agent-browser-protocol-gateway.test.mjs b/tests/agent-browser-protocol-gateway.test.mjs index cc7a553c..070a6859 100644 --- a/tests/agent-browser-protocol-gateway.test.mjs +++ b/tests/agent-browser-protocol-gateway.test.mjs @@ -354,6 +354,43 @@ test("AgentGateway restarts a failed Windows host and cancels recovery when disa assert.equal(transports.length, 2); }); +test("AgentGateway restarts the Windows host on its published pipe name and gives up on a host that keeps dying at once", async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + const transports = []; + const gateway = new AgentGateway(core(), { + platform: "win32", + windowsHostPath: "/fake/host.exe", + windowsPipeHostFactory: (options) => { + const transport = new EventEmitter(); + transport.options = options; + transport.isRunning = false; + transport.start = async () => { + transport.isRunning = true; + return options.pipeName ?? "published-pipe"; + }; + transport.close = async () => { transport.isRunning = false; }; + transports.push(transport); + return transport; + } + }); + t.after(() => gateway.close()); + + assert.equal(await gateway.start(), "published-pipe"); + assert.equal(transports[0].options.pipeName, undefined); + // Each replacement dies right after it starts: the attempts must not reset on such a short-lived success. + for (let attempt = 0; attempt < 6; attempt += 1) { + const live = transports.at(-1); + live.isRunning = false; + live.emit("fatal", new Error("host exited")); + t.mock.timers.tick(30_000); + await new Promise(setImmediate); + } + assert.equal(transports.length, 4, "three replacements, then recovery gives up"); + assert.deepEqual(transports.slice(1).map((transport) => transport.options.pipeName), [ + "published-pipe", "published-pipe", "published-pipe" + ]); +}); + test("AgentGateway idempotently authenticates live helpers and rotates reconnect capability", POSIX_GATEWAY_TEST, async (t) => { let connected = 0; const disconnects = []; diff --git a/tests/orchestration-gateway.test.mjs b/tests/orchestration-gateway.test.mjs index bdeb6cd7..e2f0a524 100644 --- a/tests/orchestration-gateway.test.mjs +++ b/tests/orchestration-gateway.test.mjs @@ -544,6 +544,48 @@ test("Windows host recovery is bounded to three retries and disabling cancels ba assert.equal(enabledLease.address, "fake-pipe-1"); }); +test("a Windows host that keeps dying right after starting is given up on; one that served a while gets a fresh budget", async (t) => { + t.mock.timers.enable({ apis: ["setTimeout"] }); + let now = 0; + const { factory, transports } = fakeWindowsPipeHostFactory(); + const gateway = new OrchestrationGateway({ + runtimeDirectory: "unused", + platform: "win32", + windowsHostPath: "C:\\fake\\host.exe", + windowsPipeHostFactory: factory, + now: () => now, + handler: { execute: async () => ({}) } + }); + t.after(() => gateway.stop()); + + await gateway.start(); + // Each replacement starts and dies at once: three replacements, then recovery stops. + for (const delay of [500, 1_000, 2_000]) { + transports.at(-1).emit("fatal", new Error("host exited")); + t.mock.timers.tick(delay); + await new Promise(setImmediate); + } + assert.equal(transports.length, 4); + assert.equal(gateway.address, "fake-pipe-3"); + transports[3].emit("fatal", new Error("host exited")); + t.mock.timers.tick(30_000); + await new Promise(setImmediate); + assert.equal(transports.length, 4, "a host that crashes on start-up is not restarted forever"); + assert.equal(gateway.address, null); + + // An explicit start brings it back; a host that then serves past the window earns three new attempts. + await gateway.start(); + assert.equal(transports.length, 5); + for (let round = 0; round < 4; round += 1) { + now += 61_000; + transports.at(-1).emit("fatal", new Error("host exited")); + t.mock.timers.tick(500); + await new Promise(setImmediate); + } + assert.equal(transports.length, 9, "long-lived hosts are replaced every time"); + assert.equal(gateway.address, "fake-pipe-8"); +}); + test("an explicit stop wins over a replacement host that is still starting", async (t) => { t.mock.timers.enable({ apis: ["setTimeout"] }); let finishReplacement; diff --git a/tests/windows-pipe-host-transport.test.mjs b/tests/windows-pipe-host-transport.test.mjs index b44dca2c..39a30eec 100644 --- a/tests/windows-pipe-host-transport.test.mjs +++ b/tests/windows-pipe-host-transport.test.mjs @@ -191,3 +191,45 @@ test("Windows pipe transport: a failed host that exits late does not end the hos assert.equal(fatal.length, 1, "no failure is reported for the new host"); await transport.close(); }); + +test("Windows pipe transport asks a restarted host for the published pipe name and rejects any other", async () => { + const name = `\\\\.\\pipe\\canvastty-agent-${"0f".repeat(16)}`; + assert.throws(() => new WindowsPipeHostTransport({ + platform: "win32", + hostPath: join(process.cwd(), "package.json"), + pipeName: "\\\\.\\pipe\\someone-else" + }), /not one the host generates/u); + + const child = fakeHost(); + const spawned = []; + const transport = new WindowsPipeHostTransport({ + platform: "win32", + hostPath: join(process.cwd(), "package.json"), + pipeName: name, + spawnHost: (_path, args) => { + spawned.push(args); + return child; + } + }); + const starting = transport.start(() => undefined); + while (spawned.length === 0) await new Promise((resolve) => setTimeout(resolve, 1)); + assert.deepEqual(spawned[0].slice(2), ["--pipe-name", name]); + child.stdout.write(frame(protocol.hostToParent.ready, 0, Buffer.from(name, "utf8"))); + assert.equal(await starting, name); + await transport.close(); + + const other = fakeHost(); + const mismatched = new WindowsPipeHostTransport({ + platform: "win32", + hostPath: join(process.cwd(), "package.json"), + pipeName: name, + spawnHost: () => other + }); + mismatched.on("fatal", () => undefined); + let otherSpawned = false; + other.stdout.once("resume", () => { otherSpawned = true; }); + const failing = mismatched.start(() => undefined); + while (!otherSpawned) await new Promise((resolve) => setTimeout(resolve, 1)); + other.stdout.write(frame(protocol.hostToParent.ready, 0, Buffer.from(`\\\\.\\pipe\\canvastty-agent-${"1".repeat(32)}`))); + await assert.rejects(failing, /different endpoint/u); +}); From 5b7db6f6d54ce6f3d62c15593666424f9fbf4717 Mon Sep 17 00:00:00 2001 From: BIackFIame <77388790+BIackFIame@users.noreply.github.com> Date: Wed, 30 Sep 2026 19:50:53 +0000 Subject: [PATCH 2/7] perf(browser): pause hidden tabs, and put long-hidden ones to sleep Background throttling leaves a hidden tab's requestAnimationFrame loop and one timer wake-up per second running. A tab that stays hidden, with no agent driving it, for 30 s is now frozen through its existing debugger attachment (CDP Page.setWebLifecycleState). After 10 minutes, or when more than 6 hidden tabs are alive, the least recently used one goes to sleep: its WebContents closes. The tab keeps its address, title, favicon, back/forward history (with each entry's scroll and form state) and a small picture for the card. Showing a tab wakes it, and so does any command for it. A paused tab resumes before any automation CDP command reaches the page. A sleeping tab reloads on its history entry, and the agent result says "Browser tab was reloaded" (details.tabReloaded on errors). Closing or reloading a sleeping tab does not wake it first. A tab is never paused while it plays media, downloads, has a dialog open, or is being captured or inspected. It is never put to sleep while an agent is on it or while the page has a beforeunload handler. An unanswered check counts as "has one". A blocked tab is checked again later and never forced. Settings: one toggle, "Pause hidden browser tabs" (on by default). The card marks tabs as "paused" or "sleeping" and shows a sleeping tab's last picture. Agents never receive that picture. Measured on the hidden built app with a fake HOME, 2 runs: 5 hidden tabs, each with a 16 ms timer and a rAF loop. CPU over 30 s: throttling only 13.4 / 9.7 % of a core, paused 0.4 / 0.4 %, sleeping 0.2 / 0.2 % (the app alone uses 0.1 %). RSS: throttling only and paused +496 / +340 MB over the app alone; sleeping +30 / +20 MB. A command reaches a paused tab in 13-19 ms and a sleeping one in 80-87 ms (reloaded, with the notice). Electron's view.webContents getter returns undefined once a WebContents has closed. Each tab's view now pins its WebContents object, so a closed tab still answers isDestroyed(). This also covers destroyed tabs, which already had the problem. --- src/main/index.ts | 15 +- src/main/services/BrowserService.ts | 267 ++++++++++++++- src/main/services/SettingsStore.ts | 4 + .../browser/BrowserAutomationService.ts | 80 ++++- .../browser/BrowserCommandDispatcher.ts | 4 +- src/main/services/browser/BrowserCore.ts | 37 +- .../services/browser/BrowserTabLifecycle.ts | 324 ++++++++++++++++++ src/renderer/src/App.tsx | 1 + .../src/features/browser/BrowserCard.tsx | 16 + .../src/features/settings/SettingsPanel.tsx | 10 + src/renderer/src/lib/i18n.ts | 8 + src/renderer/src/styles/app.css | 3 + src/shared/contracts.ts | 13 + .../browser-tab-lifecycle-automation.test.mjs | 163 +++++++++ tests/browser-tab-lifecycle.test.mjs | 233 +++++++++++++ tests/settings-normalizer.test.mjs | 11 +- 16 files changed, 1172 insertions(+), 17 deletions(-) create mode 100644 src/main/services/browser/BrowserTabLifecycle.ts create mode 100644 tests/browser-tab-lifecycle-automation.test.mjs create mode 100644 tests/browser-tab-lifecycle.test.mjs diff --git a/src/main/index.ts b/src/main/index.ts index 3bd22a55..d8c09c71 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -51,7 +51,7 @@ import { AgentControlService } from "./services/AgentControlService"; import { AgentIsolation } from "./services/isolation/AgentIsolation"; import type { AgentProviderId, LaunchProfileId } from "../shared/contracts"; import { HermesHudService } from "./services/HermesHudService"; -import { BrowserService } from "./services/BrowserService"; +import { BrowserService, type BrowserServiceOptions } from "./services/BrowserService"; import { CanvasNavigationInputController } from "./services/CanvasNavigationOverride"; import { activeCanvasWheelBinding } from "../shared/canvasNavigation"; import type { ProviderSmokeTarget } from "./services/browser/ProviderElectronSmoke"; @@ -402,6 +402,8 @@ async function initializeServices(ipc: IpcRegistrar): Promise { browserService = new BrowserService(() => mainWindow, { userDataPath, restoreTabs: settings.get().browserRestoreTabs, + pauseHiddenTabs: settings.get().browserPauseHiddenTabs, + ...browserLifecycleTimingOverride(process.env.CANVASTTY_BROWSER_LIFECYCLE_MS), canvasWheelCaptureMode: settings.get().canvasWheelCaptureMode, canvasNavigationInput, ...(process.env.CANVASTTY_BROWSER_SMOKE_URL @@ -758,6 +760,7 @@ async function initializeServices(ipc: IpcRegistrar): Promise { browserService?.setRestoreTabs(next.browserRestoreTabs).catch((error: unknown) => { console.warn("CanvasTTY browser tab restore setting could not be applied.", error); }); + browserService?.setPauseHiddenTabs(next.browserPauseHiddenTabs); browserService?.cancelCanvasNavigationGesture(); browserService?.setCanvasWheelCaptureMode(next.canvasWheelCaptureMode); canvasNavigationInput?.setBindings({ @@ -1267,3 +1270,13 @@ function securePluginStorageAvailable(): boolean { if (!safeStorage.isEncryptionAvailable()) return false; return process.platform !== "linux" || safeStorage.getSelectedStorageBackend() !== "basic_text"; } + +/** + * Shorter pause/sleep delays for measurements and smoke runs: "freezeMs,discardMs" (for example "3000,8000"). + * Anything else is ignored and the defaults (30 s, 10 min) apply. + */ +function browserLifecycleTimingOverride(value: string | undefined): Pick { + const match = /^(\d{3,9}),(\d{3,9})$/u.exec(value ?? ""); + if (!match) return {}; + return { tabLifecycle: { freezeAfterMs: Number(match[1]), discardAfterMs: Number(match[2]) } }; +} diff --git a/src/main/services/BrowserService.ts b/src/main/services/BrowserService.ts index 8f893c17..b074eeae 100644 --- a/src/main/services/BrowserService.ts +++ b/src/main/services/BrowserService.ts @@ -10,7 +10,7 @@ import { View, WebContentsView } from "electron"; -import type { DownloadItem, Session, WebContents, WebPreferences } from "electron"; +import type { DownloadItem, NavigationEntry, Session, WebContents, WebPreferences } from "electron"; import type { AgentPresenceSnapshot, BrowserActivityEvent, @@ -44,6 +44,11 @@ import { clipBrowserViewportBounds, normalizeBrowserViewportBounds, sameBrowserV import { BrowserCore, type BrowserCoreHost, type BrowserCoreTab } from "./browser/BrowserCore.ts"; import { BrowserKernelError } from "./browser/BrowserErrors.ts"; import { NativeViewSync } from "./browser/NativeViewSync.ts"; +import { + BrowserTabLifecycle, + type TabLifecycleOptions, + type TabLifecycleState +} from "./browser/BrowserTabLifecycle.ts"; import { BrowserPolicyService, DEFAULT_BROWSER_URL, @@ -61,6 +66,12 @@ const BROWSER_PARTITION = "persist:canvastty-browser"; const MAX_DOWNLOAD_HISTORY = 100; const MAX_FAVICON_BYTES = 256 * 1024; const HUMAN_ACTOR: BrowserActor = { kind: "human", connectionId: "canvastty-renderer" }; +/** A sleeping tab's card picture: small enough to ride along in every browser state update. */ +const SLEEP_PREVIEW_MAX_WIDTH = 480; +const SLEEP_PREVIEW_JPEG_QUALITY = 60; +/** Waits bounded so a stuck page cannot hold a tab's lifecycle queue (and the commands behind it). */ +const SLEEP_CAPTURE_TIMEOUT_MS = 2_000; +const WAKE_LOAD_TIMEOUT_MS = 10_000; // Mirrors the `.browser-card { border-radius: 17px }` declaration in src/renderer/src/styles/app.css: the // stylesheet owns this visual property, so the two have to stay in sync by hand. @@ -87,6 +98,22 @@ interface BrowserTab { canvasSinkViewport: BrowserCanvasSinkViewportController; /** Mirrors `view.setVisible` so background throttling can be recomputed without an Electron getter. */ visible: boolean; + /** Paused by the hidden-tab lifecycle (CDP Page.setWebLifecycleState frozen). */ + frozen: boolean; + /** Between media-started-playing and media-paused: a playing tab is never paused. */ + mediaPlaying: boolean; + /** Set while the tab sleeps: its WebContents is closed and this is what brings it back. */ + sleeping: SleepingTab | null; +} + +interface SleepingTab { + title: string; + canGoBack: boolean; + canGoForward: boolean; + /** Back/forward history with each entry's page state (scroll position, form state), for navigationHistory.restore. */ + entries: NavigationEntry[]; + index: number; + preview: string | null; } interface DownloadWaiter { @@ -107,6 +134,10 @@ export interface BrowserServiceOptions { canvasWheelCaptureMode?: CanvasWheelCaptureMode; now?: () => number; canvasNavigationInput?: CanvasNavigationInputController; + /** "Pause hidden browser tabs": freeze hidden tabs after a while and put long-hidden ones to sleep. */ + pauseHiddenTabs?: boolean; + /** Timing overrides for the hidden-tab lifecycle (measurements and smoke runs). */ + tabLifecycle?: Pick; } export class BrowserService { @@ -119,7 +150,13 @@ export class BrowserService { private readonly policy: BrowserPolicyService; private readonly audit: BrowserAuditStore; private readonly busyAutomationTabs = new Set(); - private readonly automation = new BrowserAutomationService((tabId, busy) => this.setTabAutomationBusy(tabId, busy)); + private readonly lifecycle: BrowserTabLifecycle; + /** Sleeping tabs a show has already asked to wake, so a burst of view syncs asks once. */ + private readonly wakingTabs = new Set(); + private readonly automation = new BrowserAutomationService( + (tabId, busy) => this.setTabAutomationBusy(tabId, busy), + { beforeCommand: async (tabId) => { await this.lifecycle.ensureLive(tabId); } } + ); private readonly agents: AgentRegistry; private readonly canvasGestures: BrowserCanvasGestureController; private readonly canvasPointers: BrowserCanvasPointerRouter; @@ -159,6 +196,18 @@ export class BrowserService { const userDataPath = options.userDataPath ?? app.getPath("userData"); const downloadRoot = join(options.downloadRoot ?? join(app.getPath("downloads"), "CanvasTTY"), randomUUID()); this.restoreTabsEnabled = options.restoreTabs ?? true; + this.lifecycle = new BrowserTabLifecycle({ + freezeBlocker: (tabId) => this.freezeBlocker(tabId), + discardBlocker: (tabId) => this.discardBlocker(tabId), + freeze: (tabId) => this.freezeTab(tabId), + resume: (tabId) => this.resumeTab(tabId), + discard: (tabId) => this.sleepTab(tabId), + restore: (tabId) => this.wakeTab(tabId), + stateChanged: (tabId, state) => this.lifecycleChanged(tabId, state) + }, { + enabled: options.pauseHiddenTabs ?? true, + ...options.tabLifecycle + }); this.store = new BrowserStore(userDataPath); this.policy = new BrowserPolicyService({ downloadRoot, @@ -230,7 +279,8 @@ export class BrowserService { waitForDownload: (tabId, timeoutMs, signal) => this.waitForDownload(tabId, timeoutMs, signal), touchActor: (actor, tabId, cursor) => this.touchActor(actor, tabId, cursor), heartbeatActor: (actor, timestamp) => this.heartbeatActor(actor, timestamp), - disconnectActor: (actor) => this.disconnectActor(actor) + disconnectActor: (actor) => this.disconnectActor(actor), + prepareTab: (tabId) => this.lifecycle.ensureLive(tabId) }; this.core = new BrowserCore({ host, @@ -323,10 +373,15 @@ export class BrowserService { } } + setPauseHiddenTabs(enabled: boolean): void { + this.lifecycle.setEnabled(enabled); + } + async dispose(): Promise { if (this.disposed) return; await this.readyPromise.catch(() => undefined); this.disposed = true; + this.lifecycle.dispose(); const draining = this.core.shutdown(); await this.persistRuntime().catch(() => undefined); this.visible = false; @@ -485,7 +540,7 @@ export class BrowserService { this.requireOwner(); this.visible = true; for (const [id, tab] of this.tabs) { - if (!tab.view.webContents.isDestroyed()) continue; + if (tab.sleeping || !tab.view.webContents.isDestroyed()) continue; this.automation.unregister(id); this.tabs.delete(id); } @@ -615,6 +670,12 @@ export class BrowserService { private async hostReload(tabId: string): Promise { await this.ensureRuntime(); + this.requireTab(tabId); + // A sleeping tab's wake is a load of its page already; a paused one resumes before it reloads. + if ((await this.lifecycle.ensureLive(tabId)).reloaded) { + this.emit(); + return this.getState(); + } let tab = this.requireTab(tabId); if (tab.view.webContents.isDestroyed()) { const url = tab.lastSafeUrl; @@ -642,6 +703,9 @@ export class BrowserService { const view = existingContents ? new WebContentsView({ webContents: existingContents }) : new WebContentsView({ webPreferences: remoteBrowserWebPreferences() }); + // Electron's `view.webContents` getter returns undefined once the WebContents is closed (a sleeping or destroyed + // tab), and every `tab.view.webContents.isDestroyed()` check would throw. Pin the object: closed, it still answers. + Object.defineProperty(view, "webContents", { value: view.webContents, configurable: true, enumerable: true }); const tab: BrowserTab = { id, view, @@ -653,9 +717,13 @@ export class BrowserService { lastSafeUrl: url, canvasCursor: new BrowserCanvasCursorController(view.webContents), canvasSinkViewport: new BrowserCanvasSinkViewportController(view.webContents), - visible: false + visible: false, + frozen: false, + mediaPlaying: false, + sleeping: null }; this.tabs.set(id, tab); + this.lifecycle.track(id, false); // A reused popup WebContents may already be running unthrottled; recompute from our own state. this.applyBackgroundThrottling(tab); tab.canvasCursor.set(browserCanvasNavigationCursor(this.canvasNavigationInput?.active ?? false, false)); @@ -774,6 +842,8 @@ export class BrowserService { this.emit(); }); contents.on("did-finish-load", () => tab.canvasCursor.refresh()); + contents.on("media-started-playing", () => { tab.mediaPlaying = true; }); + contents.on("media-paused", () => { tab.mediaPlaying = false; }); contents.on("did-fail-load", (_event, errorCode, _errorDescription, _url, isMainFrame) => { if (!isMainFrame || errorCode === -3) return; tab.loading = false; @@ -828,7 +898,8 @@ export class BrowserService { if (this.activeTabId === tab.id) { this.invalidateCanvasSequence(false); } - if (!this.tabs.has(tab.id)) return; + // A closed tab, or one put to sleep (or already woken into a new WebContents), did not crash. + if (this.tabs.get(tab.id) !== tab || tab.sleeping) return; tab.loading = false; tab.status = "crashed"; tab.crashState = "destroyed"; @@ -868,7 +939,9 @@ export class BrowserService { event.preventDefault(); return; } - const tabId = [...this.tabs.values()].find((tab) => tab.view.webContents.id === contents.id)?.id ?? null; + const tabId = [...this.tabs.values()].find((tab) => ( + !tab.view.webContents.isDestroyed() && tab.view.webContents.id === contents.id + ))?.id ?? null; const download: BrowserDownloadSnapshot = { id, tabId, @@ -1033,6 +1106,8 @@ export class BrowserService { } private destroyTab(tab: BrowserTab): void { + this.lifecycle.untrack(tab.id); + this.wakingTabs.delete(tab.id); this.canvasPointers.cancelTab(tab.id); if (this.activeTabId === tab.id) { this.invalidateCanvasSequence(false); @@ -1062,6 +1137,9 @@ export class BrowserService { const right = left + (visibleRectangle?.width ?? 0); const bottom = top + (visibleRectangle?.height ?? 0); const active = this.activeTabId ? this.tabs.get(this.activeTabId) : undefined; + if (active?.sleeping && this.visible && this.viewport.surface === "native" && visibleRectangle !== null) { + this.wakeForShow(active.id); + } if (!active || active.view.webContents.isDestroyed()) { this.hideClipView(); this.syncPresenceOverlay(null); @@ -1150,9 +1228,16 @@ export class BrowserService { /** Mirrors `view.setVisible` into `tab.visible` and recomputes background throttling for it. */ private setTabVisible(tab: BrowserTab, visible: boolean): void { + // A sleeping tab has no page to show; syncViews wakes it first. + if (tab.sleeping) { + tab.visible = false; + return; + } this.native.setVisible(tab.view, visible); + const changed = tab.visible !== visible; tab.visible = visible; this.applyBackgroundThrottling(tab); + if (changed && this.tabs.get(tab.id) === tab) this.lifecycle.setVisible(tab.id, visible); } private setTabAutomationBusy(tabId: string, busy: boolean): void { @@ -1160,6 +1245,126 @@ export class BrowserService { else this.busyAutomationTabs.delete(tabId); const tab = this.tabs.get(tabId); if (tab) this.applyBackgroundThrottling(tab); + this.lifecycle.setBusy(tabId, busy); + } + + private wakeForShow(tabId: string): void { + if (this.wakingTabs.has(tabId)) return; + this.wakingTabs.add(tabId); + void this.lifecycle.ensureLive(tabId).catch((error: unknown) => { + console.warn("CanvasTTY could not wake a sleeping browser tab.", error); + }).finally(() => { + this.wakingTabs.delete(tabId); + if (this.disposed) return; + this.syncViews(); + this.emit(); + }); + } + + /** Why a hidden tab must keep running now: anything the person or a page could lose by a pause. */ + private freezeBlocker(tabId: string): string | null { + const tab = this.tabs.get(tabId); + if (!tab || tab.sleeping || tab.view.webContents.isDestroyed()) return "closed"; + const contents = tab.view.webContents; + if (tab.status === "crashed") return "crashed"; + if (tab.loading) return "loading"; + if (tab.mediaPlaying || contents.isCurrentlyAudible()) return "media"; + if (this.pendingDialogs.has(tabId)) return "dialog"; + if (this.downloads.some((download) => download.tabId === tabId && download.completedAt === null)) return "download"; + if (contents.isBeingCaptured()) return "capture"; + if (contents.isDevToolsOpened()) return "devtools"; + return null; + } + + /** Sleeping also loses what the page holds in memory: never with an agent on the tab or a beforeunload handler. */ + private async discardBlocker(tabId: string): Promise { + const blocker = this.freezeBlocker(tabId); + if (blocker) return blocker; + if (this.agents.forTab(tabId).length > 0) return "agent"; + if (await this.automation.hasBeforeUnload(tabId)) return "beforeunload"; + return this.freezeBlocker(tabId); + } + + private async freezeTab(tabId: string): Promise { + const tab = this.tabs.get(tabId); + if (!tab || tab.sleeping || tab.view.webContents.isDestroyed()) return; + // Chromium keeps a page frozen only while it may be throttled; a hidden, undriven tab already is. + this.applyBackgroundThrottling(tab); + await this.automation.setLifecycleState(tabId, "frozen"); + } + + private async resumeTab(tabId: string): Promise { + const tab = this.tabs.get(tabId); + if (!tab || tab.sleeping || tab.view.webContents.isDestroyed()) return; + await this.automation.setLifecycleState(tabId, "active"); + } + + private lifecycleChanged(tabId: string, state: TabLifecycleState): void { + const tab = this.tabs.get(tabId); + if (tab) tab.frozen = state === "frozen"; + this.emit(); + } + + /** + * Puts a hidden tab to sleep: its WebContents is closed, and the tab keeps its address, title, favicon, history + * (with each entry's scroll and form state) and a small picture for the card. False when the tab woke meanwhile. + */ + private async sleepTab(tabId: string): Promise { + const tab = this.tabs.get(tabId); + if (!tab || tab.sleeping || tab.view.webContents.isDestroyed()) return false; + const contents = tab.view.webContents; + const preview = await boundedWait(capturePreview(contents), SLEEP_CAPTURE_TIMEOUT_MS).catch(() => null); + if (this.tabs.get(tabId) !== tab || tab.visible || this.busyAutomationTabs.has(tabId) || contents.isDestroyed()) { + return false; + } + const history = contents.navigationHistory; + tab.lastSafeUrl = this.tabUrl(tab); + tab.sleeping = { + title: contents.getTitle(), + canGoBack: history.canGoBack(), + canGoForward: history.canGoForward(), + entries: history.getAllEntries(), + index: history.getActiveIndex(), + preview + }; + this.canvasPointers.cancelTab(tab.id); + if (this.activeTabId === tab.id) this.invalidateCanvasSequence(false); + tab.canvasCursor.dispose(); + tab.canvasSinkViewport.dispose(); + this.automation.unregister(tab.id); + this.clipView.removeChildView(tab.view); + if (this.clipTabId === tab.id) this.clipTabId = null; + if (this.pointerTabId === tab.id) this.pointerTabId = null; + this.busyAutomationTabs.delete(tab.id); + this.pendingDialogs.delete(tab.id); + tab.frozen = false; + tab.loading = false; + tab.mediaPlaying = false; + contents.close({ waitForBeforeUnload: false }); + this.emit(); + return true; + } + + /** Brings a sleeping tab back in a new WebContents, on the history entry it slept on. */ + private async wakeTab(tabId: string): Promise { + const asleep = this.tabs.get(tabId); + const saved = asleep?.sleeping; + if (!asleep || !saved || this.disposed) return; + const tab = this.createRuntimeTab(tabId, asleep.lastSafeUrl, asleep.documentRevision + 1); + tab.favicon = asleep.favicon; + const current = saved.entries[saved.index]; + let loading: Promise; + if (current && isSafeBrowserUrl(current.url)) { + loading = tab.view.webContents.navigationHistory.restore({ entries: saved.entries, index: saved.index }) + .catch(() => this.loadTab(tab, asleep.lastSafeUrl)); + } else { + loading = this.loadTab(tab, asleep.lastSafeUrl); + } + this.syncViews(); + this.emit(); + // A command after the wake finds the page loaded (or as far as it got): a slow page cannot hold the queue. + await boundedWait(loading, WAKE_LOAD_TIMEOUT_MS).catch(() => undefined); + if (this.tabs.get(tabId) === tab) void this.persistRuntime(); } /** @@ -1315,6 +1520,26 @@ export class BrowserService { private tabSnapshot(tab: BrowserTab, agents: readonly AgentPresenceSnapshot[]): BrowserTabSnapshot { const contents = tab.view.webContents; const url = this.tabUrl(tab); + const tabAgents = agents + .filter((presence) => presence.currentTabId === tab.id) + .map((presence) => structuredClone(presence)); + if (tab.sleeping) { + return { + id: tab.id, + url, + title: tab.sleeping.title || displayUrl(url), + loading: false, + canGoBack: tab.sleeping.canGoBack, + canGoForward: tab.sleeping.canGoForward, + documentRevision: tab.documentRevision, + status: "ready", + favicon: tab.favicon, + agents: tabAgents, + crashState: null, + lifecycle: "sleeping", + ...(this.activeTabId === tab.id ? { preview: tab.sleeping.preview } : {}) + }; + } const title = contents.isDestroyed() ? "" : contents.getTitle(); return { id: tab.id, @@ -1326,8 +1551,9 @@ export class BrowserService { documentRevision: tab.documentRevision, status: tab.status, favicon: tab.favicon, - agents: agents.filter((presence) => presence.currentTabId === tab.id).map((presence) => structuredClone(presence)), - crashState: tab.crashState + agents: tabAgents, + crashState: tab.crashState, + ...(tab.frozen ? { lifecycle: "paused" as const } : {}) }; } @@ -1455,3 +1681,26 @@ async function readBoundedResponse(response: Response, maxBytes: number): Promis } return Buffer.concat(chunks, size); } + +/** A small JPEG of the page for a sleeping tab's card, or null when the page draws nothing. */ +async function capturePreview(contents: WebContents): Promise { + let image = await contents.capturePage(undefined, { stayHidden: true, stayAwake: true }); + if (image.isEmpty()) return null; + const size = image.getSize(); + if (size.width <= 0 || size.height <= 0) return null; + if (size.width > SLEEP_PREVIEW_MAX_WIDTH) { + image = image.resize({ width: SLEEP_PREVIEW_MAX_WIDTH, quality: "good" }); + } + return `data:image/jpeg;base64,${image.toJPEG(SLEEP_PREVIEW_JPEG_QUALITY).toString("base64")}`; +} + +function boundedWait(promise: Promise, ms: number): Promise { + let timer: NodeJS.Timeout | undefined; + return Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error("Timed out.")), ms); + timer.unref?.(); + }) + ]).finally(() => clearTimeout(timer)); +} diff --git a/src/main/services/SettingsStore.ts b/src/main/services/SettingsStore.ts index 5ba49188..90c21cf0 100644 --- a/src/main/services/SettingsStore.ts +++ b/src/main/services/SettingsStore.ts @@ -407,6 +407,7 @@ function createDefaults(systemLocale: string, platform: string): AppSettings { browserAgentAccess: true, browserShowAgentPresence: true, browserRestoreTabs: true, + browserPauseHiddenTabs: true, attentionNotifications: true, attentionQueueVisible: true, attentionQueuePlacement: "bottom-right", @@ -676,6 +677,9 @@ export function normalizeSettings( browserRestoreTabs: typeof source.browserRestoreTabs === "boolean" ? source.browserRestoreTabs : fallback.browserRestoreTabs, + browserPauseHiddenTabs: typeof source.browserPauseHiddenTabs === "boolean" + ? source.browserPauseHiddenTabs + : fallback.browserPauseHiddenTabs, attentionNotifications: typeof source.attentionNotifications === "boolean" ? source.attentionNotifications : fallback.attentionNotifications, diff --git a/src/main/services/browser/BrowserAutomationService.ts b/src/main/services/browser/BrowserAutomationService.ts index d6b721e8..d4d2ecd4 100644 --- a/src/main/services/browser/BrowserAutomationService.ts +++ b/src/main/services/browser/BrowserAutomationService.ts @@ -28,6 +28,8 @@ const WAIT_POLL_MAX_MS = 1_000; const AUTOMATION_BUSY_GRACE_MS = 500; export const BROWSER_SCREENSHOT_MAX_BINARY_BYTES = 340 * 1024; const CDP_VERSION = "1.3"; +/** How long a lifecycle probe (beforeunload listeners, scroll position) may take before the answer is "unknown". */ +const LIFECYCLE_PROBE_TIMEOUT_MS = 2_000; const PRESENCE_WORLD = "canvastty-agent-presence"; const INTERACTIVE_ROLES = new Set([ "button", "checkbox", "combobox", "link", "listbox", @@ -147,13 +149,65 @@ export interface BrowserPointerResult { y: number; } +export interface BrowserAutomationOptions { + /** + * Runs before every automation command reaches the page, after the tab is marked busy: BrowserService resumes a + * paused (frozen) tab here, so no CDP command waits on a page whose tasks are stopped. + */ + beforeCommand?(tabId: string): Promise; +} + export class BrowserAutomationService { private readonly sessions = new Map(); private readonly busyTimers = new Map(); private readonly onBusyChange?: (tabId: string, busy: boolean) => void; + private readonly beforeCommand?: (tabId: string) => Promise; - constructor(onBusyChange?: (tabId: string, busy: boolean) => void) { + constructor(onBusyChange?: (tabId: string, busy: boolean) => void, options: BrowserAutomationOptions = {}) { this.onBusyChange = onBusyChange; + this.beforeCommand = options.beforeCommand; + } + + /** + * Freezes or resumes the page through the tab's own debugger attachment (CDP Page.setWebLifecycleState). This is + * not an automation command: it does not mark the tab busy. Chromium only keeps a page frozen while background + * throttling is allowed for it, which BrowserService guarantees for the hidden, undriven tabs it freezes. + */ + async setLifecycleState(tabId: string, state: "frozen" | "active"): Promise { + const session = this.sessions.get(tabId); + if (!session || session.contents.isDestroyed()) return; + await this.attach(session); + await session.contents.debugger.sendCommand("Page.setWebLifecycleState", { state }); + } + + /** + * Whether the top document has a beforeunload handler (listener or `onbeforeunload`): the page may hold input the + * person has not saved. Any failure or a slow answer counts as yes, so an unknown page is never discarded. + */ + async hasBeforeUnload(tabId: string): Promise { + const session = this.sessions.get(tabId); + if (!session || session.contents.isDestroyed()) return true; + const group = "canvastty-lifecycle"; + const probe = async (): Promise => { + await this.attach(session); + const debuggerApi = session.contents.debugger; + try { + const windowObject = await debuggerApi.sendCommand("Runtime.evaluate", { + expression: "window", + objectGroup: group, + silent: true + }) as { result?: { objectId?: string } }; + const objectId = windowObject.result?.objectId; + if (!objectId) return true; + const listeners = await debuggerApi.sendCommand("DOMDebugger.getEventListeners", { objectId }) as { + listeners?: Array<{ type?: string }>; + }; + return (listeners.listeners ?? []).some((listener) => listener.type === "beforeunload"); + } finally { + await debuggerApi.sendCommand("Runtime.releaseObjectGroup", { objectGroup: group }).catch(() => undefined); + } + }; + return await withTimeout(probe(), LIFECYCLE_PROBE_TIMEOUT_MS).catch(() => true); } /** @@ -901,6 +955,19 @@ export class BrowserAutomationService { } if (session.revision !== revision) throw staleRef(session.revision); this.markBusy(tabId); + if (this.beforeCommand) { + try { + await this.beforeCommand(tabId); + } catch (error) { + throw new BrowserKernelError("BRIDGE_UNAVAILABLE", "Browser tab could not be resumed.", { + retryable: true, + cause: error + }); + } + if (this.sessions.get(tabId) !== session || session.contents.isDestroyed()) { + throw new BrowserKernelError("TAB_CLOSED", "Browser tab is closed."); + } + } await this.attach(session); return session; } @@ -1489,3 +1556,14 @@ function abortableDelay(ms: number, signal?: AbortSignal): Promise { else signal.addEventListener("abort", abort, { once: true }); }); } + +function withTimeout(promise: Promise, ms: number): Promise { + let timer: NodeJS.Timeout | undefined; + return Promise.race([ + promise, + new Promise((_resolve, reject) => { + timer = setTimeout(() => reject(new Error("Timed out.")), ms); + timer.unref?.(); + }) + ]).finally(() => clearTimeout(timer)); +} diff --git a/src/main/services/browser/BrowserCommandDispatcher.ts b/src/main/services/browser/BrowserCommandDispatcher.ts index a69718bb..b3d743d3 100644 --- a/src/main/services/browser/BrowserCommandDispatcher.ts +++ b/src/main/services/browser/BrowserCommandDispatcher.ts @@ -37,6 +37,7 @@ export interface BrowserAuditWriter { export interface BrowserDispatchExecution { data?: unknown; tabId?: string | null; + notice?: string; } export interface BrowserCommandDispatcherOptions { @@ -277,7 +278,8 @@ export class BrowserCommandDispatcher { commandSequence, revisionBefore, revisionAfter, - ...(execution.data === undefined ? {} : { data: execution.data }) + ...(execution.data === undefined ? {} : { data: execution.data }), + ...(execution.notice ? { notice: execution.notice } : {}) }; } catch (error) { const normalized = timed.timedOut diff --git a/src/main/services/browser/BrowserCore.ts b/src/main/services/browser/BrowserCore.ts index 2397aea8..886ff2bd 100644 --- a/src/main/services/browser/BrowserCore.ts +++ b/src/main/services/browser/BrowserCore.ts @@ -39,8 +39,16 @@ export interface BrowserCoreHost { touchActor(actor: BrowserActor, tabId: string | null, cursor?: BrowserPointerResult): void; heartbeatActor(actor: BrowserActor, timestamp: number): void; disconnectActor(actor: BrowserActor): void; + /** + * Wakes a tab CanvasTTY paused or put to sleep while it was hidden, before a command for it runs. `reloaded` means + * the tab's page was loaded again: its earlier element refs are stale. + */ + prepareTab?(tabId: string): Promise<{ reloaded: boolean }>; } +export const BROWSER_TAB_RELOADED_NOTICE = + "Browser tab was reloaded: CanvasTTY had put it to sleep while it was hidden. Element refs from before are stale; observe it again."; + export interface BrowserCoreOptions { host: BrowserCoreHost; automation: BrowserAutomationService; @@ -116,7 +124,7 @@ export class BrowserCore { actor: BrowserActor, command: BrowserCommand, signal: AbortSignal - ): Promise<{ data?: unknown; tabId?: string | null }> { + ): Promise<{ data?: unknown; tabId?: string | null; notice?: string }> { throwIfAborted(signal); assertCommandArguments(command); const tabId = this.resolveTabId(command); @@ -152,6 +160,30 @@ export class BrowserCore { const requiredTabId = tabId ?? (() => { throw new BrowserKernelError("TAB_NOT_FOUND", "Browser command requires a tab."); })(); + // Closing needs no live page, and reload wakes a sleeping tab itself (that load is the reload). + const wake = command.type === "browser_close_tab" || command.type === "browser_reload" || !this.host.prepareTab + ? { reloaded: false } + : await this.host.prepareTab(requiredTabId); + throwIfAborted(signal); + if (!wake.reloaded) return await this.executeTabCommand(actor, command, requiredTabId, signal); + try { + return { ...await this.executeTabCommand(actor, command, requiredTabId, signal), notice: BROWSER_TAB_RELOADED_NOTICE }; + } catch (error) { + if (!(error instanceof BrowserKernelError)) throw error; + throw new BrowserKernelError(error.code, error.message, { + retryable: error.retryable, + details: { ...(error.details ?? {}), tabReloaded: true }, + cause: error + }); + } + } + + private async executeTabCommand( + actor: BrowserActor, + command: BrowserCommand, + requiredTabId: string, + signal: AbortSignal + ): Promise<{ data?: unknown; tabId?: string | null }> { const tab = this.host.getTab(requiredTabId); if (!tab) throw new BrowserKernelError("TAB_NOT_FOUND", "Browser tab is unavailable."); if (tab.status === "crashed" && command.type !== "browser_reload" && command.type !== "browser_close_tab") { @@ -445,7 +477,8 @@ function sanitizeAgentValue(value: unknown, key = "", depth = 0): unknown { if (depth > 12) return "[REDACTED]"; if (value === null || typeof value === "number" || typeof value === "boolean") return value; const normalizedKey = key.toLowerCase(); - if (normalizedKey === "favicon") return null; + // The favicon and a sleeping tab's preview are for the person's card: raw page pixels, never redacted for agents. + if (normalizedKey === "favicon" || normalizedKey === "preview") return null; if (isSensitiveName(normalizedKey)) { return "[REDACTED]"; } diff --git a/src/main/services/browser/BrowserTabLifecycle.ts b/src/main/services/browser/BrowserTabLifecycle.ts new file mode 100644 index 00000000..6a9c8a42 --- /dev/null +++ b/src/main/services/browser/BrowserTabLifecycle.ts @@ -0,0 +1,324 @@ +/** + * When a hidden browser tab is paused (frozen) or put to sleep (discarded), and when it is woken again. + * + * Hidden tabs already run with Chromium background throttling, but throttling leaves requestAnimationFrame loops and + * a timer wake-up every second running. A tab that stays hidden and undriven for `freezeAfterMs` is frozen (Chromium + * stops its timers, rAF and tasks); one that stays so for `discardAfterMs`, or the least recently used ones when more + * than `maxLiveHiddenTabs` hidden tabs are alive, is discarded (its WebContents is closed; the host keeps what it needs + * to bring it back). Showing a tab, or any command for it, wakes it first: `ensureLive` resolves only once the tab + * runs again, so no automation reaches a frozen page. + * + * Every transition of one tab runs on that tab's own queue, so a wake that arrives while a freeze or discard is in + * flight waits for it and then undoes it. The host decides what blocks a transition (playing media, a download, an + * open dialog, a beforeunload handler, an agent on the tab); a blocked tab is retried later, never forced. + * This class holds no Electron objects: BrowserService is the host, and tests drive it with fake timers. + */ + +export type TabLifecycleState = "active" | "frozen" | "discarded"; + +export interface TabLifecycleHost { + /** Why the tab must not be frozen now, or null. */ + freezeBlocker(tabId: string): string | null; + /** Why the tab must not be discarded now, or null. May consult the page (a beforeunload handler). */ + discardBlocker(tabId: string): Promise; + freeze(tabId: string): Promise; + resume(tabId: string): Promise; + /** False when the tab could not be discarded after all (it was shown or driven meanwhile); it then stays as it was. */ + discard(tabId: string): Promise; + restore(tabId: string): Promise; + stateChanged(tabId: string, state: TabLifecycleState): void; +} + +export interface TabLifecycleOptions { + enabled?: boolean; + freezeAfterMs?: number; + discardAfterMs?: number; + maxLiveHiddenTabs?: number; + /** How long a blocked freeze or discard waits before it is tried again. */ + retryAfterMs?: number; + now?: () => number; + setTimer?: (callback: () => void, ms: number) => unknown; + clearTimer?: (handle: unknown) => void; + onError?: (error: unknown) => void; +} + +export const TAB_FREEZE_AFTER_MS = 30_000; +export const TAB_DISCARD_AFTER_MS = 10 * 60_000; +export const MAX_LIVE_HIDDEN_TABS = 6; + +interface Entry { + id: string; + state: TabLifecycleState; + visible: boolean; + busy: boolean; + /** When the tab was last shown, driven or woken; idle time counts from the later of this and hiding. */ + lastUsedAt: number; + hiddenSince: number | null; + freezeNotBefore: number; + discardNotBefore: number; + /** A discard for the hidden-tab limit is queued; the tab no longer counts as live. */ + discardQueued: boolean; + timer: unknown; + queue: Promise; +} + +export class BrowserTabLifecycle { + private readonly host: TabLifecycleHost; + private readonly entries = new Map(); + private readonly freezeAfterMs: number; + private readonly discardAfterMs: number; + private readonly maxLiveHiddenTabs: number; + private readonly retryAfterMs: number; + private readonly now: () => number; + private readonly setTimer: (callback: () => void, ms: number) => unknown; + private readonly clearTimer: (handle: unknown) => void; + private readonly onError: (error: unknown) => void; + private enabled: boolean; + private disposed = false; + + constructor(host: TabLifecycleHost, options: TabLifecycleOptions = {}) { + this.host = host; + this.enabled = options.enabled ?? true; + this.freezeAfterMs = options.freezeAfterMs ?? TAB_FREEZE_AFTER_MS; + this.discardAfterMs = options.discardAfterMs ?? TAB_DISCARD_AFTER_MS; + this.maxLiveHiddenTabs = options.maxLiveHiddenTabs ?? MAX_LIVE_HIDDEN_TABS; + this.retryAfterMs = options.retryAfterMs ?? this.freezeAfterMs; + this.now = options.now ?? Date.now; + this.setTimer = options.setTimer ?? ((callback, ms) => { + const timer = setTimeout(callback, ms); + timer.unref?.(); + return timer; + }); + this.clearTimer = options.clearTimer ?? ((handle) => clearTimeout(handle as NodeJS.Timeout)); + this.onError = options.onError ?? ((error) => console.warn("CanvasTTY browser tab lifecycle step failed.", error)); + } + + get isEnabled(): boolean { + return this.enabled; + } + + state(tabId: string): TabLifecycleState { + return this.entries.get(tabId)?.state ?? "active"; + } + + /** Starts tracking a tab (a no-op for one already tracked, such as a tab being restored). */ + track(tabId: string, visible: boolean): void { + if (this.disposed || this.entries.has(tabId)) return; + const now = this.now(); + const entry: Entry = { + id: tabId, + state: "active", + visible, + busy: false, + lastUsedAt: now, + hiddenSince: visible ? null : now, + freezeNotBefore: 0, + discardNotBefore: 0, + discardQueued: false, + timer: null, + queue: Promise.resolve() + }; + this.entries.set(tabId, entry); + this.schedule(entry); + } + + untrack(tabId: string): void { + const entry = this.entries.get(tabId); + if (!entry) return; + this.cancelTimer(entry); + this.entries.delete(tabId); + } + + setVisible(tabId: string, visible: boolean): void { + const entry = this.entries.get(tabId); + if (!entry || entry.visible === visible) return; + entry.visible = visible; + if (visible) { + entry.hiddenSince = null; + this.touch(entry); + if (entry.state !== "active") void this.ensureLive(tabId).catch(this.onError); + return; + } + entry.hiddenSince = this.now(); + this.resetRetries(entry); + this.schedule(entry); + } + + /** Automation started or stopped driving the tab (BrowserAutomationService's busy window). */ + setBusy(tabId: string, busy: boolean): void { + const entry = this.entries.get(tabId); + if (!entry || entry.busy === busy) return; + entry.busy = busy; + this.touch(entry); + if (!busy) this.schedule(entry); + } + + /** + * Resolves once the tab runs: a frozen tab is resumed, a discarded one restored (`reloaded`). Waits for any + * transition of the tab already in flight, so a wake never overtakes the freeze or discard it has to undo. + */ + ensureLive(tabId: string): Promise<{ reloaded: boolean }> { + const entry = this.entries.get(tabId); + if (!entry) return Promise.resolve({ reloaded: false }); + this.touch(entry); + const result = this.enqueue(entry, async () => { + if (this.entries.get(tabId) !== entry) return { reloaded: false }; + if (entry.state === "frozen") { + try { + await this.host.resume(tabId); + } finally { + // A resume that failed (the automation channel went away) must not leave the tab marked paused forever. + this.setState(entry, "active"); + } + return { reloaded: false }; + } + if (entry.state === "discarded") { + await this.host.restore(tabId); + this.setState(entry, "active"); + return { reloaded: true }; + } + return { reloaded: false }; + }); + return result.finally(() => this.schedule(entry)); + } + + /** Off: nothing new is paused and paused tabs resume; sleeping tabs wake when they are next shown or used. */ + setEnabled(enabled: boolean): void { + if (this.enabled === enabled) return; + this.enabled = enabled; + for (const entry of this.entries.values()) { + this.resetRetries(entry); + if (!enabled) { + this.cancelTimer(entry); + if (entry.state === "frozen") void this.ensureLive(entry.id).catch(this.onError); + } else { + // Idle time counts from now: turning the setting on must not put long-hidden tabs to sleep at once. + entry.lastUsedAt = this.now(); + this.schedule(entry); + } + } + } + + dispose(): void { + this.disposed = true; + for (const entry of this.entries.values()) this.cancelTimer(entry); + this.entries.clear(); + } + + private touch(entry: Entry): void { + entry.lastUsedAt = this.now(); + this.resetRetries(entry); + this.cancelTimer(entry); + } + + private resetRetries(entry: Entry): void { + entry.freezeNotBefore = 0; + entry.discardNotBefore = 0; + } + + private idle(entry: Entry): boolean { + return this.enabled && !this.disposed && !entry.visible && !entry.busy && this.entries.get(entry.id) === entry; + } + + private idleSince(entry: Entry): number { + return Math.max(entry.hiddenSince ?? entry.lastUsedAt, entry.lastUsedAt); + } + + private freezeAt(entry: Entry): number { + return Math.max(this.idleSince(entry) + this.freezeAfterMs, entry.freezeNotBefore); + } + + private discardAt(entry: Entry): number { + return Math.max(this.idleSince(entry) + this.discardAfterMs, entry.discardNotBefore); + } + + private schedule(entry: Entry): void { + this.cancelTimer(entry); + if (!this.idle(entry) || entry.state === "discarded") return; + const due = entry.state === "active" + ? Math.min(this.freezeAt(entry), this.discardAt(entry)) + : this.discardAt(entry); + entry.timer = this.setTimer(() => { + entry.timer = null; + void this.enqueue(entry, () => this.step(entry)).catch(this.onError); + }, Math.max(0, due - this.now())); + } + + private cancelTimer(entry: Entry): void { + if (entry.timer === null) return; + this.clearTimer(entry.timer); + entry.timer = null; + } + + private async step(entry: Entry): Promise { + try { + if (!this.idle(entry)) return; + const now = this.now(); + if (entry.state === "active" && now >= this.freezeAt(entry)) { + const blocker = this.host.freezeBlocker(entry.id); + if (blocker) entry.freezeNotBefore = now + this.retryAfterMs; + else { + await this.host.freeze(entry.id); + if (this.entries.get(entry.id) === entry) this.setState(entry, "frozen"); + } + } + if (entry.state !== "discarded" && this.idle(entry) && this.now() >= this.discardAt(entry)) { + await this.tryDiscard(entry); + } + this.enforceHiddenLimit(); + } finally { + this.schedule(entry); + } + } + + private async tryDiscard(entry: Entry): Promise { + if (!this.idle(entry) || entry.state === "discarded") return; + const blocker = await this.host.discardBlocker(entry.id); + // Shown, driven or untracked while the page was asked: leave it. + if (!this.idle(entry) || this.state(entry.id) === "discarded") return; + if (blocker) { + entry.discardNotBefore = this.now() + this.retryAfterMs; + return; + } + const discarded = await this.host.discard(entry.id); + if (discarded && this.entries.get(entry.id) === entry) this.setState(entry, "discarded"); + } + + /** More than maxLiveHiddenTabs hidden tabs alive: discard the least recently used that have been idle a while. */ + private enforceHiddenLimit(): void { + const live = [...this.entries.values()].filter((entry) => ( + !entry.visible && entry.state !== "discarded" && !entry.discardQueued + )); + let excess = live.length - this.maxLiveHiddenTabs; + if (excess <= 0) return; + const now = this.now(); + const candidates = live + .filter((entry) => this.idle(entry) && now - this.idleSince(entry) >= this.freezeAfterMs && now >= entry.discardNotBefore) + .sort((left, right) => left.lastUsedAt - right.lastUsedAt); + for (const entry of candidates) { + if (excess <= 0) break; + excess -= 1; + entry.discardQueued = true; + void this.enqueue(entry, async () => { + try { + await this.tryDiscard(entry); + } finally { + entry.discardQueued = false; + this.schedule(entry); + } + }).catch(this.onError); + } + } + + private setState(entry: Entry, state: TabLifecycleState): void { + if (entry.state === state) return; + entry.state = state; + this.host.stateChanged(entry.id, state); + } + + private enqueue(entry: Entry, task: () => Promise): Promise { + const run = entry.queue.then(task, task); + entry.queue = run.catch(() => undefined); + return run; + } +} diff --git a/src/renderer/src/App.tsx b/src/renderer/src/App.tsx index 6d36491a..c7b4a429 100644 --- a/src/renderer/src/App.tsx +++ b/src/renderer/src/App.tsx @@ -153,6 +153,7 @@ const FALLBACK_SETTINGS: AppSettings = { browserAgentAccess: true, browserShowAgentPresence: true, browserRestoreTabs: true, + browserPauseHiddenTabs: true, attentionNotifications: true, attentionQueueVisible: true, attentionQueuePlacement: "bottom-right", diff --git a/src/renderer/src/features/browser/BrowserCard.tsx b/src/renderer/src/features/browser/BrowserCard.tsx index 4a1f80a9..457a772e 100644 --- a/src/renderer/src/features/browser/BrowserCard.tsx +++ b/src/renderer/src/features/browser/BrowserCard.tsx @@ -127,6 +127,8 @@ export function BrowserCard({ const freezeFrameDataUrl = freezeFrame && freezeFrame.tabId === activeTab?.id ? freezeFrame.dataUrl : null; + // A sleeping tab has no page behind the card: its last picture stands in until it reloads. + const sleepPreview = activeTab?.lifecycle === "sleeping" ? safeFavicon(activeTab.preview ?? null) : null; useEffect(() => { liveBounds.current = bounds; @@ -456,6 +458,11 @@ export function BrowserCard({ > {tab.title || t(locale, "newTab")} + {tab.lifecycle && ( + + {t(locale, tab.lifecycle === "paused" ? "browserTabPaused" : "browserTabSleeping")} + + )} {showAgentPresence && }