From f9de3404c5ddd79107651b55ef20ee4c26cd2529 Mon Sep 17 00:00:00 2001 From: dhiyaan Date: Tue, 1 Sep 2026 20:01:46 -0400 Subject: [PATCH] Open the thread's simulator tab when its agent starts driving (opt-in) A new boolean setting, openSimulatorOnDrive, off by default and also offered in the panel's gear menu. When it is on and a thread takes the simulator lease through the agent tools or the CLI, the server publishes a simulator-driven signal naming that thread, and the thread's composer opens the simulator threadPanelAction in its own side panel. A driving session, not a lease, is what gets announced: every tool call takes and releases the lease, so DriveAnnouncer collapses acquisitions by one thread with no gap longer than 90s into one announcement at the start. A tab the person closes therefore stays closed until the agent has been quiet and comes back, rather than reopening on the next gesture. The signal rides its own channel so the panels that refetch on every simulator-changed signal do not pay for one that is not about state. --- README.md | 7 +++++ app.tsx | 3 +- app/sim/ActivityBanner.tsx | 5 ++++ app/sim/useOpenOnDrive.ts | 37 +++++++++++++++++++++++ src/sim/announce.ts | 45 ++++++++++++++++++++++++++++ src/sim/channel.ts | 14 +++++++++ src/sim/options.ts | 6 ++++ src/sim/settings.ts | 11 +++++++ src/sim/wire.ts | 20 +++++++++++-- test/sim/announce.test.ts | 60 ++++++++++++++++++++++++++++++++++++++ test/sim/options.test.ts | 8 ++++- 11 files changed, 212 insertions(+), 4 deletions(-) create mode 100644 app/sim/useOpenOnDrive.ts create mode 100644 src/sim/announce.ts create mode 100644 test/sim/announce.test.ts diff --git a/README.md b/README.md index 1ab1ed3..2ad00d6 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,13 @@ The four simulator tools are off by default and gated behind the that is a decision the plugin refuses to make for you. Flipping it off revokes already-registered tools on their next call. +With `openSimulatorOnDrive` on (off by default; also in the panel's gear +menu), a thread's simulator tab opens in its side panel the moment its agent +starts driving — through the tools or the CLI — so you watch what the agent +does without going looking for it. It fires once per driving session, not per +gesture: a tab you close stays closed until the agent has been quiet for a +minute and a half and starts again. + Stream evidence is generation-scoped and correlated with `simulator_capture` in the tool's text; it never replaces or rewrites the full-resolution durable JPEG. `sourceFps` remains approximate because the current v1 AVCC stream has no diff --git a/app.tsx b/app.tsx index d4945c4..26f97ce 100644 --- a/app.tsx +++ b/app.tsx @@ -31,6 +31,7 @@ import { StillsDirective } from "./app/sim/StillsDirective"; import { ActivityBanner as SimulatorBanner } from "./app/sim/ActivityBanner"; import { ServerConfirm } from "./app/sim/ServerConfirm"; import { ThreadSimulator } from "./app/sim/ThreadSimulator"; +import { SIMULATOR_PANEL_ACTION } from "./app/sim/useOpenOnDrive"; import { PANEL_PATH } from "./app/sim/route"; export default definePluginApp((app) => { @@ -59,7 +60,7 @@ export default definePluginApp((app) => { * host's padded scroll container. */ app.slots.threadPanelAction({ - id: "simulator", + id: SIMULATOR_PANEL_ACTION, title: "Open simulator", icon: "Smartphone", layout: "flush", diff --git a/app/sim/ActivityBanner.tsx b/app/sim/ActivityBanner.tsx index f52fbe7..0ec1001 100644 --- a/app/sim/ActivityBanner.tsx +++ b/app/sim/ActivityBanner.tsx @@ -16,6 +16,7 @@ import { Icon } from "@/components/ui/icon"; import type { rpcContract } from "../../src/sim/wire"; import { PANEL_PATH } from "./route"; import { TONE_CLASS } from "./copy"; +import { useOpenOnDrive } from "./useOpenOnDrive"; interface Row { id: string; @@ -50,6 +51,10 @@ export function ActivityBanner() { useEffect(refresh, [refresh]); useRealtime("simulator-changed", refresh); + // This banner is the one plugin surface mounted in every thread's composer + // with the thread id and navigation in scope, so the auto-open lives here + // even though it draws nothing. + useOpenOnDrive(threadId); const visible = rows.filter((row) => !hidden.has(row.id)); if (visible.length === 0) return null; diff --git a/app/sim/useOpenOnDrive.ts b/app/sim/useOpenOnDrive.ts new file mode 100644 index 0000000..fbf33ac --- /dev/null +++ b/app/sim/useOpenOnDrive.ts @@ -0,0 +1,37 @@ +/** + * Open this thread's simulator tab when its agent starts driving. + * + * The server says so on `DRIVE_CHANNEL` — once per driving session, and only + * when the `openSimulatorOnDrive` setting is on, so a client never has to ask + * whether the feature is enabled. Every connected client hears every signal + * (V1 realtime has no per-channel subscriptions); this thread's composer is + * the one that acts, and only on a signal naming it. + * + * Opening the same action twice focuses the existing tab, so an announcement + * that lands while the tab is already open is a no-op rather than a duplicate. + * A tab the person closed stays closed until the next session — the server's + * quiet window, not this hook, decides when that is. + */ +import { useBbNavigate, useRealtime } from "@get-bb/plugin-sdk/app"; +import { DRIVE_CHANNEL, type DriveSignal } from "../../src/sim/channel.js"; + +/** The `threadPanelAction` id registered in `app.tsx`. */ +export const SIMULATOR_PANEL_ACTION = "simulator"; + +function isDriveSignal(payload: unknown): payload is DriveSignal { + return ( + typeof payload === "object" && + payload !== null && + typeof (payload as { threadId?: unknown }).threadId === "string" + ); +} + +export function useOpenOnDrive(threadId: string | null): void { + const navigate = useBbNavigate(); + useRealtime(DRIVE_CHANNEL, (payload: unknown) => { + if (threadId === null || !isDriveSignal(payload) || payload.threadId !== threadId) return; + // A declined open (no side panel on this surface) is logged by the host; + // there is nothing for a banner to do about it. + navigate.openThreadPanel({ actionId: SIMULATOR_PANEL_ACTION }); + }); +} diff --git a/src/sim/announce.ts b/src/sim/announce.ts new file mode 100644 index 0000000..f75239c --- /dev/null +++ b/src/sim/announce.ts @@ -0,0 +1,45 @@ +/** + * When does "an agent is driving the simulator" begin? + * + * Every tool call and every CLI gesture takes the lease and gives it back, so + * the lease alone cannot tell a *session* from a *step*: a thread tapping five + * times in ten seconds acquires five times. Announcing each one would reopen + * the side panel five times — and, worse, reopen a tab the person had just + * closed because they did not want it. + * + * So a session is a run of acquisitions by one thread with no gap longer than + * `quietMs`, and it is announced exactly once, at its first acquisition. The + * window slides: as long as the thread keeps driving, nothing new is said. A + * closed tab therefore stays closed until the agent stops for a while and + * comes back — the same rhythm the lease TTL already gives the human. + * + * Pure, and keyed on the thread rather than the device: it is the thread's + * panel that opens, whichever simulator it happens to be driving. + */ + +export const DRIVE_QUIET_MS = 90_000; + +export class DriveAnnouncer { + private lastSeen = new Map(); + + constructor( + private readonly quietMs: number = DRIVE_QUIET_MS, + private readonly now: () => number = Date.now, + ) {} + + /** + * Record that `threadId` just took the lease. Returns true when this is the + * first acquisition of a new session — the one worth telling the UI about. + */ + touch(threadId: string): boolean { + const at = this.now(); + const previous = this.lastSeen.get(threadId); + this.lastSeen.set(threadId, at); + return previous === undefined || at - previous > this.quietMs; + } + + /** Forget a thread, so its next acquisition announces again. */ + forget(threadId: string): void { + this.lastSeen.delete(threadId); + } +} diff --git a/src/sim/channel.ts b/src/sim/channel.ts index 3e52011..2c45192 100644 --- a/src/sim/channel.ts +++ b/src/sim/channel.ts @@ -23,3 +23,17 @@ export interface Signal { export function signal(kind: SignalKind): Signal { return { kind }; } + +/** + * A thread has started driving the simulator through the agent tools or the + * CLI. Its own composer, and nothing else, reacts by opening the simulator tab + * — see `app/sim/useOpenOnDrive.ts`. Separate from `CHANNEL` so the panels + * that refetch on every `simulator-changed` do not pay for a signal that is + * not about state. + */ +export const DRIVE_CHANNEL = "simulator-driven"; + +export interface DriveSignal { + threadId: string; + deviceUdid: string; +} diff --git a/src/sim/options.ts b/src/sim/options.ts index 67018bf..a627c4f 100644 --- a/src/sim/options.ts +++ b/src/sim/options.ts @@ -45,6 +45,12 @@ const SPECS: readonly UiOptionSpec[] = [ detail: "A bezel around the live frame.", defaultValue: false, }, + { + key: "openSimulatorOnDrive", + label: "Open the simulator when an agent drives it", + detail: "The thread's simulator tab opens as its agent starts driving.", + defaultValue: false, + }, ]; export function isUiOptionKey(key: string): boolean { diff --git a/src/sim/settings.ts b/src/sim/settings.ts index 0e83e6d..0762fbe 100644 --- a/src/sim/settings.ts +++ b/src/sim/settings.ts @@ -28,6 +28,8 @@ export interface Settings { allowIntelLive: boolean; allowAgentCapture: boolean; postChangedPreviews: boolean; + /** Open the thread's simulator tab when its agent starts driving. Off by default. */ + openSimulatorOnDrive: boolean; } export const DEFAULTS: Settings = { @@ -42,6 +44,7 @@ export const DEFAULTS: Settings = { allowIntelLive: false, allowAgentCapture: false, postChangedPreviews: true, + openSimulatorOnDrive: false, }; /** @@ -124,6 +127,13 @@ export const SETTINGS_DESCRIPTORS = { description: "Show a banner above the composer when a preview render finishes in this thread.", default: DEFAULTS.postChangedPreviews, }, + openSimulatorOnDrive: { + type: "boolean", + label: "Open the simulator when an agent drives it", + description: + "When a thread starts driving the simulator through the agent tools or the CLI, open the simulator tab in that thread's side panel. Off by default; needs allowAgentCapture to have anything to react to.", + default: DEFAULTS.openSimulatorOnDrive, + }, } as const; /** The raw shape `settings.get()` returns, before normalization. */ @@ -185,5 +195,6 @@ export function normalizeSettings(raw: RawSettings): Settings { allowIntelLive: parseBoolean(raw.allowIntelLive, DEFAULTS.allowIntelLive), allowAgentCapture: parseBoolean(raw.allowAgentCapture, DEFAULTS.allowAgentCapture), postChangedPreviews: parseBoolean(raw.postChangedPreviews, DEFAULTS.postChangedPreviews), + openSimulatorOnDrive: parseBoolean(raw.openSimulatorOnDrive, DEFAULTS.openSimulatorOnDrive), }; } diff --git a/src/sim/wire.ts b/src/sim/wire.ts index db85a7b..df351e7 100644 --- a/src/sim/wire.ts +++ b/src/sim/wire.ts @@ -20,7 +20,8 @@ import { isAbsolute, join, relative, resolve, sep } from "node:path"; import { tmpdir } from "node:os"; import { rpcContract } from "./contract.js"; -import { CHANNEL } from "./channel.js"; +import { CHANNEL, DRIVE_CHANNEL, type DriveSignal } from "./channel.js"; +import { DriveAnnouncer } from "./announce.js"; import { prepareConnection } from "./store.js"; import { dataDirOf, framesRootOf, type Ctx, type ThreadScope } from "./context.js"; import { FrameStore } from "./framestore.js"; @@ -230,6 +231,14 @@ export async function installSimulators(bb: BbPluginApi, host: SimulatorHost): P // resource: `stillsDevice` is one shared UDID by design, and two callers with // different scopes still drive the same simulator. const leases = new LeaseRegistry(() => live.currentDevice()?.name ?? "the simulator"); + // "Started driving" is a session, not a lease: see `src/sim/announce.ts`. + // Announced straight to the socket rather than through the coalescer, which + // exists for state signals; this one carries its own payload and is already + // at most once per thread per quiet window. + const drives = new DriveAnnouncer(); + const announceDrive = (payload: DriveSignal): void => { + safely(isDisposed, () => bb.realtime.publish(DRIVE_CHANNEL, payload)); + }; // The stills queue is keyed on the device UDID rather than the project: the // contended resource is the simulator, and `stillsDevice` is one shared UDID @@ -603,7 +612,14 @@ export async function installSimulators(bb: BbPluginApi, host: SimulatorHost): P leases: { acquire: (threadId) => { const device = live.currentDevice(); - return leases.acquire(device?.udid ?? "none", threadId); + const outcome = leases.acquire(device?.udid ?? "none", threadId); + // A person in the panel is `null` and already has the panel open. + if (outcome.ok && threadId !== null && device !== null && drives.touch(threadId)) { + if (settings.openSimulatorOnDrive) { + announceDrive({ threadId, deviceUdid: device.udid }); + } + } + return outcome; }, }, log, diff --git a/test/sim/announce.test.ts b/test/sim/announce.test.ts new file mode 100644 index 0000000..2e6427a --- /dev/null +++ b/test/sim/announce.test.ts @@ -0,0 +1,60 @@ +/** + * A driving session is announced once, at its start, and not again until the + * thread has been quiet for longer than the window. + */ +import { describe, expect, it } from "vitest"; +import { DriveAnnouncer } from "../../src/sim/announce.js"; + +function announcer(quietMs = 1_000) { + let now = 0; + const instance = new DriveAnnouncer(quietMs, () => now); + return { instance, advance: (ms: number) => (now += ms) }; +} + +describe("DriveAnnouncer", () => { + it("announces the first acquisition of a thread", () => { + const { instance } = announcer(); + expect(instance.touch("thr_a")).toBe(true); + }); + + it("stays silent while the same thread keeps driving inside the window", () => { + const { instance, advance } = announcer(1_000); + expect(instance.touch("thr_a")).toBe(true); + advance(400); + expect(instance.touch("thr_a")).toBe(false); + advance(900); + // 900ms since the last touch, not since the first: the window slides. + expect(instance.touch("thr_a")).toBe(false); + }); + + it("announces again once the thread has been quiet for longer than the window", () => { + const { instance, advance } = announcer(1_000); + instance.touch("thr_a"); + advance(1_001); + expect(instance.touch("thr_a")).toBe(true); + }); + + it("treats a gap of exactly the window as still driving", () => { + const { instance, advance } = announcer(1_000); + instance.touch("thr_a"); + advance(1_000); + expect(instance.touch("thr_a")).toBe(false); + }); + + it("tracks threads independently", () => { + const { instance, advance } = announcer(1_000); + expect(instance.touch("thr_a")).toBe(true); + advance(100); + expect(instance.touch("thr_b")).toBe(true); + advance(100); + expect(instance.touch("thr_a")).toBe(false); + expect(instance.touch("thr_b")).toBe(false); + }); + + it("announces again after forget, whatever the clock says", () => { + const { instance } = announcer(1_000); + instance.touch("thr_a"); + instance.forget("thr_a"); + expect(instance.touch("thr_a")).toBe(true); + }); +}); diff --git a/test/sim/options.test.ts b/test/sim/options.test.ts index 2ffd246..7e9b947 100644 --- a/test/sim/options.test.ts +++ b/test/sim/options.test.ts @@ -13,6 +13,7 @@ describe("the ui options allowlist", () => { ["showThreadActivity", true], ["postChangedPreviews", true], ["showDeviceChrome", false], + ["openSimulatorOnDrive", false], ]); const flipped = uiOptions({ showThreadActivity: false, showDeviceChrome: true }); @@ -48,7 +49,12 @@ describe("the ui options allowlist", () => { // Every allowlisted key must be one of the known display toggles — a new // key added to the menu has to be added HERE too, which is the point: two // lists that must agree force the security question to be asked twice. - const display = new Set(["showThreadActivity", "postChangedPreviews", "showDeviceChrome"]); + const display = new Set([ + "showThreadActivity", + "postChangedPreviews", + "showDeviceChrome", + "openSimulatorOnDrive", + ]); for (const option of uiOptions({})) { expect(display.has(option.key)).toBe(true); }