diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c0f4f188..0febab7f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -160,6 +160,17 @@ jobs: mkdir -p resources/bin cp scripts/dictation-hotkey/dictation-hotkey resources/bin/dictation-hotkey chmod +x resources/bin/dictation-hotkey + # Compile the native actions helper (EventKit) and stage it into resources/bin so + # extraResources bundles it at Contents/Resources/bin — the path runNativeAction + # resolves. Self-contained: no committed binary, built fresh against the pinned + # target. If it ever fails to ship, the calendar tools report "not available" and + # the rest of the app is unaffected, so it can't break a release. + - name: Build native actions helper (computer use, semantic rail) + run: | + bash scripts/build-actions-helper.sh + mkdir -p resources/bin + cp scripts/actions-helper/actions-helper resources/bin/actions-helper + chmod +x resources/bin/actions-helper # Stage the Parakeet STT runtime (sherpa-onnx CLI + ONNX model) into # resources/bin/parakeet. Additive: with SHERPA_ONNX_URL / PARAKEET_MODEL_URL # unset this is a no-op and transcription stays on whisper, so it can't break a @@ -228,6 +239,10 @@ jobs: # drifted lock stops the release instead of resolving a graph nobody committed. npm --prefix ../shared ci npm --prefix ../shared/packages/sync run build + # @offgrid/use (the durable action engine) is consumed the same way as sync + # and needs the same explicit build - file-dep prepare alone does not emit + # its dist/ types before the typecheck runs. + npm --prefix ../shared/packages/use run build - name: Install dependencies run: npm ci # Stamp the resolved version into package.json so electron-builder picks the @@ -468,6 +483,10 @@ jobs: # drifted lock stops the release instead of resolving a graph nobody committed. npm --prefix ../shared ci npm --prefix ../shared/packages/sync run build + # @offgrid/use (the durable action engine) is consumed the same way as sync + # and needs the same explicit build - file-dep prepare alone does not emit + # its dist/ types before the typecheck runs. + npm --prefix ../shared/packages/use run build - name: Install dependencies run: npm ci - name: Fetch Windows native binaries (llama/whisper/sd/ffmpeg) diff --git a/.github/workflows/windows-build.yml b/.github/workflows/windows-build.yml index b0f0bc4b..077c2687 100644 --- a/.github/workflows/windows-build.yml +++ b/.github/workflows/windows-build.yml @@ -75,6 +75,43 @@ jobs: with: python-version: '3.12' + # `@offgrid/sync` and `@offgrid/use` are file: dependencies on the shared + # monorepo, so it must sit BESIDE this checkout before `npm ci` runs - + # mirrors release.yml's build-win. The branch ref decides the shared ref: + # a build from an integration branch takes the shared branch of the same + # name when one exists, else shared main. + - name: Checkout shared at the matching ref + id: shared_branch + continue-on-error: true + uses: actions/checkout@v4 + with: + repository: off-grid-ai/shared + token: ${{ secrets.CI_CROSS_REPO_TOKEN }} + ref: ${{ github.ref_name }} + path: _shared + persist-credentials: false + - name: Fall back to shared main + if: ${{ steps.shared_branch.outcome != 'success' }} + continue-on-error: true + uses: actions/checkout@v4 + with: + repository: off-grid-ai/shared + token: ${{ secrets.CI_CROSS_REPO_TOKEN }} + path: _shared + persist-credentials: false + - name: Put shared beside this checkout + shell: bash + run: | + if [ ! -d _shared ]; then + echo "::error::off-grid-ai/shared was not checked out - @offgrid/sync and @offgrid/use cannot resolve. Check CI_CROSS_REPO_TOKEN." + exit 1 + fi + rm -rf ../shared + mv _shared ../shared + npm --prefix ../shared ci + npm --prefix ../shared/packages/sync run build + npm --prefix ../shared/packages/use run build + - name: Install dependencies run: npm ci diff --git a/.gitignore b/.gitignore index b01caaae..ac2277f5 100644 --- a/.gitignore +++ b/.gitignore @@ -52,6 +52,7 @@ test-results/ # Marketing material (dev.to drafts, emails, assets) — publishing, not app code /marketing/ scripts/dictation-hotkey/dictation-hotkey +scripts/actions-helper/actions-helper # Local demo profile — synthetic-data run target for `npm run demo` (never real userData) .demo-profile/ diff --git a/docs/ASSISTANT_ARCHITECTURE.md b/docs/ASSISTANT_ARCHITECTURE.md new file mode 100644 index 00000000..82eba1e6 --- /dev/null +++ b/docs/ASSISTANT_ARCHITECTURE.md @@ -0,0 +1,320 @@ +# The assistant - system architecture (the act pipeline) + +**Status:** high-level design, August 13, 2026, from the architecture discussion. For team review. +Companion to `COMPUTER_USE.md` (the product model), `COMPUTER_USE_PLAN.md` (the build doc + schedule), and `PORTING_MAP.md` (port-vs-bespoke). This is the *design reference* for the system that executes actions - how it stays reliable on a weak local model and identical across desktop and mobile. The build order and timeline live in `COMPUTER_USE_PLAN.md`; follow that to build. + +--- + +## 1. The problem this design solves + +Two hard constraints shape everything: + +1. **Local models are unreliable at tool-calling.** A bundled small model malforms calls, hallucinates arguments, or answers in prose instead of calling the tool. We cannot couple "decide" and "do" in a single model turn, or a bad turn means a lost or half-done action. +2. **The core must be identical on desktop and mobile.** We do not want two implementations of the thing that decides and guarantees actions. + +The design below answers both: a durable action pipeline where the model is the least-trusted component, wrapped by deterministic machinery that guarantees execution. + +## 2. The core reframe: the model proposes, the pipeline guarantees + +Most agent systems fail because the model both decides and executes in one turn. We invert it: + +**The model only ever proposes a structured Action. A durable, deterministic pipeline guarantees it happens - exactly once, gated, verified.** + +The model does the smallest, most-constrained job (produce a valid Action), and everything downstream is deterministic. A bad proposal is caught at a validation boundary and discarded (fail closed); a good proposal is executed with exactly-once guarantees and effect-verification. This is the tenet the whole system rests on: + +> **Reliability lives in the system, not the model.** + +A capable model makes the *proposals* better (fewer rejections, better resolution). It never changes whether an approved action actually executes. That is what lets us swap in a smaller or fine-tuned model later with no change to the guarantee. + +## 3. The Action: a durable record and a state machine + +Everything - a proactive come-up, a tool the chat model called, a routine step, a scheduled trigger - normalizes into one durable **Action** record in local SQLite. That store is the queue. + +An Action carries: `id`, `type` (message / email / calendar / open / file-share / web-task / ...), `source` (reasoning / chat / routine / schedule), `intent` (the natural-language ask), `args` (resolved slots), `payloadHash` (the immutable contract of exactly what will run), `risk` (read / navigate / mutate / irreversible), `rail`, `idempotencyKey`, `attempts`, `verification`, `state`, `triggerAt`, and audit references. + +It moves through a persisted state machine: + +```mermaid +stateDiagram-v2 + [*] --> proposed + proposed --> rejected: invalid (grammar / schema) + proposed --> scheduled: has a trigger + proposed --> resolving: valid, run now + scheduled --> resolving: trigger fires + resolving --> awaiting_approval: mutate / irreversible + resolving --> ready: read / low-risk + awaiting_approval --> ready: approved + awaiting_approval --> rejected: rejected + ready --> executing + executing --> verifying + verifying --> done: effect confirmed + verifying --> executing: failed, retry once + verifying --> needs_help: still failed +``` + +Because the record is persisted, not a transient turn: a crash resumes it, a scheduled action waits durably, a retry does not double-send (idempotency), and an action is not `done` until its effect is verified. This durability is also exactly why the pattern fits mobile - a queue drained by a background worker survives the OS killing the app, which mobile does aggressively. + +## 4. The reliability stack (how it survives a weak model) + +Layered, weakest-model-work first: + +1. **Constrain the output.** Grammar-constrained decoding (GBNF) so the model can only emit a valid Action on valid arguments. For GUI steps, generate the grammar per step so it can only pick elements that exist right now. +2. **Validate at the boundary, fail closed.** A malformed or off-schema proposal never becomes an Action. Keep the action schema small and closed (fewer types = far better local accuracy); rank and prune available tools to the token budget. +3. **Decouple decision from execution.** The durable queue means a bad turn is a no-op, not a lost or half-done action. +4. **Bind the executed payload to the approved one.** The `payloadHash` the gate showed is exactly what runs - no re-resolution between confirm and act. +5. **Prefer determinism over the model.** Route to semantic rails and recorded traces first; the model does the least, most-constrained work, least often. Vision/GUI is the last resort. +6. **Verify, then retry once, then ask.** Observe the effect. If it did not happen, retry once; if it still did not, mark `needs_help` and surface it rather than looping. +7. **A cross-rail escalation is a re-fire, under the same policy.** Falling back from one rail to another (semantic timed out -> try the browser) is another execution attempt on the same Action, so it is governed by the same retry rules: only a retryable action (reversible, reliably verifiable, retry budget left) may escalate, and only after verification confirms the effect did NOT happen. A timed-out irreversible action goes to `needs_help`, never to another rail - that is how a double-send is made impossible even across rails. The durable Action record is the effect journal: every attempt records the rail it ran on. + +## 5. Focused, not general: a registry of typed action handlers + +Per the lead's steer, the assistant is not a general "call any tool" agent - it is a **curated set of first-class action types**, each with its own schema, grammar, resolver, rail, and verification. Adding a capability = adding a handler, not retraining anything. + +The v1 scope (things you do on your own machine), grouped by type and honest about reliability tier: + +| Action type | Examples | Rail | Reliability in v1 | +| --- | --- | --- | --- | +| Message | send a text | semantic (AppleScript / iMessage) | high | +| Email | send / compose | semantic (Mail, or Gmail connector) | high | +| Calendar and reminders | create event / reminder | semantic (EventKit) | high | +| Open / launch | open tabs, a URL, a YouTube video, an app | semantic (deep link / open) | high | +| Look up | contacts, "what's on my calendar" | semantic (read, inline) | high | +| File share | share a file over WhatsApp | GUI vision (Catalyst, dead AX tree) | best-effort, supervised | +| Web task | flight check-in, book a hotel, order | agent browser + takeover | best-effort, supervised | +| Proactive notice | "flight tonight, not checked in", "you promised the deck" | reasoning engine -> feeds the above | new, memory-driven | + +The pipeline is identical across all of them; only the rail and the reliability differ. Two tiers to set expectations honestly: **semantic actions (text, email, reminders, open) are solid; GUI and web tasks (WhatsApp file share, check-in, booking) are supervised and improving.** Same product, honestly tiered. + +## 6. One core, two platforms + +The pipeline is the `@offgrid/use` engine in `shared` (consumed as `file:../shared/packages/use`). The reliable parts are pure logic, so they are shared; only the platform-specific edges are adapters. + +**Naming (canonical).** Two layers: **the assistant** (the brain - reasoning, resolve, the queue, the router, the gate, verify) and **the rails** (the actuation layer - the executors that actually perform actions, behind the `DeviceController` interface). Each concrete path is a rail: the **semantic rail**, the **browser rail**, the **accessibility rail**, and the **vision rail**. "Computer use" means the vision rail specifically, not the whole layer - most actions never touch it. + +**The shape, at a glance.** This is a component diagram in the **ports-and-adapters (hexagonal)** pattern: the assistant is the core, the `DeviceController` is the port, and the rails are the swappable adapters implemented per platform. + +```mermaid +flowchart TB + RE[Reasoning engine] --> IN + CH[Chat / routine] --> IN + SC[Scheduler / trigger] --> IN + MEM[(Memory:
Replay, entities, RAG)] -.-> RE + MEM -.-> RS + + subgraph BRAIN["THE ASSISTANT · brain · @offgrid/use (shared, platform-free)"] + direction TB + IN[Intake + validate
grammar · schema · fail closed] + Q[(Durable queue · state machine)] + RS[Resolver · slots from memory + confidence] + GT{Gate · evidence + confidence} + RO[Router · cheapest reliable rail] + VF[Verify · retry once · else ask] + IN --> Q --> RS --> GT --> RO + VF -.re-queue on fail.-> Q + end + + RO ==>|"execute(action)"| DC{{DeviceController · the port}} + DC -.result.-> VF + + subgraph RAILS["THE RAILS · actuation · platform adapter"] + direction LR + R1[Semantic rail] + R2[Browser rail] + R3[Accessibility rail] + R4[Vision rail
= computer use] + end + DC --> R1 + DC --> R2 + DC --> R3 + DC --> R4 + + RAILS -.implemented per platform.-> PLAT["macOS · Windows · Android · iOS"] +``` + +**Shared core (platform-free):** the Action model + durable queue + state machine; the reasoning engine (commitment / gap detection); the resolver (slot-filling over memory, with confidence); the router (cheapest reliable rail); verification + retry / idempotency policy; the action-handler registry; the gate seam (a callback the host implements). + +**Per-platform adapters (behind interfaces the core calls):** +- **The rails (behind the `DeviceController` interface)** - how to actually run a thing. **Desktop v1 is macOS + Windows, in scope from day 1.** macOS: the Swift helper (EventKit / AppleScript), the agent browser, AX + CGEvent, vision. Windows: **local Outlook automation (COM / PowerShell) first** where Outlook exists - like the mac rail, a local write that syncs when the network returns - with Microsoft Graph as the fallback for setups without a local Outlook, and online-only actions labeled honestly; the shell for open / launch; the agent browser (shared, Electron); UI Automation + SendInput; vision. Android: intents + content providers + an accessibility-service portal. iOS: App Intents / Shortcuts only (no GUI or vision rail - the platform forbids reading or driving other apps). +- **Accessibility is primarily the eyes, not a fourth pair of hands.** The AX / UIA tree is the observation and verification layer serving every rail: anchors for recorded traces, read-back for verification, drift checks. Actuation through it stays deliberately capped - macOS keeps `AXPress` / set-value only (the Swift helper already has them; set-value beats replaying keystrokes), and Windows acts through SendInput at UIA-located targets rather than growing a second actuation surface. One maintenance surface less, per platform. +- **Offline scope, stated precisely:** the brain - detection, resolution, gating, the queue, verification logic - runs with zero network on every platform. An action whose effect lives on an external service (send an email, book a flight) needs that service reachable at execution time on any OS; the design preference is local-app rails whose writes land locally and sync later, which is exactly why local Outlook beats Graph as the Windows default. +- **MemoryStore** - read observations / entities / RAG. +- **Scheduler** - fire time and event triggers. +- **Approval and feed UI** - render the gate and the come-up feed (desktop renderer; mobile React Native). +- **Model client** - both call the local model through the OpenAI-compatible gateway. + +So "the core stays the same" is concrete: the queue, resolver, router, reasoning, and verification are one codebase; only the executor, store, scheduler, and UI are swapped per platform. Mobile is an adapter project on the same engine, not a rewrite. + +--- + +## 7. Decisions locked (present these as answered) + +1. **The model proposes, the durable queue guarantees.** Decision-and-execution are separated. The model produces a validated Action; the pipeline executes it. This is what makes the system reliable on a weak model. +2. **Reliability lives in the system, not the model.** The execution guarantee comes from the pipeline (constrain, validate, queue, deterministic rails, verify), never from the model being good. +3. **Model choice: capable now, model-agnostic pipeline, fine-tuning deferred.** We start with a good, capable model to prove the experience feels right. The pipeline is built to hold with a smaller model, so the bundled model (or a future LoRA fine-tuned on our action schema) slots in with zero change to the guarantee. Fine-tuning is an optional later reliability boost, not a v1 dependency. +4. **The queue lives in `shared` (`@offgrid/use`).** The queue engine and state machine are platform-free core; the storage and UI are platform adapters. This keeps the execution guarantee identical on desktop and mobile. +5. **Mutations go through the queue and gate; reads run inline.** Anything that changes the world (send, create, delete) - even when asked in chat - flows through the durable pipeline. Pure reads ("what is on my calendar") can run inline for latency, since there is nothing to guarantee. To the user this is invisible; chat can still act, it is just durable and gated underneath. +6. **Retry once, then ask.** On a verified failure, retry a single time; if it still fails, stop and surface `needs_help` rather than looping. +7. **The gate shows resolved values, evidence, and confidence, bound to the approved payload.** The approval card shows what was inferred and why ("Send Q3.pptx to Ali because ..."), and the exact payload approved is the exact payload that runs. +8. **Cheapest reliable rail first, vision last.** The router prefers a deterministic surface (deep link / API / AppleScript) over the agent browser over the accessibility tree over the vision-grounding model. +9. **Scope: a curated set of typed action handlers** (Section 5), spanning two honest reliability tiers - semantic actions are solid, GUI / web tasks are supervised. + +## 8. Open questions (for the team) - explained + +Each is a real decision with a tradeoff. Where we have a lean, it is stated so the team reacts to a proposal rather than a blank. + +### 8.1 Exactly-once per rail +**What it is.** The guarantee that an action runs one time and only one time, even across a retry or a crash. Example: the executor sends an iMessage, then the app crashes before recording success; on restart it must not send a second copy. +**Why it matters.** Double-sending a message, or creating two calendar events, is a visible, trust-damaging failure - worse than a clean failure. +**Options.** (a) *Idempotency key* - tell the target "this is operation X, ignore a duplicate" (works only if the target supports it). (b) *Check-before-act* - before creating, ask "does this already exist?" (c) *Verify-after* - after the attempt, look for the effect and only retry if it is missing. Feasibility is per-rail: calendar / reminders / mail are verifiable and roughly idempotent; iMessage / WhatsApp / a website form are fuzzy (no key, and "did it send?" is hard to answer cleanly). +**The decision.** Do we require every action handler to declare a verification or existence-check capability? And for the fuzzy rails, is the policy single-attempt-behind-the-gate, or verify-then-accept-a-small-residual-risk? +**Our lean.** Handlers declare how they verify; reversible actions retry-once with verify; irreversible fuzzy actions (an outbound send) are single-attempt behind the gate, so a wrong verify can never double-fire. Escalating to another rail is a re-fire under the same rule (Section 4, item 7) - a non-retryable action never escalates. + +### 8.2 Scheduling and triggers +**What it is.** How a routine fires at 09:00, or an event trigger fires ("when I open Slack", "20 minutes before a meeting"). +**Why it matters.** Proactive delivery and routines depend on triggers, and they must work when the app is backgrounded or killed - especially on mobile, where the OS controls wakeups. +**Options.** (a) *Core-owned trigger model* - the shared core holds the trigger definitions and a durable schedule table, and a thin platform adapter wakes the worker (launchd / a timer on Mac, WorkManager / BackgroundTasks on mobile). (b) *Platform-native scheduling wrapped* - each OS's scheduler owns the timing, the core just registers callbacks. +**The decision.** How much scheduling logic lives in the core vs the OS, and how we survive the app being closed. +**Our lean.** Core owns the trigger model and the durable schedule; a thin per-platform adapter is responsible only for waking the worker at the right time. + +### 8.3 Trust graduation (Suggest to Auto) +**What it is.** When an action or routine moves from Suggest (ask before each run) to Auto (runs unattended). +**Why it matters.** This is the whole "proactive but safe" arc. Too eager feels invasive or dangerous; too timid and it never saves time. +**Options.** (a) *Per action type* - reads auto, sends always ask. (b) *User-set per routine* - a manual Suggest/Auto toggle. (c) *Confidence threshold* - auto when confidence is high and the action is reversible. (d) *Learned* - auto after N successful approvals of the same shape. +**The decision.** What is the default, who controls the dial, and do irreversible actions ever run Auto. +**Our lean.** Default Suggest; the user promotes a routine to Auto; irreversible actions always gate even inside an Auto routine; reversible high-confidence actions may auto after a few confirmations. + +### 8.4 Mobile v1 target +**What it is.** What actually ships on mobile first, given the same core but very different rails. +**Why it matters.** The rail capabilities differ enormously by platform, and this sets expectations. Android can host the full stack (an accessibility-service portal plus intents and content providers). iOS is intents-only - Apple forbids an app from reading or driving other apps, so there is no GUI or vision rail there. Also, the mobile app does not consume the shared monorepo yet, which is a prerequisite regardless. +**The decision.** Is mobile v1 Android-first (full experience), iOS-first (intents-only, limited), or desktop-only for v1 with mobile as a fast-follow - and on what timeline. +**Our lean.** Desktop v1; mobile as an adapter project afterward, Android-first for the full experience, iOS shipped as intents-only with honest scope. + +### 8.5 Open-core placement +**What it is.** Which parts of the pipeline are open core (AGPL, in `shared` / the public repo) vs pro (in `desktop-pro`). +**Why it matters.** Open-core is a hard rule - pro business logic must not live in core. The reasoning engine, the resolver policy, the approval-queue UI, and routines are the "act pillar" and follow the existing pro spine; the rail primitives and the queue engine are closer to infrastructure. +**The decision.** Draw the line: what is the inert core shell vs the pro business logic. +**Our lean.** The queue engine, the action-handler interfaces, and the rail primitives live in `shared` / core (infrastructure); the reasoning engine, the resolver's policy, the approval and feed UI, and routines live in `desktop-pro`. + +### 8.6 Verification depth per rail +**What it is.** How thoroughly we confirm an action's effect actually happened before marking it `done`. +**Why it matters.** Verification is what makes retry-once safe and catches silent failures and false confirmations (the field's number-one trust failure is an agent saying "done" when the backend failed). +**Options.** (a) *None* - trust the executor's return. (b) *Light* - parse the return / status. (c) *Full re-observe* - query the world (is the event in the calendar, is the mail in Sent, re-read the AX tree or screenshot). Cost vs safety, and it differs per rail. +**The decision.** The minimum verification bar per rail, and whether irreversible or GUI actions require full effect-verification. +**Our lean.** At least "executor reported success and the effect is observable" for every mutation; full re-observe for irreversible actions and for the GUI / vision rail, where drift is most likely. + +--- + +## 9. How to present this + +The narrative for the team: the vision (the demo) is validated; the scope is a curated set of action types across two honest reliability tiers; the system is a durable action pipeline where the model only proposes and the pipeline guarantees, so it survives a weak local model and stays identical on desktop and mobile; the decisions in Section 7 are locked; and Section 8 is the six open questions we want the team to weigh in on. The natural next step after alignment is the detailed `@offgrid/use` spec - the Action schema, the handler interfaces, and the reliability policy in code form. + +--- + +## 10. System architecture diagrams (C4, swimlane, user flows) + +The TRD / PRD deliverables, in the standard house style. The component diagram in Section 6 is the C4 **component** level (Level 3); the two views below add the **context** (Level 1) and **container** (Level 2) levels above it, then a runtime swimlane and the product user flows. + +### 10.1 System context (C4 - Level 1) + +Who uses the system and what it touches. The assistant is on-device; the only external things are the user and the apps and services it acts on. + +```mermaid +C4Context + title System Context - Off Grid AI assistant + Person(user, "User", "Knowledge worker, on their Mac or phone") + System(oga, "Off Grid AI", "Private on-device assistant that notices what you need and acts, with approval") + System_Ext(apps, "The user's apps and services", "Calendar, Mail, Messages, WhatsApp, the browser, connectors") + Rel(user, oga, "Asks in chat, approves actions") + Rel(oga, user, "Surfaces come-ups, asks to confirm") + Rel(oga, apps, "Acts on the user's behalf, with approval") +``` + +### 10.2 Containers (C4 - Level 2) + +The parts inside Off Grid AI and how they talk. The assistant engine is the brain; the rails are the hands; everything runs on-device. + +```mermaid +C4Container + title Container view - Off Grid AI assistant (all on-device) + Person(user, "User", "") + System_Boundary(oga, "Off Grid AI (on-device)") { + Container(ui, "Approval and feed UI", "React / React Native", "Day feed, approval card, routines") + Container(assistant, "Assistant engine", "@offgrid/use, shared TypeScript", "Reasoning, resolve, durable queue, router, gate, verify") + Container(rails, "The rails", "DeviceController adapters, native per platform", "Semantic, browser, accessibility, vision") + ContainerDb(memory, "Memory", "SQLite plus LanceDB", "Replay observations, entities, RAG") + Container(model, "Local model gateway", "llama.cpp, OpenAI-compatible", "On-device LLM, grammar-constrained") + } + System_Ext(apps, "The user's apps and services", "Calendar, Mail, Messages, WhatsApp, web, connectors") + Rel(user, ui, "Sees come-ups, approves") + Rel(ui, assistant, "Proposes and approves actions") + Rel(assistant, model, "Proposes a validated action") + Rel(assistant, memory, "Detects patterns, resolves slots") + Rel(assistant, rails, "execute(action)") + Rel(rails, apps, "Deep links, EventKit, AppleScript, GUI") +``` + +### 10.3 Sequence / swimlane + +Swimlane by actor: it makes clear who is responsible at each step, and which part of the system assists the user. Example flow: the user acts on a proactive come-up ("send the deck I promised Ali"). The lanes are the actors: User, Assistant, Memory, Gate, Rails, and the target app. + +```mermaid +sequenceDiagram + actor U as User + participant A as Assistant (brain) + participant M as Memory + participant G as Gate / Approval + participant R as Rails (DeviceController) + participant T as Target app (Mail) + + Note over A: Reasoning engine notices a commitment + A->>U: Come-up "you promised Ali the deck" + U->>A: "Send it" + A->>A: Validate and enqueue a durable Action + A->>M: Resolve "the deck" and "Ali" + M-->>A: Q3-strategy.pptx, Ali Chherawalla (with confidence) + A->>G: Propose (mutate) with the evidence + G->>U: Approval card - resolved values plus evidence + U->>G: Approve and send + G-->>A: Approved, payload locked + A->>R: execute(action) on the cheapest reliable rail + R->>T: Send via Mail (semantic rail) + T-->>R: Sent + R-->>A: Result + A->>A: Verify the effect (retry once if needed) + A-->>U: "Sent to Ali" (real confirmation, not a guess) +``` + +For a GUI action (say a WhatsApp file share) the same lanes hold; only the rail changes to vision, and the target app is driven step by step with a pause at the send. + +### 10.4 User flows + +The paths a user can take through the product: the two entry points (the proactive Day feed, or asking in Chat) through the gate to a verified result, plus the two ways a routine is born. + +```mermaid +flowchart TD + S([Open Off Grid AI]) --> DAY[Day - the Needs you feed] + ASK([Ask in Chat]) --> REV[Review the action] + + DAY -->|reasoned come-up| REV + DAY -->|routine proposal| TR[Turn into routine] + DAY -->|record a routine| DEMO[Demonstrate it once] + + REV --> CARD[Approval card:
resolved values + evidence + confidence] + CARD -->|low confidence| PICK[Pick the right one] + PICK --> CARD + CARD -->|edit| CARD + CARD -->|dismiss| DAY + CARD -->|approve| EXE[Assistant runs it on a rail] + + EXE --> VER{Verified?} + VER -->|yes| DONE([Done - toast confirms]) + VER -->|no, retry once| EXE + VER -->|still no| HELP([Needs help - asks you]) + + TR --> CONF[Confirm the learned steps
and set a trigger] + DEMO --> CONF + CONF --> SAVE([Saved - starts as Suggest]) + SAVE -.runs on its trigger.-> REV +``` + +Two entry points - the proactive Day feed and Chat. Both land on the approval card, which shows the resolved values with their evidence and confidence; low confidence branches to a quick "which one did you mean" pick. Approve runs it on a rail, then verify decides done, retry-once, or ask you. A routine is born two ways - the assistant proposes a detected pattern, or you record one by demonstrating it - both converge on confirming the learned steps and setting a trigger, and a saved routine starts as Suggest until you trust it. diff --git a/docs/COMPETITIVE_RESEARCH.md b/docs/COMPETITIVE_RESEARCH.md new file mode 100644 index 00000000..8670b5f1 --- /dev/null +++ b/docs/COMPETITIVE_RESEARCH.md @@ -0,0 +1,115 @@ +# Competitive and prior-art research - the proactive assistant + +Researched August 2026. Every facet of what we are building has prior art; none of the incumbents ship the whole loop, and two big pieces are open whitespace. This is reference material for product and design. Sources are linked inline. + +## Three strategic findings (read first) + +1. **Local-first is open whitespace, and the market just proved why it matters.** The two flagship local screen/audio-memory products both got acquired by Meta in Dec 2025 and effectively ended as local products (Rewind capture disabled Dec 19 2025; Limitless pendant pulled). Dot (New Computer), a beloved memory-driven companion, shut down Oct 2025 and users "grieved" lost months of context. The lesson every Rewind-alternative now leads with: **local means your memory survives the vendor and never leaves the device.** Screenpipe (local SQLite + OCR + on-device model, MIT) is the architecture to benchmark against. This is our moat, validated the hard way. + - https://the-gadgeteer.com/2026/05/05/best-ai-wearables-2026/ · https://techcrunch.com/2025/09/05/personalized-ai-companion-app-dot-is-shutting-down · https://github.com/screenpipe/screenpipe + +2. **"Resolve the reference, show the evidence, confirm before acting" is essentially unshipped.** Every assistant resolves a vague reference the same way (hybrid retrieval -> rerank -> LLM answer) and shows provenance as *post-hoc citations*. None show ranked candidates with the evidence for each, surface a confidence, and ask you to confirm the pick *before* acting. Shortwave computes per-feature confidence and discards it. Gmail's forgotten-attachment detector is the only shipping confirm-before-send gate, and it cannot even name the file. **The thing our approval card does - "Send Q3.pptx to Ali because you called it 'the deck' in Tuesday's call and it is the only deck shared with Ali" - is the exact whitespace.** + - https://arxiv.org/abs/2503.15739 (ECLAIR) · https://arxiv.org/abs/2206.07836 (PEL/CREL) · https://patents.google.com/patent/US10812427 + +3. **The GUI-automation reliability ceiling is real, and everyone hit it in 2026.** Google killed Project Mariner (May 2026) - screenshot-per-step vision was too slow, costly, and error-prone at scale. OpenAI quietly killed ChatGPT travel checkout (~Mar 2026) - "travel was too hard." Perplexity Comet's agentic mode is "wildly inconsistent" ("faster to do it yourself"). This validates our whole architecture: **route to the cheapest reliable rail, prefer demonstrated traces over novel automation, and gate everything.** Do not bet the product on pixel-level autonomy. + - https://en.wikipedia.org/wiki/Project_Mariner · https://www.tourismtribe.com/chatgpt-instant-checkout-travel-operators/ · https://www.eesel.ai/blog/perplexity-comet-reviews + +--- + +## 1. Proactive surfacing (the "come-up") + +**Who does it:** Rewind/Limitless and Microsoft Recall (recall, not proactive push), Screenpipe (local infra), Apple Siri Suggestions / Call Context, Google **Magic Cue** + **Daily Hub** (Pixel), Microsoft Copilot ("Your Day at a Glance"), **ChatGPT Pulse** (the reference morning-briefing), Martin / Ohai (act + reach you in your channel). + +**The recurring patterns (what to copy):** +- **The morning card feed** - a once-daily, scannable set that owns "the first five minutes of your day." Pulse, Daily Hub, Copilot, OpenClaw's briefing all converge here. Value is *density and relevance per card, not volume*. +- **Inline point-of-need chip (the best pattern)** - Magic Cue surfaces the thing *where you are already acting* (a chip in the message box, a confirmation code on the call screen), single tap to use, no feed to visit. Preferred over a feed for actionable items. +- **Notification -> answer-ready, never a dead alert** - Copilot's push opens straight into the pre-run answer and next action. Never surface "you have items waiting" with a blank prompt behind it. +- **Feedback + forward-preview** - Pulse ends each briefing previewing tomorrow's topics with a "curate" control, so the feed feels steerable. +- **Recall is a separate surface** - the scrubbable DVR timeline (Recall, Screenpipe) is for "find what I saw," kept distinct from the proactive push. + +**The hard constraint - the notification budget.** Independent research and Pulse's own complaints converge: **~3-5 unsolicited notifications/day total is the ceiling**; exceeding it means users mute by Friday. "Notifications sent is a vanity metric; dismissals look like engagement but predict churn." An interruption costs ~23 minutes of recovery. Prescription: a hard daily cap the surfacing engine must respect, value-vs-attention scoring per candidate, learned per-user dismiss thresholds, and displacement logic (a new item must out-rank the queued one to fire). Treat each notification as a withdrawal from a finite account. + - https://tianpan.co/blog/2026-05-13-background-agents-notification-budget-attention-economy · https://www.platformer.news/chatgpt-pulse-proactive-ai/ + +**Avoid:** a high-frequency engagement-optimized feed (Pulse's worst reviews: fatigue, "creepy," "my calendar does this free"); the come-up that only restates what the calendar/email already shows (the bar is *net-new synthesis*); over-automation without control (Motion's complaint); always-on capture without visible opt-in + encryption + per-app exclusions (Recall's 2024 near-death). Google's **Magic Cue** is the single best pattern to study. + - https://store.google.com/us/magazine/magic-cue · https://9to5google.com/2025/08/20/pixel-10-magic-cue-launch/ + +## 2. Context resolution ("which deck did they mean") + +**Who does it, and how (all the same shape):** ChatGPT memory + connectors (RAG over an index, live source sidebar), Gemini Workspace ("Sources" list, admits it "can make up a source"), **Glean** (the most sophisticated - a per-company entity knowledge graph that collapses variant names to one canonical identity, auditable traversal path), Microsoft 365 Copilot ("/" typeahead picker - the closest shipping "pick which one you meant", but only on explicit "/", not vague prose), Notion Q&A, Dropbox Dash, Slack AI (auto-extracts filters from a NL reference: author=Sarah, type=slides, last week), **Shortwave** (the best-documented pipeline: coref query-reformulation -> parallel feature extraction *with confidence* -> hybrid retrieval -> two-stage cross-encoder rerank). + +**The whitespace (finding #2 above):** every product shows provenance as *post-hoc citation*, never a pre-action evidence panel with candidates + confidence + a confirm/correct control. Confidence is computed and thrown away. The research blueprint exists (ECLAIR interactive disambiguation; PEL/CREL personal-entity linking = coref to trace "the deck" back to its first mention + bind to the file entity - a two-step our on-device entity graph is well-suited to) but is unshipped in consumer products. **Caveat:** entity-reference ambiguity is only ~23% of real ambiguity - the rest is which *version*, which *date*, a missing constraint - so a resolver must handle more than the noun. + +**The universal failure story:** confident wrong-source grounding. The Tow Center found >60% citation errors across AI search tools (ChatGPT ~67%); Google admits Gemini cites unused docs; Notion cannot reconcile duplicate/stale pages. The trust gap is precisely that these systems act on an unconfirmed pick and back-fill a citation users have learned not to trust. **Our answer:** show the evidence and confidence *before* acting, gate on it. + - https://www.glean.com/perspectives/what-role-does-a-knowledge-graph-play-inside-modern-enterprise-ai-software · https://www.zenml.io/llmops-database/building-a-production-grade-email-ai-assistant-using-rag-and-multi-stage-retrieval · https://support.microsoft.com/en-us/microsoft-365-copilot/refer-to-specific-files-and-more-in-microsoft-365-copilot + +## 3. Commitment / reasoned detection + +**Email tools mostly do NOT do semantic "I promised X" detection - they detect the structural proxy "you sent mail, got no reply in N days":** Gmail/Gemini **Nudges** (the canonical *cautionary tale* - right idea, but on-by-default, breaks inbox order, induces guilt, fires on already-closed threads; the textbook example of resurfacing done annoyingly), Superhuman Auto Reminders (with the key anti-nag lever: scope to "external recipients only"), Spark, Boomerang, SaneBox (the quieter "no-replies folder" vs Gmail's loud inbox-bump - a useful design axis). **Mailbutler** does real semantic commitment extraction with urgency tiers; **Shortwave deliberately keeps task-creation manual** (human-confirm to avoid false-positive spam). + +**Meeting-notes tools are where real "who owes what" extraction happens** (LLM over the transcript, owner by speaker, deadline from prose): Otter (cross-meeting dashboard, links to the transcript moment, weekly digest), Fireflies (cue-phrase extraction, ~90% after 2 weeks of correction, but speaker attribution "hit-or-miss"), Fathom (strong attribution, but **ownership is understood then lost at handoff** to task tools), Granola (uses your sparse notes as anchors to cut hallucination), Zoom (best-practice format "Owner + verb + deliverable + date"). **Failure mode to design against:** hallucinated action items and invented commitments ("assigned stories they didn't agree to write") - so link every extracted commitment to its exact source utterance and keep a confirm step. + +**The durable formal model** (Microsoft Research, HP Labs): a commitment is a **commissive speech act with a debtor (who owes), a creditor (who is owed), and an optional deadline**, detected at the *sentence* level. That cleanly gives our two lists: "you owe" (user is debtor) and "waiting on" (user is creditor). Commitment vocabulary generalizes across domains (so a bundled local model is plausible) but models overfit, and precision tops out ~80-90% = **1 in 5-10 flags is wrong** - which is exactly why every shipping product hedges ("suggested"), batches into a digest, or requires a human confirm. + - https://www.microsoft.com/en-us/research/blog/email-overload-using-machine-learning-to-manage-messages-commitments/ · https://techcrunch.com/2018/06/15/gmail-proves-that-some-people-hate-smart-suggestions/ · https://www.careful.industries/blog/2025-11-nine-risks-caused-by-ai-notetakers + +**Anti-nag levers actually used:** granular independent opt-outs; scope narrowing ("external only"); batching over real-time; hedged framing ("suggested," not "your tasks"); human-confirm-before-commit; urgency tiers as a soft confidence gate; link every item to its source. The louder the surface, the more a false positive hurts. + +## 4. Routines / teach-by-demonstration + +**The two failed ends of the spectrum:** coordinate/pixel replay (Apple Automator **"Watch Me Do"** - it *observed* via the accessibility tree then *replayed* via absolute coordinates, "playback continues regardless" of drift; that one choice is the entire failure mode) and pure-vision replay (Mariner - "learn the plan not the pixels" was the right idea but cloud vision every step was too slow/costly/error-prone to ship). **Our AX-anchored trace + memory-filled slots + local model sits in the gap both missed.** + +**Best authoring patterns (Apple Shortcuts, Keyboard Maestro, BetterTouchTool):** +- **Magic Variables** (Shortcuts) - every action's output is automatically a droppable, icon-tagged token you click to reinterpret. Best data-flow UX in the field. +- **Ask Each Time** (Shortcuts) - the simplest run-time slot; prompt when the value is not known. Pair with memory-fill: *resolve the slot from memory if known, fall back to Ask Each Time.* +- **Named Triggers with passed variables** (BTT) - the routine as a function with named arguments, invocable by many triggers; **Conditional Activation Groups** = context predicates gating when it may fire. +- **Use Model as one action in the stack** (Shortcuts, iOS 26) - Apple's own "an LLM step inside a deterministic routine," not "the model runs everything." Mirror this for slot-filling. +- **The reliability spectrum shown to the author** (KM: AX/semantic > found-image > coordinates, with "not found -> empty string -> branch"). + +**The RPA recorders are the gold standard for element anchoring and self-healing** (UiPath, Power Automate Desktop, Automation Anywhere): +- **Descriptor = target + anchors, not a bare selector.** UiPath's Unified Target captures the element *plus* 1-3 stable neighbor elements, with type-aware anchor selection (input -> label to the left/above via aria-labelledby; checkbox -> right). For an AX trace, record the target AX node **plus its labeling neighbor(s)**. +- **A redundant stack of targeting methods that race, first-match-wins** - strict path, fuzzy/Levenshtein match, visual/CV fallback - never a single point of failure, never raw coordinates except last resort. Critical refinement (Selenium's lesson): make the fallbacks *different in kind* (semantic + text + structural + visual), so one redesign cannot kill all at once. +- **Self-healing fires at the failure boundary, not the happy path.** UiPath **Healing Agent** and PAD **self-healing** (GA/preview 2025-26) run only after the element times out, give the model the **screenshot of the missing element + parent-window title + full-screen image**, and regenerate a fresh selector preserving intent. PAD runs this with GPT-4.1-mini + Claude Sonnet 4.5 - **a local model doing the same visual-grounding + AX-tree reasoning is a direct fit for our on-device design.** Two modes (auto-fix vs propose-for-approval), and the healed descriptor is *persisted* so the routine self-improves. Cascade cheap heuristics (close overlays, adaptive waits, semantic relabel match) before the LLM. +- **Record-with-narration** (PAD "Record with Copilot") - the user demonstrates while narrating; video + audio + UI metadata -> a flow with conditions and loops. The closest analog to us; voice narration disambiguates intent and variable slots that pure action capture cannot infer. + +**The one-line macro-vs-smart test:** if changing a button's CSS class, moving it in the DOM, or swapping its tag breaks the routine, it is a macro. If it still finds the control a user would call "Submit" and can re-derive it from accessibility semantics, it is smart. + +**The research is the actual build blueprint for slot induction + self-healing (this is what phase 4 implements):** +- **Agent Workflow Memory** (AWM, ICML 2025, arXiv:2409.07429) - the canonical "trace -> parameterized routine" mechanism: an LM extracts reusable workflows from trajectories and **represents the non-fixed parts with descriptive variable names** (literal "dry cat food" -> `{product-name}`). Works online (induce a workflow after each success, add to memory immediately - self-improving, no training). Proves **an LLM can induce named, described slots from as little as one successful trace** - directly how our local model turns a recorded AX trace into a parameterized routine. WebArena 23.5% -> 35.5%. +- **Alloy** (arXiv:2510.10049) - single demo -> a task-level graph (nodes with conditionals/loops); an Identifier agent replaces literals with **semantic placeholders** carrying a documented meaning, a Filter agent fills them from the user's stated intent. Two-level review UX to copy: **structural** editing (nodes/edges) + **behavioral** (edit a node's prompt or **re-record just that one step**). Re-record-one-step is the killer repair affordance. +- **SUGILITE (CHI 2017) / APPINITE (2018)** - our exact primitive from 2017: capture via the **accessibility API**, generalize a **single demo into a parameterized script** by combining **verbal command + demonstrated procedure + UI hierarchy**; APPINITE targets elements by **semantic "data descriptions" (property queries), not coordinates** - the canonical answer to semantic-grounding-vs-pixels. PLOW (2007): NL identifies which demonstrated values are the parameters. +- **LUMOS** (arXiv:2606.30697) - the closest published articulation of *our* thesis: ground actions to **OS accessibility-tree elements (role, label, state, hierarchy) not pixels**, because "when applications update visual styling or layout, the accessibility tree typically remains stable, preserving action validity"; it names the **macOS Accessibility API** as the surface. Cite as the robustness rationale for AX anchoring. Contrast: frontier GUI models (UI-TARS-2) are pixel-and-coordinate grounded, drift-fragile, and expose **no editable parameterized routine artifact** - our inspectable AX routine is a different, more robust design point. +- **Segment into subtasks, never a flat event log** (arXiv:2606.20978) - hierarchy "separates what to do from how to do it," which is what makes a routine reusable and parameterizable. The flat event list is exactly the Automator mistake. +- **Verify each step's effect** ("Don't Act Blindly", ACL 2026; VeriSafe pre-action logic checks) - the expected effect at step t becomes the verification hypothesis at t+1; the dominant *silent* failure is that "agents don't recognize they've failed, leading to cascading errors," so re-snapshot after a consequential action and replan on `NO_CHANGE`. **Morae** (arXiv:2508.21456) - confirm only at *consequential or ambiguous* steps (a critical-vs-non-critical classifier + ambiguity-gated pause), not every step. A clean tiered permission model from the 2026 survey: **Silent (read) -> Logged (writes shown) -> Confirmed (shell/network) -> Blocked (credentials)**. And graduated trust exactly like Shortcuts: default a new routine to **Run After Confirmation**, let the user promote it to **Run Immediately**. +- **trycua/cua** (MIT) already does the **screenshot + AX-tree hybrid** and, in 2026, drives macOS apps **in the background without stealing the cursor** - directly relevant to a local-first assistant that must not hijack the session. + +The four properties that separate smart from brittle, converged across the literature: **semantic anchoring** (AX role/label + vision fallback, not coordinates), **described slots induced from the trace + intent** (sourced from memory / ask-each-time / a data loop), **verify-and-self-heal** (check each effect, regenerate the anchor or replan on drift, persist the fix), and **confirm at the right moments** (graduated trust, consequential-step gating). A literal macro has none; a smart routine has all four. None of the systems that produce an editable parameterized artifact (Alloy, SUGILITE, AWM, Mirage-1) is a local-first, on-device macOS product with AX anchoring + memory-sourced slots - that combination is ours. + - https://www.dssw.co.uk/blog/2014-11-10-automator-watch-me-do/ · https://support.apple.com/guide/shortcuts-mac/variable-types-apdd2b316022/mac · https://www.uipath.com/blog/product-and-updates/technical-tuesday-how-healing-agent-solves-ui-automation-challenges · https://learn.microsoft.com/en-us/power-automate/desktop-flows/self-healing · https://learn.microsoft.com/en-us/power-automate/desktop-flows/create-flow-using-ai-recorder · https://arxiv.org/abs/2409.07429 (AWM) · https://arxiv.org/html/2510.10049 (Alloy) · https://toby.li/publications/c4/ (SUGILITE) · https://arxiv.org/pdf/2606.30697 (LUMOS) · https://arxiv.org/html/2508.21456 (Morae) · https://github.com/trycua/cua + +## 5. Confirm-before-acting (the gate) + +**Two distinct designs exist:** +- **Inline pause + human takeover** (OpenAI Operator, Gemini Auto Browse, Comet) - the human re-enters the surface to type sensitive data or press the final button; the "edit" is "do it yourself." +- **Structured resolved-action card** (Manus Plan Mode, OpenAI Agents SDK / LangChain approval interrupts, mrmr, NN/g "Intent Preview") - shows resolved parameters (To / Subject / Body, amount, file, date) with Proceed / **Edit** / Cancel. **Manus is the standout**: "click into the plan and rewrite anything; when you Confirm, that plan becomes the source of truth." **Editing the resolved value is the differentiator** - most agents make you take over instead. Our card maps to this pattern. + +**Converged rules across everyone:** +- **Handoff for sensitive steps is universal** - payments, logins, CAPTCHAs -> human takeover; do not screenshot what the user types in takeover; use stored credentials only with permission; route payment through a tokenized intermediary; decline some categories (banking) outright. +- **Calibrate friction by reversibility, not uniformly** - auto-do the reversible long tail, confirm the sensitive, hard-gate the irreversible. "Confirm everything" measurably degrades into rubber-stamping (Anthropic's own data: full auto-approve drifts from ~20% of new-user sessions to >40% for experienced users). A user-set autonomy dial (Suggest / Confirm / Auto) is the emerging control. +- **Enforce the confirm deterministically, below the model.** Every real incident (Replit deleting a prod DB despite an approval rule; Comet's OTP exfiltration; Manus SilentBridge) proves a prompt-level "ask first" instruction is not an enforcement boundary. The card must gate the actual side-effecting call and match that exact action and its exact arguments, so injection or model drift cannot act on values the user never saw. + +**The #1 trust killer - false confirmations.** It appears in every task-doer: Ohai "tells you it completed tasks it hasn't," Comet "booked a hotel for the wrong dates," ChatGPT's "invented confirmations when the backend fails," Alexa+ got both Uber addresses wrong. **An agent must return a real backend confirmation record, never a model-generated "done."** Bake this in: our post-action toast must reflect the actual result of the executor call, never the model's claim. + - https://manus.im/blog/manus-plan-mode · https://getmrmr.com/blog/approval-fatigue · https://www.anthropic.com/research/measuring-agent-autonomy · https://brave.com/blog/comet-prompt-injection/ · https://www.nngroup.com/articles/impressions-chatgpt-agent/ + +--- + +## What this means for us + +Our design holds up remarkably well against the field; several of our choices are the exact documented best-practice (route to cheapest rail, demonstrated traces over novel automation, gate everything, memory as the moat). Concrete things to fold in: + +1. **Own the two whitespaces**: local-first (memory survives the vendor) and **context-resolution-with-evidence-and-confidence-shown-before-acting**. The approval card that shows *why* it resolved a value is the single most differentiated thing we can ship, and nobody has it. +2. **Make the notification budget a real module** (hard 3-5/day cap, value-vs-attention scoring, learned dismiss thresholds, displacement) - test it as a pure ranking unit. This is the difference between "proactive" and "muted by Friday." +3. **The gate must show a real confirmation, never a model "done."** Wire the post-action toast to the executor's actual result. This is the field's #1 trust failure and it is cheap to get right. +4. **Self-healing = AX-anchor (target + neighbor anchors) + racing heterogeneous fallbacks + LLM recovery at the failure boundary, with the healed descriptor persisted.** The local model does what PAD does with GPT-4.1-mini + Claude. Two modes: auto-fix vs propose-in-review. +5. **Anti-nag levers**: scope ("external only"), quiet folder vs loud bump, batching, hedged "suggested" framing, and link every commitment to its source utterance. Commitment precision is ~80-90%, so 1 in 5-10 is wrong - never auto-act on a detected commitment without the gate. +6. **Recorder**: consider narrate-while-demonstrating (PAD "Record with Copilot") to disambiguate slots; present the trace as an editable draft of semantic cards (not an event log); Magic-Variable-style tokens + Ask-Each-Time slots that resolve from memory. +7. **Detect completion and auto-retire the commitment - the single biggest anti-nag move, and one only we can make.** Every commitment tool flags "you said you'd send the deck" but none notice that you *sent* it, so they keep nagging. Because Replay watches the whole day on-device, we can see the fulfilling action (the email went out, the file was shared) and retire the item automatically. That is the difference between a tracker and a scold, and a generic cloud assistant cannot do it because it never saw you do the thing. +8. **Bind the confirmation to the exact payload that executes.** The Alexa+ lesson: a read-back is worthless if the value shown is not provably the value acted on (it read back the right address then used the wrong one). The values on the approval card must be the literal values the executor runs - no re-resolution between confirm and act; confirm returns an immutable action object. + +**For the demo brief specifically:** Screen 2 (the approval card) is where our differentiator lives - it must show the *evidence and confidence* for each resolved value, not just the resolved value. Add a low-confidence/disambiguation state (the ECLAIR "did you mean A or B" with evidence per candidate). That is the screen no competitor can show. diff --git a/docs/COMPUTER_USE.md b/docs/COMPUTER_USE.md new file mode 100644 index 00000000..77b5292b --- /dev/null +++ b/docs/COMPUTER_USE.md @@ -0,0 +1,202 @@ +# Off Grid AI - the proactive assistant (the act pillar) + +**Status:** product model agreed August 12, 2026. This supersedes the earlier "replicate the mobile-use stack" framing: that described one rail (GUI automation), not the product. The product is a proactive, context-grounded local assistant. Computer use is the last rail it reaches for, not the point. +**Standing constraints:** local models only, nothing leaves the device; all UI and copy follow `off-grid-ai/brand` (see 11). + +--- + +## 1. What we are building + +An assistant that **notices what you need and acts on it**, grounded in what OGAD already remembers about your day. Not a chatbot you command, and not a pixel-clicking robot - an assistant that: + +- **knows you** - Replay already captures your day (screen -> OCR -> observations -> entities). That memory is the raw material. +- **is proactive** - it surfaces the flight you have not checked in for, the presentation you promised, the routine you run every morning - before you ask. +- **is private** - all of it is on-device. That is the only reason a person would let something watch their whole day, and it is the moat. +- **routes to the cheapest reliable rail** - a deep link or a connector before a scripted action before driving a GUI before pixels. It acts through the app and content you actually mean, resolved from your context - "put on the show we were just talking about, in the app you use" - not a UI-clicking gamble. + +The differentiator is that combination, not raw GUI prowess. Local models will not beat frontier cloud agents at clicking arbitrary pixels this year, and chasing that is a trap. Knowing you, noticing, staying private, and routing well is the product. + +## 2. Two ways a task is born (the generators) + +Every task the assistant acts on comes from one of two generators. Both emit the same thing: **a proposed action with open slots** (the structure is known; the content is filled later, see 4). + +### 2.1 Routine proactivity - repetition + +The same flow, done again. Two authoring paths, one artifact (a routine = a trigger + an ordered, AX-anchored action trace): + +- **Auto-detected** - mined from the Replay observation log: "every weekday ~9am you open Mail then Slack and scan unread." Low fidelity (we know the sequence, not every exact target), so it is used to *propose*, then confirmed by a recording. +- **Demonstrated** - you hit record and do it once. High fidelity: the exact trace, directly replayable. See 5. + +Detection *proposes*; demonstration *records the reliable version*. "I noticed you do this every morning - show me once so I can do it exactly." They are one loop, not two features. + +### 2.2 Reasoned proactivity - situation + +No repetition at all. Given your situation, something *should* have happened and has not. The flight case: + +1. **Detect the commitment/event** - "flight tonight" from a conversation Replay captured, or a confirmation email. +2. **Know what it implies** - world knowledge the LLM already has: a flight means check-in, a boarding pass, a gate. Nobody programs "a flight entails check-in." +3. **Gap-check the actual state** - the agent goes and looks, read-only: is there a boarding pass in Gmail? any sign of check-in? +4. **Surface the gap** - "You fly tonight and haven't checked in. Want me to?" +5. **Act, then gate** - check in or open the check-in page; anything with identity or payment confirms first. + +Steps 1-4 - the *smart* part - are pure memory + LLM + read-only connectors. No vision, no risky automation. That is the most magical and the most reliable part; it lands early - R2 in the build plan, right after the chat action tool is released (R1). See `COMPUTER_USE_PLAN.md` for the order. + +**The routine engine gives reliable *doing*; the reasoning engine gives an assistant that *notices*.** Same spine underneath. + +## 3. One gated spine + +Both generators feed one path: + +```mermaid +flowchart TD + RG["routine generator\n(detected + demonstrated)"] --> P[proposed action + open slots] + XG["reasoning generator\n(commitment + world-knowledge + gap-check)"] --> P + P --> R["resolve slots\n(RAG over Replay + conversation + entities + files)"] + R --> C{gate} + C -->|read / reversible / high-confidence| X + C -->|sensitive OR low-confidence| A["approval card\nshows the RESOLVED values"] + A --> X[execute via the rails] + X --> V[verify: AX diff / screenshot / connector result] + V --> P +``` + +- **Resolve** - the slots ("the presentation", "the person I promised") are filled from memory at run time, each with a confidence. This is the "which presentation" intelligence (see 6). +- **Gate** - the approval card shows the *resolved* values: "Send `Q3-strategy.pptx` to Ali Chherawalla." One glance confirms the AI inferred correctly *and* that the action is safe. The gate is where inference and safety are confirmed together - it is the guard against a confident-but-wrong resolution, and the same mechanism handles "is it right" and "is it allowed." +- **Trust graduation** - suggest -> approve-each-run -> auto-run trusted routines. Irreversible steps (send, pay, delete, account-create) gate by default even inside a trusted routine. + +## 4. The rail hierarchy - cheapest reliable first + +The router picks the cheapest rail that will reliably do the step. Vision is the last resort, not the engine. + +| Rail | What it is | Reliability | Status | +| --- | --- | --- | --- | +| 0. Perception | Replay OCR + the accessibility tree - structured "sight", no ML grounding model | n/a | capture ships; AX reader exists | +| 1. Semantic | deep links / URL schemes, AppleScript / Apple Events, EventKit, Shortcuts, MCP connectors | ~100%, deterministic | **built** (calendar, reminders, contacts, messages, mail, open_url) | +| 2. Agent browser | embedded browser pane driven in-process, for novel web tasks (check-in, ordering) | good; no OS permissions | designed, not built | +| 3. AX-tree GUI | structured native control (AXPress / set-value) + replay of a demonstrated trace | good on well-behaved apps | AX read exists; act primitives not built | +| 4. Vision grounding | a downloadable model (GUI-Owl / Qwen3-VL) mapping pixels -> coordinates | the frontier ceiling (~35-45% novel, local) | fallback, last to build | + +**Three different things get called "seeing", and only rail 4 is the heavy one:** Replay OCR (rail 0, ships) powers detection and context; the AX tree (rail 0/3, exists) powers precise recording and reliable replay with no ML model; the grounding vision model (rail 4) only earns its place when the AX tree is dead (WhatsApp-class apps) or a recorded step drifted. So the assistant can do a great deal - and ship real value - before rail 4 exists. + +Your examples, mapped to rails: + +- **Open Maps** - rail 1, `open_url` (`maps://`). Built. Flawless. +- **Call a cab** - rail 1, deep link (`uber://?action=setPickup&dropoff=...`) opens the ride pre-filled; you confirm. Reliable. +- **Put on a movie** - rail 1 if the app has a title deep link (many do); rail 2/4 if it means driving the streaming UI. Mixed. +- **Order from Amazon** - rail 2, the agent browser driving the real site (no consumer API), ideally a pre-authored recipe for the reorder flow, payment behind the gate. Best-effort, improving. + +The rule is always: does the service expose a clean surface (deep link / API / connector / AppleScript)? If yes, reliable and cheap. If it is GUI-only, it is the hard long tail - the same ceiling every agent hits, worse with local models. "Does everything" is honest as a direction, delivered as: the clean-surface majority done flawlessly, the GUI long tail done assistively and improving, always honest about confidence. + +## 5. The demonstration recorder + +Record-by-showing turns "novel GUI automation is ~40% reliable" into "replay a known trace", because replaying a *known* path is a far easier task than figuring out a UI from scratch. + +- **Capture the action trace, not raw input** - for each meaningful step: the app, the AX element (fallback coordinate), the action (click/type/scroll/navigate), any typed text. The AX context is what turns a raw click into "clicked Send in Slack" and what makes replay survive window moves and resizes. +- **Primitives we already have** - the CGEvent tap (we ship the *listening* half in dictation-hotkey), the AX reader, and Replay frames for context and step verification. The recorder is Replay-with-intent plus AX-tagging, a new mode, not a new system. +- **Review + edit** - after recording we show the steps in plain language ("Open Slack", "Click Send", "Type: ..."); you delete, reorder, or **mark a step as a variable slot** (see 6). +- **Store** - as a skill with a trigger (manual / schedule / event), reusing the existing skills format. +- **Never record secrets** - secure-input detection (`IsSecureEventInputEnabled()`) hard-skips keystrokes into password fields. Recording credentials would be a serious mistake. + +On replay, deterministic trace execution runs through the rails; the LLM/vision comes in only as **recovery** when a step's AX target is gone or a verification fails. Deterministic automation with model fallback is strictly more reliable than model-drives-everything. + +## 6. Memory-grounded resolution (the recording gives the *how*, memory gives the *what*) + +A demonstrated trace stores the reliable UI path but leaves the content open. The slots - "the presentation I mentioned", "the person I promised" - resolve at run time by RAG over the memory spine: Replay observations + recent conversation + entity graph + files you touched, scoped by temporal and entity proximity, returning a value **plus a confidence**. + +- A generic assistant cannot do "send the deck I promised" - it has no record of your day. OGAD can, because it has both halves (the memory and the action). +- **Confidence drives the gate**: high + non-sensitive -> preview-and-go; sensitive -> gate with the resolved preview; ambiguous ("which of three decks?") -> disambiguate or show the top candidate for one-tap confirm. +- **Honest edges**: recency window needs temporal decay (grab *this* deck, not last month's); resolution quality rises and falls with what Replay captured (a healthy incentive to invest in memory); the dangerous case is confident-and-wrong, which the preview-at-gate catches for sensitive actions and a higher confidence bar catches for auto-run. + +**Slot resolution (data, from memory) is a different intelligence from UI-drift recovery (elements, from AX + vision).** Keep them separate: one finds content, one finds buttons. + +## 7. What is already built (the reliable foundation) + +The semantic rail and the shared gate exist on this branch (11 commits), and they are exactly the reliable execution layer this assistant needs: + +- **Transport-agnostic approval seam** - `actions:proposeApproval` with a read/navigate/mutate/irreversible risk taxonomy; the single gate every rail routes through. Backward-compatible with the current pro build. +- **Native actions helper (macOS)** - one Swift one-shot backend behind `runNativeAction`, covering calendar (create/list), reminders (create/list), contacts (search), Messages send, Mail send, and `open_url`. Mutations gate; reads run free; lenient date parsing; AppleScript values escaped against injection. +- **Wired into the chat tool loop** macOS-only, and shipped in CI. Fully unit-tested through an injected boundary. +- **TCC packaging** - the Info.plist usage strings and apple-events entitlement a signed build needs, brand-clean, guarded by a test. + +None of this is wasted by the reframe. It is rail 1, and rail 1 carries most of the value. + +## 8. What is genuinely new to build + +On top of the existing foundation. **The build order and schedule live in `COMPUTER_USE_PLAN.md` (the build doc), which sequences these as releases R1-R4** - the chat action tool ships first (R1, the lead's steer), then the reasoning/resolve layer, then routines, then the hard rails. Mapped to the releases: + +- **R1 - the chat action tool + the durable spine.** Turn the existing semantic rail (7) into a released, gated, verified tool the chat model calls, on a durable Action queue + state machine. This is the released foundation the rest layers on. +- **R2 - the reasoning engine + the slot/resolve layer.** Commitment/event detection + world-knowledge of required steps + read-only gap-checking + surfacing (the magic, and the safest - no risky automation); and RAG over the memory spine to fill "the presentation" with a confidence. +- **R3 - the demonstration recorder + the routine store.** Record-by-showing (recorder + AX tagging + review UI, see 5) and skills with schedule/event triggers; auto-detection feeds the "record this?" proposal. +- **R4 - the agent browser (rail 2) + the AX act-primitives and grounding vision model (rails 3-4).** The reasoned novel web tasks (check-in, ordering) and the dead-AX / drift-recovery fallback. Last. + +## 9. What we reuse (do not reinvent) + +This is the curated shortlist. The deep, component-by-component port map for the whole system (durable queue, brain, memory, routines, rails, models) with a port-vs-bespoke verdict per component lives in `PORTING_MAP.md`. + +| Source | License | What we take | +| --- | --- | --- | +| `@ui-tars/sdk` + the UI-TARS desktop app | Apache-2.0 | Operator seam + action parser; the ScreenMarker overlay trio (animated border, content-protected control widget, pre-action markers); desktopCapturer scaling; the macOS permission gate | +| nanobrowser | Apache-2.0 | TypeScript DOM-to-indexed-elements serialization for the agent browser | +| `@computer-use/nut-js` (or the community fork) | Apache-2.0 | input synthesis on the native rail | +| macos-automator-mcp | MIT | wrapped AppleScript/JXA intents plus its recipe knowledge base | +| bytebot (archived) | Apache-2.0 | takeover-as-recorded-actions (the human demonstration lands in the same action log) and the needs_help state - directly relevant to the recorder | +| Peekaboo (OpenClaw org) | MIT | reference for the AX-tree + vision hybrid on the native rail | +| OpenAdapt (MLDSAI) | MIT | the recorder / routines rail (R3): record once -> deterministic, self-healing local replay; each step carries a template crop, an OCR label, geometry, a structural locator, and postconditions (our per-step verify), and the model touches the script only to repair on drift. Port the trace format + self-heal rather than build one. | +| FlaUI / pywinauto | MIT / BSD-3 | the Windows UI Automation act-primitives reference for the accessibility rail (R3 Windows fast-follow) - UIA2/UIA3 element find + invoke / set-value, the analogue of the macOS AX act-primitives | +| Agent-S2 (Simular) | Apache-2.0 | open computer-use agent loop + router structure as a reference for the brain | +| OpenClaw | AGPL (patterns only) | the proactive cron/skills pattern and the killer briefing workflow; equally its incident record as the avoid-list (exposed gateways, sandbox-off, weak auth, unvetted skills) - we ship none of those surfaces | + +Everything adopted as code is Apache-2.0, MIT, or BSD - clean for the AGPL core + proprietary pro split (verify each license at the point of adoption; minitap/mobile-use asks for attribution). + +### Mobile (the adapter after v1) + +Mobile is a `DeviceController` adapter on the same engine (Section 6 and `ASSISTANT_ARCHITECTURE.md`), not a rewrite - and the actuation layer already exists to port rather than build: + +| Source | License | What we take | +| --- | --- | --- | +| Mobilerun (droidrun) | MIT | the mobile actuation rail for Android + iOS: inspect UI state, screenshot, tap / swipe / type, model-agnostic and local-model-capable (Ollama / OpenAI-compatible). The mobile `DeviceController` wraps this instead of writing driver glue. | +| minitap/mobile-use | Apache-2.0 (credit Minitap) | the mobile agent loop reference (first to 100% on AndroidWorld), a LangGraph multi-agent over low-level control | +| AppAgent / AppAgent-v2 (Tencent) | MIT | learn-by-demonstration + tagged-element perception (numeric tags over the Android view hierarchy) - mobile routines by showing | +| Mobile-Agent-v3 / GUI-Owl (X-PLUG) | MIT | full mobile agent reference + GUI-Owl as the shared grounding model (desktop + mobile trained) | +| Appium + appium-webdriveragent (iOS) + UiAutomator2 (Android) | Apache-2.0 | the low-level device drivers under the mobile rail (iOS via WebDriverAgent / XCTest, Android via UiAutomator2) | +| Maestro (mobile.dev) | Apache-2.0 | the YAML flow format as inspiration for the mobile routine trace | + +iOS stays intents-only for driving other apps (App Intents / Shortcuts) - Apple forbids reading or driving other apps, so the mobile GUI / vision rails are Android-first, exactly as the plan states. + +## 10. Safety + +- The **gate** is the confirmation of inference and safety together (3): irreversible classes (send, pay, delete, account-create) confirm even inside trusted routines; the card shows resolved values so a confident-but-wrong resolution is caught before it acts. +- **Screen content is untrusted input** - published studies show 86% attack success from adversarial pop-ups against GUI agents; prompt-level defenses fail, so the gate and an app allowlist are system-level. +- **Never see or type credentials** - secure-input detection hands password fields to the user; the recorder hard-skips them. +- **Takeover with a guarantee** - at logins and payments the agent pauses and frame capture stops while the user controls the surface. +- **Kill switch** - user input / Esc halts execution with the keypress consumed; the existing abort guard already guarantees a cancelled turn fires no side effects. +- **Everything executed lands in the approvals audit log.** +- **Trust graduates** - suggest -> approve-each -> auto-run; never jump to autonomous (the OpenClaw MoltMatch lesson). + +## 11. Build guidelines (binding, all surfaces) + +- **Design** - `off-grid-ai/brand` `DESIGN_PHILOSOPHY.md`: brutalist/terminal, Menlo, emerald as the only accent, black/white base, hierarchy by size and weight not color, no gradients, no emojis. All values from `@offgrid/design` tokens (`off-grid-ai/shared`) - no hardcoded hex. Desktop density per this repo's `docs/DESIGN.md`. +- **Copy** - `off-grid-ai/brand` `brand_tone_voice.md` + the outcomes-first rule: lead with what the user gets, mechanism as proof; no em dashes, no curly quotes, no exclamation marks, banned-word list applies. Applies to every approval card, suggestion, and notification. + +## 12. Open-core placement + +Rail-1 helper primitives and adapter plumbing are core infrastructure (like OCR). The reasoning engine, the recorder, the routine store, the resolve layer, and the approvals integration follow the existing pro spine. `pro/` changes land in `desktop-pro` first, submodule bump after. The engine (agent loop / router / resolver) lives in `off-grid-ai/shared` as `@offgrid/use`, consumed via `file:../shared/packages/use`. + +## 13. Decisions and open questions + +1. **Decided** - the model above: two generators (routine + reasoned) on one gated spine, rails cheapest-first, memory-grounded resolution, vision last. +2. **Decided** - `@offgrid/use` package name and `file:../shared/packages/use` consumption; the sibling `../shared` checkout is a build requirement (main already adopted it). +3. **Decided** - the semantic rail (rail 1) is the foundation and is built. +4. **Open** - parameterization depth: start faithful-with-marked-slots resolved by memory, layer richer LLM generalization on top. Confirmed lean: start faithful. +5. **Decided** - build order is release-led (`COMPUTER_USE_PLAN.md`): the chat action tool + durable spine ships first (R1, the lead's steer), then the reasoning + resolve layer (R2), then routines (R3), then the hard rails (R4). +6. **Decided** (per `PORTING_MAP.md` Section 6) - the default grounding model is UI-TARS-1.5-7B on desktop (Apache-2.0, GGUF + mmproj already published and mainline-runnable); GUI-Owl-1.5 / Qwen3-VL for mobile. Still not on the critical path (R4). + +## 14. Sources + +- Mobile-use agent loop: minitap/mobile-use (100% AndroidWorld) https://github.com/minitap-ai/mobile-use ; Mobile-Agent-v3 / GUI-Owl https://github.com/X-PLUG/MobileAgent +- Product UX: Claude Desktop browser pane https://code.claude.com/docs/en/desktop ; Codex embedded browser https://chierhu.medium.com/openai-codexs-browser-use-feature-b7dffa761d45 ; browser-use raw CDP https://browser-use.com/posts/playwright-to-cdp ; nanobrowser https://github.com/nanobrowser/nanobrowser ; UI-TARS desktop https://github.com/bytedance/UI-TARS-desktop ; bytebot takeover https://github.com/bytebot-ai/bytebot +- OpenClaw teardown: https://github.com/openclaw/openclaw ; Peekaboo https://github.com/openclaw/Peekaboo ; exposed gateways https://www.bitsight.com/blog/openclaw-ai-security-risks-exposed-instances +- Reliability calibration: OSWorld https://os-world.github.io/ ; pop-up injection (86%) arXiv:2411.02391 +- Grounding models: GUI-Owl-1.5-8B https://huggingface.co/mPLUG/GUI-Owl-1.5-8B-Instruct ; Qwen3-VL grounding arXiv:2511.21631 ; Holo3.1 https://huggingface.co/blog/Hcompany/holo31 +- macOS surface: Electron `AXManualAccessibility` https://www.electronjs.org/docs/latest/tutorial/accessibility/ ; secure input TN2150 https://developer.apple.com/library/mac/technotes/tn2150/_index.html ; Electron debugger https://www.electronjs.org/docs/latest/api/debugger ; node-mac-permissions https://github.com/codebytere/node-mac-permissions +- Brand: https://github.com/off-grid-ai/brand ; `@offgrid/design` in https://github.com/off-grid-ai/shared diff --git a/docs/COMPUTER_USE_PLAN.md b/docs/COMPUTER_USE_PLAN.md new file mode 100644 index 00000000..810d84cf --- /dev/null +++ b/docs/COMPUTER_USE_PLAN.md @@ -0,0 +1,118 @@ +# The proactive assistant - build plan and timeline + +Companion to `COMPUTER_USE.md` (the product model), `ASSISTANT_ARCHITECTURE.md` (the system design), and `PORTING_MAP.md` (the port-vs-bespoke research). + +> **This is the doc to build from.** Work release by release, top to bottom: a release is not done until its checkpoint passes, and the next release does not start until it does. The other three docs are references. Adjust the plan here at each checkpoint; never fork a second plan. + +**Re-cut (August 14, 2026 - the lead's steer + R1 field feedback).** The release after R1 is **all four rails, chat-driven, on both platforms**, plus the approval UX rebuild the R1 pro-path test demanded. The reasoning engine (proactive) and routines move after it. R1 itself is done: 17/19 checklist boxes, both PRs open and green (OGAD #81, shared #4). + +**Standing assumptions** + +- Solo developer, AI authoring the code end to end. +- Release-led: each release is a real, demoable, shippable increment. Desktop = macOS + Windows. +- **Port the plumbing, build the product** - each release names its ports (all MIT / Apache-2.0 / BSD, all in-process); the full map is `PORTING_MAP.md`. +- **Offline scope, stated precisely.** The brain runs with zero network on every platform. An action whose effect lives on an external service needs that service reachable at execution time - so the rails prefer local apps whose writes land locally and sync later (EventKit / Mail on macOS, local Outlook on Windows), and online-only actions are labeled honestly. +- **Reliability rules the router must honor** (architecture doc, Section 4): effect-verification lives in the engine; a cross-rail escalation is a re-fire under the same retry policy - a non-retryable action never escalates. The DeviceController routing is a thin layer over these. +- Checkpoint discipline: a checkpoint is a verifiable, demoable milestone. + +## Build guidelines (standing, all releases) + +- **Design** - `off-grid-ai/brand` `DESIGN_PHILOSOPHY.md`: brutalist/terminal, Menlo, emerald-only accent, black/white base, hierarchy by size and weight not color, no gradients, no emojis. All values from `@offgrid/design` tokens. Desktop density per `docs/DESIGN.md`. +- **Copy** - `off-grid-ai/brand` `brand_tone_voice.md` + outcomes-first: no em dashes, no curly quotes, no exclamation marks, banned-word list applies. +- **Cross-platform from the seam.** Callers depend on the `DeviceController` port and the shared engine, never on a concrete OS. +- **Port before writing.** Check `PORTING_MAP.md` / `COMPUTER_USE.md` Section 9 first; verify the license at the point of adoption; honor the AGPL / source-available avoid-list. + +## Releases + +| Release | What ships | Status | +| --- | --- | --- | +| **R1. Chat actions on the durable engine** | The semantic rail in chat on macOS (reminders, calendar, messages, mail, open, lookups) through the `@offgrid/use` engine: durable queue, payload-hash gate, retry-once-with-verify, read-back verification, effect journal. Windows toolchain green (installer artifact); the Windows semantic rail (local Outlook COM) built behind the port. | **Done.** PRs: OGAD #81, shared #4. Record: `R1_CHECKLIST.md` | +| **R2. Full rails in chat, both platforms + Approval UX v2** | Windows chat exposure; the browser rail (watched web tasks, takeover at login); the vision rail (supervised GUI actions, UI-TARS-1.5-7B); the approval experience rebuilt (inline in chat, outcome feedback, risk-tiered auto-run); the safety pass. ~5-6 working days. | **next** | +| **R3. Notices you** (was R2) | Reasoning + resolve + gate: commitment/gap detection over Replay, memory-resolved slots with confidence, the proactive Day surface. Cross-platform (memory + LLM). Pro-side code lands in desktop-pro (access in place). ~3 days. | after R2 | +| **R4. Routines** (was R3) | Record-by-showing + self-healing, per-step-verified replay (OpenAdapt design). macOS-first; the Windows UIA adapter as the fast-follow (napi-rs over the `uiautomation` crate + SendInput, Terminator head-start). ~2-3 days + fast-follow. | after R3 | + +The split: `shared` holds the durable cross-platform brain (`@offgrid/use`, reused by mobile later); this repo holds the rails, surfaces, and product integration; pro business logic lands in `desktop-pro`. + +## R1 - chat actions on the durable engine (DONE) + +Shipped scope, guarantees, and evidence live in `R1_CHECKLIST.md` and the PR bodies. Merge order: **shared #4 before OGAD #81** (main's CI resolves `@offgrid/use` from shared main). The release DISPATCH waits for R2 per the re-cut - one versioned release ships both. + +**R1 field verdicts driving R2** (from the pro-path smoke test): + +- Approving a card gives no completion feedback - the chat message says "pending" forever and nothing reports the run. (The engine path already reports verified outcomes; the legacy pro path is the old system.) +- Reversible simple actions (a reminder) should not need a human gate at all. +- Chat-originated approvals belong INLINE in the conversation, not on a separate screen; the Actions screen's job is unattended actions (proactive, scheduled) plus the audit log. + +## R2 - full rails in chat, both platforms + Approval UX v2 (~5-6 days) + +Everything chat-drivable on both OSes, honestly tiered, with an approval experience that reads like a conversation instead of a queue. + +### A. Windows chat exposure (~1 day) + +- Per-platform tool specs: win32 exposes the engine-routed set the Outlook rail supports (calendar_create_event, reminders_create, mail_send, open_url); reads stay macOS-only until the Outlook read verbs land. +- A win32 inline runner for open/navigate; the engine path handles mutations end to end (the rail shipped in R1). +- Outlook read-back verifiers (list verbs mirroring the mac ones) so Windows gets verified outcomes too. + +### B. Approval UX v2 (~1-1.5 days, core + desktop-pro) + +- **Inline approval card in chat**: resolved values + Approve / Edit / Reject in the conversation flow, driven by the engine gate (`resolveActionGate`). The Actions screen remains the queue for unattended actions plus the audit log. +- **Outcome feedback everywhere**: approve -> the engine executes -> the verified result lands back in the chat turn and on the card ("Created - verified", or the honest failure). This is the pro approval-executor migration: pro's queue resolves the engine gate instead of running its own executor, so payload binding and verification hold on the pro path too. +- **Risk-tiered gating** (decision 8.3's lean, now policy): reads/navigate free; reversible mutations (reminder, calendar) auto-run with a verified confirmation and an Undo affordance; sends and irreversible actions keep the gate. + +### C. The browser rail (~1.5-2 days) - cross-platform on arrival + +- Embedded pane over Electron's `webContents.debugger` (raw CDP): **nanobrowser's** TS dom module + overlay as starting code, **browser-use's** snapshot + AX-merge + numeric-index as the algorithm, **Stagehand's** act/observe/extract + Zod as the API. +- Chat-drivable web tasks (check-in, ordering) - watched live, takeover at any login/identity step, gated at the identity boundary. + +### D. The vision rail (~1.5-2 days) - the supervised tier, labeled so + +- **UI-TARS-1.5-7B** catalog entry (Apache-2.0, GGUF + mmproj published; a ~5GB download via the Models screen); **OmniParser v3** (MIT) set-of-marks fallback for the bundled model. +- The operator spine from **@ui-tars/sdk** (nut.js swapped for **@nut-tree-fork**/robotjs); mac input via CGEvent, Windows via SendInput. +- Supervised UX: the ScreenMarker-style overlay, pause-on-user-input, the kill switch (Esc halts with the keypress consumed). +- The WhatsApp file-share recipe as the showcase (behind the gate). + +### E. Safety pass + the release + +- Injection-resistance review (screen content is untrusted input), kill-switch e2e, per-rail verification depth honored, release-readiness checklist. +- **Checkpoint / release dispatch:** on macOS AND Windows - a semantic action, a watched web task with takeover, and a supervised vision action all run from chat, gated by tier, with verified outcomes reported inline. One versioned release: the signed/notarized .dmg + the Windows NSIS .exe (unsigned until the cert - decision open with the lead). + +**R2 risks:** the vision tier on a 7B local grounder is best-effort - ship it labeled supervised or not at all; Windows browser/vision needs a human on a real Windows machine (CI proves builds, not clicks); the model download adds a Models-screen surface; Approval UX v2 touches the live chat surface (the R1 lesson stands - behavior tests per branch, the plain path untouched for non-action turns). + +## R3 - notices you (was R2, ~3 days) + +Scope unchanged: commitment and gap detection over the Replay observation + entity spine; the resolve layer (RAG over memory returning value + confidence); proposals surfacing on the Day feed and executing through the same engine and inline approval UX. Ports: sqlite-vec (inside the app DB), LlamaIndex.TS memory blocks, Mem0's dedup loop, Orama hybrid ranking; techniques: HippoRAG PageRank, bi-temporal facts, the WSDM commitment rubric. Pro-side code (reasoning, resolve policy, feed UI) lands in desktop-pro. Checkpoint: on a seeded profile, on both OSes, an un-actioned commitment surfaces and "send the deck I promised" resolves from context and runs, gated by tier. + +## R4 - routines (was R3, ~2-3 days + the Windows fast-follow) + +Record-by-showing + faithful replay per the OpenAdapt design (compiled-step schema, resolution ladder, postconditions, repair-as-diff); Playwright codegen for the browser lane; memory-resolved variable slots; the plain-language review UI. macOS AX-as-eyes with actuation capped at press/set-value; the Windows UIA adapter (reader + SendInput) as the fast-follow. Checkpoint: record a routine once; it replays per-step-verified with a slot resolved from memory at run time. + +## Dependencies + +| What | Needed by | Note | +| --- | --- | --- | +| shared #4 merged before OGAD #81 | now | main's CI resolves `@offgrid/use` from shared main | +| Windows signing cert | R2 release | wiring exists (WIN_CSC_LINK secrets); publishes unsigned until then | +| A human on a real Windows machine | R2 | browser/vision click-through + the model-load smoke (`WINDOWS_TEST_PLAN.md`) | +| UI-TARS-1.5-7B GGUF + mmproj catalog entry | R2-D | the vision model install | +| desktop-pro access | R2-B, R3 | in place (cloned at pro/) | +| Seeded memory fixtures | R3 | detection + resolution tests without a live profile | +| OpenAdapt trace/replay port + the `axuielement` napi addon | R4 | the recorder + the mac AX read | + +## Risks + +| Risk | Mitigation | +| --- | --- | +| Vision reliability (the frontier ceiling) on a local 7B | supervised tier, labeled; cheapest-rail-first routing; set-of-marks fallback; the gate on everything consequential | +| Approval UX v2 touches the live chat surface | behavior tests per branch; the plain path stays untouched for non-action turns | +| The Windows human-testing gap | recorded dependency; release notes honest about machine-verified vs human-verified | +| Solo schedule | releases independently valuable; scope trims at the tail (the vision showcase, Windows polish), never the shipped core | + +## Out of scope (unchanged) + +The mobile adapter (post-v1: Appium/WebdriverIO + DroidRun Portal + GUI-Owl-1.5/Qwen3-VL), background/headless autonomous runs, store distribution. + +## Tracking + +- R1 record: `R1_CHECKLIST.md`. R2 gets its own checklist when it starts. +- Small commits per verified unit, merge not squash. PR evidence rules apply. +- Checkpoint review against this doc at each release; plan changes are edits here. diff --git a/docs/DEMO_DESIGN_BRIEF.md b/docs/DEMO_DESIGN_BRIEF.md new file mode 100644 index 00000000..894899a4 --- /dev/null +++ b/docs/DEMO_DESIGN_BRIEF.md @@ -0,0 +1,126 @@ +# Design brief - Off Grid AI proactive assistant demo + +**For:** whoever is generating the design artifacts (Claude, or a designer). +**Deliverable:** high-fidelity mockups of the "ideal outcome" demo as an **interactive HTML artifact** (desktop), that **looks like the real Off Grid AI Desktop app** - same shell, same components, same feel. Real content throughout, never lorem. Five screens tied into one story (Section 6). +**Most important instruction:** match the actual app in Section 4. The app already exists; do not invent a new visual language. If a screen would not sit comfortably next to the real Models or Chat screen, it is wrong. +**Self-contained:** the app's look and tokens are inlined below; you do not need the repo. + +--- + +## 1. What the product is + +Off Grid AI Desktop is a **private, on-device assistant that notices what you need and acts on it.** It already watches your day locally (screen capture -> on-device OCR -> a private memory of what you saw and did). We are adding the ability to **act**. This demo shows that. + +Four things make it different, and the design must make all four feel true: +1. **It knows you** - it acts on *your* context ("the deck I promised" resolves to the actual file from memory). +2. **It is proactive** - it surfaces the flight you have not checked in for, the promise you made, the routine you run every morning, before you ask. +3. **It is private** - everything runs on your Mac, nothing leaves the device. Reinforce it quietly (a small "on-device" cue), never a banner. +4. **It is one general engine, not a pile of features.** The flight nudge, the promised deck, a renewal, a reply you owe - all the *same* machinery. There is no "Flights" tab, no per-situation section. These are transient items the assistant generates, here when relevant, gone when handled. + +The interaction principle to convey: **it routes to the cheapest reliable path and acts through the app and content you actually mean**, and **always shows what it will do before it does it.** + +## 2. Who it is for and the tone + +A sharp knowledge worker (beachhead: engineers) who lives in many apps. The feeling: **calm, dense, immediate, trustworthy** - a terminal/developer tool, not a friendly consumer chat app. + +## 3. The look (from the real app - copy this exactly) + +The app is **monospace, flat, outlined, and quietly technical.** It is NOT razor-sharp brutalism and it is NOT airy editorial SaaS - it sits between: flat surfaces with **1px borders and moderate corner radius (about 6-8px)**, **no drop shadows**, a very subtle **dotted-grid background texture**, and **Menlo monospace for every character on screen**. + +- **Typeface:** **Menlo** (or `ui-monospace, "SF Mono", Menlo, monospace`) everywhere - labels, headings, body, numbers. Weights stay light-to-regular. Hierarchy from size, weight, spacing, uppercase - never a second font. +- **Uppercase, letter-spaced labels** for section headers, tabs, and status tags (e.g. `MODELS`, `AVAILABLE TO DOWNLOAD`, `TEXT` / `IMAGE` / `VOICE`, `VISION`). Body and buttons are normal case. +- **Accent: emerald, only emerald.** The single accent - active nav, the one primary action per screen, focus, links, success, status tags. Everything else is a monochrome gray hierarchy. Do not add a second accent or color-code categories. +- **Semantic colors exist only for their exact job:** a muted **amber** for a single caution label (the app uses it for a `CHALLENGER` tag), a **red** only for error/health ("Model stopped"). Used rarely. +- **Exact tokens (dark mode - the primary theme for the demo):** background `#0A0A0A`, surface `#141414`, surface-light `#1E1E1E`, surface-hover `#252525`, border `#1E1E1E`, border-light `#2A2A2A`, text near-white, muted text mid-gray, accent `#34D399`. +- **Exact tokens (light mode - also ship it):** background `#FFFFFF`, surface `#F5F5F5`, surface-light `#EBEBEB`, text `#0A0A0A`, border `#E5E5E5`, accent `#059669`. +- **Component vocabulary (reuse these shapes, do not invent new ones):** + - **Outlined button** - 1px border, ~6px radius, icon + label, flat (the app's `Import .gguf`, `Download`, `+ New chat`, `Back`). Hover lightens the surface. + - **Solid emerald button** - the one primary CTA per surface, emerald fill with dark text (the app's `Configure`). Circular emerald send button with an up-arrow in the composer. + - **Outlined pill toggle** - small, rounded, icon + label, emerald when active (the composer's `All memory`, `Thinking`, `Image`). + - **Status tag** - tiny uppercase, emerald 1px outline + emerald text + a small icon (the app's `VISION` tag). Use this shape for risk/confidence tags. + - **Metadata line** - gray, dot-separated: `Qwen · 4B · 3.4GB · Mar 2026`. + - **Bottom CTA card / toast** - a flat outlined card pinned near the bottom with an icon, a title + one gray subtitle line, a solid emerald action, and an X (the app's "Set up your local AI - Configure"). **This is the exact shape to reuse for a come-up and for a toast.** +- **Density:** comfortable-dense. Rows and cards have real breathing room (this is not a cramped table); 2-column card grids where it fits; sticky headers. Design at 1440px+ wide. +- **Motion:** restrained - 150ms transitions, slide+fade for panels, subtle active-press. Nothing pops in hard. + +## 4. The actual app shell (render this frame around every screen) + +**Left sidebar** (expanded, about 240-260px; the app can also collapse to an icon-only ~64px rail - show the expanded one): +- Top: the emerald chip logo + wordmark **`Off Grid AI`**, and a small panel-collapse icon. +- A full-width outlined **`< Back`** control. +- The nav list, each row = **monochrome icon + label**, generous row height: **Search, Day, Replay, Reflect, Meetings, Actions, Entities, Projects, Chat, Voice, Vault, Clipboard, Devices, Integrations, Models, Gateway.** For the demo, **add one new item after Actions: `Routines`.** +- **Active item styling (important):** emerald icon + emerald label + a subtle emerald-tinted row background + a thin emerald bar on the row's left edge. Inactive: gray icon, near-black/near-white label. +- A divider, then quiet utility rows: a health line with a red pulse icon (e.g. `Model running`), `Theme: System`, `Settings`, `Mobile app` (with an external-link glyph). + +**Main area:** a header row with a small icon, a **title + one gray subtitle** (e.g. Chat shows `Off Grid AI` / `Private, on-device - chat, generate, and build`), and a cluster of square outlined icon-buttons top-right. Below it, the screen's content. A faint dotted-grid texture bleeds in at the top and bottom edges. + +Every demo screen must sit inside this shell (sidebar + header), so it reads unmistakably as Off Grid AI. + +## 5. Where the assistant lives (real nav, minimal additions) + +- **Come-ups live in `Day`** - the ambient home. The proactive items surface as a **"Needs you" section pinned at the top of Day**, above the retrospective day timeline. Ephemeral rows, never tabs. Day *is* the assistant, forward-looking on top. +- **The gate lives inline + in `Actions`** - a come-up expands *in place* into the approval card (fast path); `Actions` (which already exists, with a checkbox icon) is the full queue and audit. +- **`Routines` is the one new tab** - the library of saved automations; recording opens as a modal from it. +- **When you are away:** a toast (the bottom-CTA-card shape) and a menu-bar count. + +## 6. The five screens (one day, one story) + +Each screen must be **self-understandable** - legible without a caption (the come-up says what it is; the card shows exactly what it will do). Keep the one-line "proves:" note as an annotation. + +**Screen 1 - Day, with "Needs you" on top. Proves: proactive, knows you, one general engine.** +The hero, inside the real shell with **Day** active in the sidebar. Header: a calendar icon + `Day` + a gray subtitle + the date. The main column opens with an uppercase gray section label **`NEEDS YOU`**, then a short list of come-ups as flat outlined rows (reuse the bottom-CTA-card shape, one per row). Show a **mix of situations** so the generality is obvious: +- `You fly to SFO tonight, 21:40. Not checked in, no boarding pass found.` -> emerald `Check me in` + quiet `Later`. +- `You told Ali you'd send the Q3 deck by tonight.` with a gray context line `from your 10:15 call` -> `Send it` + `Later`. +- `getoffgridai.co renews tomorrow. The card on file expired.` -> `Update card` + `Dismiss`. +- A detected routine that already ran: `Morning brief - 09:02 · 12 unread, 3 need you` with a two-line synthesis from Mail and Slack. +Below `NEEDS YOU`, an uppercase `EARLIER TODAY` section with a dense retrospective timeline of what you did (a few rows), so it reads as an evolution of the existing Day view. Quiet "on-device" cue somewhere unobtrusive. + +**Screen 2 - The approval card, expanded inline from a Day row. Proves: it acts on your real context, shows the evidence and its confidence, and you confirm before it acts.** +The single most important screen, and the one no competitor ships. The user hit `Send it`; the row **expands in place** into a flat outlined card. It shows the **resolved action, each slot with its evidence and a confidence tag** (not a vague action, and not just the value - the *proof* it picked right): +- Title line: **`Send Q3-strategy.pptx to Ali Chherawalla`**. +- **Resolved slots, each a row:** a label, the resolved value as an editable pill, a gray provenance line (the evidence), and a small confidence tag using the status-tag shape: + - `File` -> `Q3-strategy.pptx` · gray: `you called it "the deck" in your 10:15 call · last edited 20m ago` · emerald tag `HIGH`. + - `To` -> `Ali Chherawalla ` · gray: `the "Ali" you promised · only deck shared with him` · emerald tag `HIGH`. + - `Via` -> `Mail` (the rail it will use). +- A risk tag near the actions in the status-tag shape but amber: `SEND · NEEDS APPROVAL`. +- Actions: solid emerald **`Approve and send`**, quiet outlined **`Edit`**, text **`Dismiss`**. +- Then show the **post-action toast** (bottom-CTA-card shape): `Sent to Ali - Q3-strategy.pptx`. (Annotate: the toast reflects the real send result, never a guess; the full queue lives in `Actions`.) +- **Also design the low-confidence variant of one slot** (a second small card state): instead of a pre-filled value, the slot becomes a picker - `Which deck did you mean?` with two candidate rows, each showing its own evidence (`Q3-strategy.pptx - shared with Ali, edited 20m ago` vs `Q3-final.pptx - edited last week`) and a select control. Low confidence disambiguates *before* the confirm, it never guesses. + +**Screen 3 - The reasoned nudge in action (the flight). Proves: it notices what should happen and helps, handing off safely.** +The flight come-up expanded into a short flow. State one: `Check me in` / `Remind me at 20:00` / `Dismiss`. State two: it opened the airline check-in and filled the known fields (confirmation number, name from memory), then **handed off** at the identity/seat step - `Your turn - confirm your seat` (capture paused, shown as a small note). End state toast: `Boarding pass saved`. + +**Screen 4 - Record a routine (modal from Routines). Proves: the user can author automations by demonstrating.** +A modal/slide-over in the app's style. State one - **recording:** a calm indicator (a thin emerald border around the app, or a small emerald status pill `Recording routine - do it once, I'll learn it`), NOT a big red dot. State two - **review the captured steps:** an editable list of semantic step cards in plain language (`Open Slack`, `Go to #standup`, `Post: Standup - {date}`), one step showing a **variable slot** as an emerald pill (`{date}`, or `the deck`) that resolves from memory each run. Controls to reorder/delete a step, an inline hint to mark a value as a variable, and a **trigger** row (`Manual` / `Schedule` / `When I ...`). Primary solid emerald `Save routine`. + +**Screen 5 - Routines tab. Proves: detected and demonstrated routines live together on one spine.** +`Routines` active in the sidebar. Header: `Routines` + subtitle. A dense list/table: a mix of **detected** (`Morning brief`, auto-found) and **recorded** (`Standup note`, `Send weekly report`). Columns: name, trigger (`09:00 weekdays` / `manual` / `event`), last run, and a trust tag in the status-tag shape (`SUGGEST` / `AUTO`). A run control per row, an outlined `Record routine` button top-right, sticky header. + +## 7. Copy voice (every string) + +- **Lead with the outcome, in the user's language:** "Send the Q3 deck to Ali", not "Execute mail.send". +- Plain and direct; proof over adjectives. +- **No em dashes** (use " - "), no curly quotes, no exclamation marks, no emojis. +- Banned words: revolutionary, seamless, empower, leverage, robust, comprehensive, crucial, delve, tapestry, testament, foster, showcase, enhance; and AI-slop ("it's not X, it's Y", "serves as"). +- A control says exactly what it does; the toast says it happened. +- Real names and content (Ali Chherawalla, `Q3-strategy.pptx`, SFO 21:40, getoffgridai.co). + +## 8. Deliverable format + +- **One interactive HTML artifact** rendering the real app shell (sidebar + header) with the five screens; the sidebar switches Day / Actions / Routines, numbered steps handle the flight/record sub-states and the low-confidence card variant. Self-contained (inline CSS, monospace stack, no external fonts/CDNs). Designed for 1440px+. +- **Dark mode primary (tokens above); include a working light-mode toggle.** Both properly styled. +- Each screen annotated with its "proves:" line, but the screen must read on its own without it. +- If one artifact is too much, deliver **Screen 1 (Day) and Screen 2 (approval card)** first - they carry the demo. + +## 9. Do not + +- **Do not invent a new app shell or visual language.** Match Section 4. No "Assistant" tab, no "Flights"/"Bills"/"Travel" tabs - come-ups are transient content in Day. +- **Do not over-round or over-soften into consumer SaaS** (big rounded cards, drop shadows, gradients, pastel fills) - the app is flat, outlined, ~6px radius, monospace. +- **Do not over-sharpen into hard brutalism either** (zero-radius, heavy black rules, cramped rows) - the real app is calmer than that. Match the screenshots' feel. +- Do not use a second accent or color-code categories; emerald only, amber/red only for caution/error. +- Do not use a non-monospace font anywhere. +- Do not design mobile-first; wide desktop only. +- Do not make the assistant a chat-bubble feed; it speaks through the Day rows and approval cards. +- Do not over-explain privacy with a banner; a quiet, constant cue. + +The north star: **it looks like it shipped inside Off Grid AI** - monospace, flat, outlined, emerald-on-dark, dotted-grid - and every screen makes it obvious the assistant knows you, acts on your real context, shows the evidence and its confidence, and always confirms before it acts. diff --git a/docs/PORTING_MAP.md b/docs/PORTING_MAP.md new file mode 100644 index 00000000..af69b5da --- /dev/null +++ b/docs/PORTING_MAP.md @@ -0,0 +1,182 @@ +# Porting map - what we port vs what we build (deep prior-art research) + +**Status:** August 13, 2026. Answering the lead: "people must have already built stuff like this - what can we port instead of building?" This is the deep sweep across every layer of the assistant, with a blunt verdict per component. Companion to `COMPUTER_USE.md` (Section 9 is the curated shortlist), `ASSISTANT_ARCHITECTURE.md`, and `COMPUTER_USE_PLAN.md`. + +## The answer in one paragraph + +The lead is right, and the honest split matters: **we port the plumbing and keep the product.** Almost every mechanism we need exists as a permissively licensed open project - a durable-queue pattern, a state machine, constrained decoding, a vector store, a record-and-replay engine, the rails, the grounding models. What does NOT exist off the shelf is the thing that makes this product: a single-process, offline, on-device pipeline that joins actions to a personal screen-memory, gates them behind human approval, and verifies their effect. So the plan is: **assemble the pipeline from small permissive libraries + documented blueprints, and write only the product-defining glue** (the Action contract, the approval policy, the resolve-with-confidence layer, the commitment-gap reasoner, effect-verification, and the DeviceController + rail-selection). That glue is bespoke by nature - no upstream targets a single-process offline device wired to a personal memory - not by choice. + +**Verdict legend:** `port-wholesale` (adopt/vendor the code), `port-components` (lift specific modules/algorithms), `port-design` (reimplement its architecture), `inspiration-only` (study, don't copy), `adopt-as-model` (ship the weights), `bespoke` (must build - explained why). + +**Method:** five parallel research tracks (durable execution/HITL, agent brain/tool-calling, memory/resolve/proactive, record-replay/routines, rails/grounding-models), licenses verified per project. + +--- + +## 1. The durable action queue + state machine + scheduling + approval gate + +The lead's exact example. Finding: **durable execution is heavily built - but every mature engine is a server backed by Postgres/Cassandra/Kafka**, which is a non-starter for a single-process offline app (the same "bundled sidecar" fragility as the llama-server saga). No embeddable engine bundles queue + state machine + approval + verify. So we assemble it. + +| Need | Port from | License | Local-first fit | Verdict | What we take | +| --- | --- | --- | --- | --- | --- | +| Action state machine | **XState v5** | MIT | Yes, zero-dep, runs on RN too | **port-wholesale** | Each Action's lifecycle as a persisted statechart; `getPersistedSnapshot()` -> SQLite, rehydrate on launch; same machine on future mobile core | +| (lighter alt) | robot3 | BSD-2 | Yes, 3kb | port-components | If XState feels heavy; you write the serialize/restore glue | +| Durable SQLite queue | **sqliteq** (TS port of **goqite**) | MIT | Yes, better-sqlite3 | **port-wholesale** (transport) | SQS-style leased-message + visibility-timeout + auto-extend + retry loop | +| Scheduling (cron/delay) | plainjob | MIT | Yes, better-sqlite3 | port-components | Cron + delayed jobs; worker-death -> re-queue | +| Idempotent enqueue | better-queue-sqlite | MIT | Yes | port-components | Task-merge/dedup by id | +| Retry-once / resilience | **cockatiel** or **p-retry** | MIT | Yes, zero-dep | **port-wholesale** | Retry-once is a one-line policy; circuit-breaker/timeout free for connector calls | +| Approval gate (HITL) | **LangGraph.js** `interrupt -> resume` | MIT | Yes (checkpoint-sqlite) | port-components | The pause-before-side-effect -> surface proposed Action -> resume-from-checkpoint contract; do it locally against our own UI | +| HITL outcome model | HumanLayer | Apache-2.0 | No (cloud broker) | inspiration-only | The typed approve/deny/respond contract; de-couple request from response | +| Design spec | **DBOS Transact** semantics + **Gunnar Morling's "durable execution on SQLite"** blueprint | MIT / blog | reference | port-design | `(action_id, step)` PK, status per step, replay COMPLETE steps, idempotency key forwarded to side effects; Morling's PoC is near copy-paste | + +**Not viable for local-first (server + external DB, or license):** Temporal (MIT, needs Cassandra/Postgres), DBOS-TS (MIT, Postgres-bound - Go build has SQLite, TS not yet), OpenWorkflow (Apache-2.0, in-process TS step-checkpointing - the right shape, but Postgres-only today with SQLite "coming soon", early and fast-moving, and no approval/HITL or risk-aware retry; its step.run ergonomics are a design reference for our engine facade), Restate (**BUSL-1.1** runtime), Inngest (**SSPL** server), Trigger.dev (Postgres+Redis+Docker), Windmill (**AGPL** + Postgres), LittleHorse (**AGPL** + Kafka), Hatchet/Cadence/Resonate (all server). Their *semantics* are the gift; their deployment model is the disqualifier. **Re-evaluate list:** DBOS-TS and OpenWorkflow, if either ships a solid SQLite backend. + +**Bespoke (build it, ~a few hundred lines):** +- The orchestration glue that wires queue -> state machine -> approval -> execute -> verify. No library combines all five on-device. +- **Effect-verification** - "did the email actually send / the file actually move" - has **zero prior-art library**; it's inherently per-connector (read-back, re-query). Ours. +- The crash-after-execute-before-record window, closed with idempotency keys on the outbound side effect (every engine concedes this and solves it the same way). +- The checkpointer/queue adapter against our existing better-sqlite3 handle (SSOT: one DB answers "what is this Action's state"). + +--- + +## 2. The agent brain: constrained output, reliability, loop, router, tools + +Finding: **we already ship the best-fit constrained-decoding engine.** llama.cpp does JSON-schema -> GBNF and `response_format` grammar-constraining today. Most of this track is thin layers on top, in TypeScript. + +| Need | Port from | License | Local fit | Verdict | What we take | +| --- | --- | --- | --- | --- | --- | +| Constrain Action shape | **llama.cpp GBNF / `response_format`** (already bundled) | MIT | Yes | **port-wholesale** | Send the Action schema as `json_schema`; the model cannot emit invalid-shaped Action JSON. Gotcha: schema is NOT injected into the prompt - still describe the tool enum in the system prompt | +| Native tool-calling | llama-server `--jinja` lazy grammars | MIT | Yes | port-wholesale | Optional OpenAI-style `tools` mode for models with a good native template; prefer our own single-Action grammar for determinism | +| Weak-model reliability | **Schema-Aligned Parsing (SAP), from BAML** | Apache-2.0 | Yes | **port-components** | The single biggest weak-model jump in the literature (e.g. 19.8% -> 92.4%): coerce sloppy-but-close output to schema post-hoc. Reimplement a focused TS coercer keyed to the Action schema | +| Validate + retry | Instructor-JS pattern | MIT | Yes | port-components | Zod validate -> feed the error back -> re-ask, bounded to N. ~50 lines | +| Wrapper patterns | node-llama-cpp | MIT | Yes | port-components | The ChatWrapper seam (per-model template behind one interface) + optional-param grammar handling | +| Faster CFG engine | llguidance | MIT | Yes (build flag) | reserve | `-DLLAMA_LLGUIDANCE=ON` only if native GBNF coverage/perf bites; adds a build-gate surface, defer | +| Agent loop | LangGraph.js pattern | MIT | Yes | port-components | Checkpointed LLM-node <-> tool-node <-> conditional-edge loop; reimplement, don't take the LangChain dep | +| Router seam | Mastra (Apache core) / VoltAgent (MIT) | Apache/MIT | Yes | port-components | One interface over interchangeable model backends (our DSP rule). VoltAgent is MIT+TS+MCP+Zod - copy concrete code | +| Cheap-first routing | semantic-router concept | MIT (Python) | reimplement | port-components | Embedding-similarity intent classifier as the router's fast lane; skip the LLM when confident. Reimplement in TS over our local-embedding path | +| Connectors / tools | **MCP TypeScript SDK** | MIT/Apache-2.0 | Yes | **port-wholesale** | The whole client/server tool transport; this is our act surface, don't reinvent | + +**Model choice (verify per-checkpoint license before bundling):** function-calling-tuned small models - Salesforce xLAM-2, Hammer 2.1, NousResearch Hermes (native XML tool parser in llama-server), MeetKai functionary - picked on the Berkeley Function-Calling Leaderboard, not vibes. + +**Inspiration-only (Python, or wrong runtime):** Outlines/outlines-core, guidance, LMQL, jsonformer (Python), XGrammar (MLC not llama.cpp), Agent-S (Python, feeds the vision rail). + +**Bespoke:** the Action schema + durable pipeline (the SSOT for "what the agent is doing"); approval-gated execution + the privacy boundary; the router *policy* tuned to our bundled model's real behavior; the SAP coercion rules + retry prompts wired to our Action contract; the engine-health/stderr-classification path (`llama-error.ts` has no upstream equivalent). + +--- + +## 3. Memory + resolve/RAG + commitment/proactive detection + +Finding: the vector layer is a clean port; the "memory frameworks" are mostly Python (algorithm inspiration, not code); commitment/proactive detection is **genuinely bespoke** over our Replay spine. + +| Need | Port from | License | Local fit | Verdict | What we take | +| --- | --- | --- | --- | --- | --- | +| Vector store | **sqlite-vec** | Apache-2.0 / MIT | Yes, inside better-sqlite3 | **port-wholesale** | Vector KNN in the SAME DB file we already ship - one file, one transaction, one backup, no new process. Highest-value, lowest-risk port | +| Scaling alt | LanceDB (`@lancedb/lancedb`) | Apache-2.0 | Yes, embedded Node | port-wholesale (alt) | Real ANN when the corpus outgrows brute-force KNN (a second store to keep in sync - an SSOT tax) | +| Embeddings | bundled **llama-server `/embedding`** first; **Transformers.js** fallback | MIT / Apache-2.0 | Yes | port-wholesale | Reuse the endpoint we ship; Transformers.js (ONNX MiniLM/bge) if we want embeddings off the LLM's critical path | +| Memory-tier skeleton | **LlamaIndex.TS Memory Blocks** | MIT | Yes, TS-native | port-components | Write-time fact-extraction + short-term -> long-term + read-optimized; the only mature MIT TS-native option | +| Consolidation loop | Mem0 (has a TS SDK) | Apache-2.0 | partial (TS) | port-components | The ADD/UPDATE/DELETE dedup-on-write loop so memory doesn't bloat | +| Hybrid ranking | Orama | Apache-2.0 | Yes, TS | port-components | BM25 + vector fusion - lexical recall matters for OCR'd names/filenames/errors | +| Entity dedup (deterministic) | talisman + fuzzball.js | MIT | Yes, TS | port-components | Phonetics, Jaro-Winkler, blocking - the deterministic side of entity resolution | +| Multi-hop resolve (technique) | HippoRAG Personalized PageRank | MIT (Python) | reimplement | inspiration | PPR over the entity graph for "the deck" -> project -> file, instead of flat top-k | +| Memory linking (technique) | A-MEM Zettelkasten | MIT (Python) | reimplement | inspiration | Atomic note + keywords + auto-link + "evolution" rewrite of neighbors | +| Commitment lifecycle (concept) | Zep/Graphiti bi-temporal facts | Apache-2.0 | concept | inspiration | valid-from/valid-to per fact; new facts invalidate old - the backbone the commitment tracker needs | +| Capture triggers (concept) | Screenpipe | **source-available now (flag)** | concept only | inspiration | Event-driven capture (app-switch/click/pause) + accessibility-first, OCR-fallback. Its current tree is off-limits; take the ideas | +| Commitment detection (technique) | Microsoft WSDM 2019 definition | paper (patented method - note IP) | reimplement | inspiration | "sender-obligated + specific + not-yet-complete" as the LLM extraction rubric; commitment language is domain-independent so a small local model generalizes (~0.75 F1 is the bar) | + +**License flags (study only, no code into our permissive pro tier):** Reor, Khoj, OpenRecall (**AGPL**); Screenpipe (**source-available/commercial** post-2026-06); Letta / Zep-platform (server / proprietary). + +**Bespoke:** the RESOLVE layer returning `{value, confidence}` for a slot (no library does retrieval + slot-value + calibrated confidence); the entity-resolution pipeline (assembled from primitives, not adopted); and above all **the commitment-gap reasoner** - detecting the *unmet* commitment by joining it against captured observations and entity timelines has no prior art because it's defined entirely over our data model. + +--- + +## 4. Routines: record-and-replay (programming-by-demonstration) + +Finding: **OpenAdapt is an almost-exact architectural twin of our routines rail** - MIT, local-first, the same loop (record -> compile to anchored self-healing trace -> zero model calls on healthy runs -> local model only to repair drift -> halt instead of guess -> verify against a system of record). It's Python, so this is a **port-design** (reimplement in TS), not a code lift. + +| Need | Port from | License | Verdict | What we take | +| --- | --- | --- | --- | --- | +| Recorder + replay spine | **OpenAdapt / openadapt-flow** | MIT | **port-design** | The compiled-step schema (template crop + OCR label + geometry + structural locator + **postconditions**), the resolution ladder, system-of-record verification (their data: screen-only verify accepted wrong effects 75% of the time -> 12.5% with a system-of-record oracle), halt-on-uncertainty, repair-as-reviewable-diff | +| Self-heal technique | OpenAdapt resolution ladder (+ Healenium DOM tree-similarity, SikuliX OpenCV+Tesseract) | MIT / Apache-2.0 / MIT | port-design | Resolve each step by trying anchors in strict order (structural tree -> local template -> global template -> OCR label -> landmark geometry -> optional local grounding model); healthy runs never leave rung 1; write successful lower-rung resolutions back as a diff | +| Browser recorder | Playwright codegen | Apache-2.0 | port-components | Native TS recorder + its locator-priority heuristic (role -> text/label -> testid -> CSS) as the browser-lane anchor order | +| Browser trace format | Chrome DevTools Recorder `steps[]` | Apache-2.0 | inspiration | Per-step *array of alternative selectors* - a standardized "multiple anchors per step" schema to align to | +| Browser variable slots | browser-use workflow-use | **AGPL (flag)** | inspiration-only | The typed variable-slot idea; do NOT vendor the code, especially into pro | +| Mobile format | Maestro YAML flows | Apache-2.0 | inspiration | Human-readable flow format for the plain-language review surface + resilient text/id/AX matching | +| macOS AX recorder ref | open-record-replay | MIT | inspiration | Clean `events.jsonl` + AX-diff schema, AX-tree-as-primary-anchor | +| Multi-anchor capture | record-and-replay-skill | MIT | port-components | Recording several selectors per action (testId -> role+name -> id -> text -> css) so replay degrades gracefully | + +**Bespoke:** memory-resolved variable slots (every project treats variables as literals or LLM-extracted or manual; binding a slot to a memory query at run time is ours); the plain-language review UI (reuse an existing viewer component, don't fork); the TS-native cross-substrate recorder/runtime (OpenAdapt is Python; we need the ladder across macOS AX, browser CDP, later mobile); the local-only postcondition oracle (verify via our memory/observation layer, not the screen). + +--- + +## 5. The rails (actuation) - desktop + mobile + +Finding: input is a solved permissive dependency; the browser rail is free via Electron's CDP; the accessibility-tree read is a build-our-own napi-rs (Rust) addon with head-starts; the semantic rail is bespoke OS glue. **Convergent insight:** every rail's agent-facing contract is the same - a serialized element list with stable IDs, act-by-ID (browser-use's numeric index = Playwright's `ref` = Agent-S's ACI = the vision model's box). Design ONE DeviceController vocabulary; the vision rail manufactures the same IDs from pixels when no tree exists. + +| Rail | Port from | License | Verdict | What we take | +| --- | --- | --- | --- | --- | +| Desktop spine | **`@ui-tars/sdk`** (UI-TARS-desktop) | Apache-2.0 | port-components | The GUIAgent loop + `Operator` interface + coordinate scaling, in Electron+TS, local-model-ready. **Swap its nut.js operator for `@nut-tree-fork`** | +| Spine design | Agent-S/S2 ACI + a11y/vision fusion; Anthropic computer-use tool-schema + coord-scaling | Apache-2.0 / MIT | port-design | Accessibility-tree + vision fusion (the reliability lever for a weak model); the action vocabulary + normalized-coordinate convention | +| Desktop input | **robotjs** (revived, prebuilds) or **`@nut-tree-fork/nut-js`** | MIT / Apache-2.0 | adopt-as-dependency | Synthetic mouse/keyboard + capture (+ template match on the fork). **Avoid official `@nut-tree/*` - paid EULA** | +| Input (longevity) | enigo via napi-rs | MIT | port-components | Self-owned Rust input layer if we build our own addon | +| Desktop a11y read | **napi-rs addon over `axuielement` (macOS) + `uiautomation` (Windows) crates**; **Terminator** (Windows) + MacosUseSDK head-starts | MIT / Apache-2.0 | port-components / bespoke | No pure-Node lib reads both trees; FlaUI (.NET) / pywinauto (Python) are API references only | +| Browser rail | **nanobrowser** `dom/` module + overlay (starting code) + **browser-use** CDP snapshot/AX-merge/numeric-index (algorithm) + **Stagehand** act/observe/extract + Zod (API) | Apache-2.0 / MIT / MIT | port-components | All over Electron `webContents.debugger` (raw CDP - no Playwright dependency needed) | +| Mobile substrate | **Appium via WebdriverIO** | Apache-2.0 / MIT | adopt-as-dependency | One W3C protocol over iOS (WDA/XCTest) + Android (UiAutomator2), TS client, local, model-independent | +| Android host-free | DroidRun AccessibilityService "Portal" | MIT | port-components | On-device a11y-tree read + gesture dispatch with no host attached | +| Mobile seam | minitap/mobile-use | Apache-2.0 | port-components | Provider-agnostic model layer + multi-transport (ADB/idb/Appium) behind one interface | + +**Not viable:** official nut.js (paid EULA), Open Interpreter OS mode (AGPL + abandoned), Skyvern (AGPL, browser-only), Sonic (AGPL), c/ua (Python + VM-first). + +**Bespoke:** the semantic rail entirely (AppleScript/JXA, App Intents/Shortcuts, Microsoft Graph, deep links, Android intents - OS SDK glue behind the interface); the unified **DeviceController + rail-selection/fallback policy** (semantic -> browser -> accessibility -> vision - no prior art has all four behind one interface); the macOS-AX + Windows-UIA napi addon; **iOS on-device actuation** (genuinely needs a Mac-signed WDA/XCTest helper reached over USB - a permanent Apple constraint, plan the product around it). + +--- + +## 6. Grounding vision models (the vision rail's model) + +Finding: llama.cpp multimodal is real but base-gated (Qwen2-VL / Qwen2.5-VL / Qwen3-VL / InternVL / SmolVLM / Gemma 3 / Pixtral). A grounder is GGUF-runnable iff its base is one of these AND someone converted it. + +| Use | Model | Weights license | GGUF today? | Verdict | +| --- | --- | --- | --- | --- | +| **Desktop default** | **UI-TARS-1.5-7B** (Qwen2.5-VL base) | **Apache-2.0** | **Yes, published + mainline** | **adopt-as-model** - the only turnkey pick, no conversion work; ScreenSpot-V2 ~94% | +| Desktop 2nd | Holo1.5-7B (Qwen2.5-VL) | Apache-2.0 (7B only) | convertible | adopt-with-conversion - strong on ScreenSpot-Pro; avoid the 72B (research license) | +| **Mobile best** | **GUI-Owl-1.5-8B/4B** (Qwen3-VL) | **MIT** | needs one-time conversion | adopt-with-conversion - best open mobile grounding, multi-platform; OSWorld-Verified 52.3, AndroidWorld 69.0 | +| Mobile zero-conversion | Qwen3-VL-8B-Instruct | Apache-2.0 | Yes, official GGUF | adopt-as-model - ship day one, prompt/finetune for grounding; also the natural finetune target if we train our own | +| Pure-vision fallback (set-of-marks for a non-grounding LLM) | OmniParser **v3** detector (YOLOv9) + Florence-2 captioner | **MIT** (v3) | ONNX (not llama.cpp) | adopt-components - lets our bundled gemma click via labeled boxes. **Avoid v1/v2 icon_detect (AGPL YOLOv8)** | + +**Avoid (license or no GGUF path):** Qwen2.5-VL 3B/72B (research), Holo 72B (research), CogAgent (GLM-4V, non-commercial, no GGUF), Ferret-UI (Apple, non-commercial), SeeClick (Qwen-VL research), Aria-UI (custom MoE, no GGUF), OS-Atlas-4B (InternVL2 base, no safe GGUF), the closed UI-TARS-1.5 flagship. + +**Bespoke:** the GGUF + mmproj conversion + a ScreenSpot re-eval after quantization for any grounder beyond the turnkey UI-TARS-1.5-7B / Qwen3-VL (routine, but ours to own). + +--- + +## 7. The whole system, at a glance + +**Port these (the plumbing):** + +- Queue/state: **XState** + **sqliteq/goqite** + **plainjob** + **cockatiel** + **LangGraph interrupt contract**, spec'd from **DBOS + Morling**. +- Brain: **llama.cpp GBNF** (shipped) + **SAP (BAML)** + **Instructor retry** + **MCP TS SDK** + router seam from **Mastra/VoltAgent**. +- Memory: **sqlite-vec** + **LlamaIndex.TS memory blocks** + **Mem0 loop** + **Orama** hybrid ranking; techniques from **HippoRAG / A-MEM / Graphiti**. +- Routines: **OpenAdapt** design (resolution ladder + postconditions + self-heal), **Playwright/DevTools** for the browser lane. +- Rails: **@ui-tars/sdk** + **robotjs/nut-fork** + **nanobrowser/browser-use/Stagehand** over Electron CDP + **Appium/WebdriverIO** + **DroidRun Portal**; a napi-rs a11y addon over **axuielement/uiautomation** with **Terminator** head-start. +- Models: **UI-TARS-1.5-7B** (desktop), **GUI-Owl-1.5 / Qwen3-VL-8B** (mobile), **OmniParser v3** (fallback). + +**Build these (the product - bespoke by nature):** + +1. The **Action contract + durable pipeline** (queue -> FSM -> gate -> execute -> verify glue). +2. **Effect-verification** per connector (zero prior art anywhere) - lands in the R1 spine (the machine's verifying state + per-handler verify); R4's router only escalates through it, never rebuilds it. +3. The **resolve layer** returning `{value, confidence}`. +4. The **commitment-gap reasoner** (join a commitment against Replay observations). +5. The unified **DeviceController + rail-selection/fallback** policy. +6. The **macOS-AX + Windows-UIA napi-rs addon**. +7. The **approval-gate UX + privacy boundary** (nothing leaves the device). +8. **iOS on-device actuation** (Mac-signed WDA constraint). + +None of the bespoke items is NIH - each is bespoke because no upstream targets a single-process, offline, on-device app wired to a personal screen-memory. That is exactly the product. + +## 8. License avoid-list (carry forward) + +- **AGPL** (no code into the permissive pro tier): browser-use workflow-use, Skyvern, Open Interpreter OS mode, Windmill, LittleHorse, Reor, Khoj, OpenRecall, Sonic, OmniParser v1/v2 icon_detect (YOLOv8). +- **Source-available / SSPL / BUSL** (avoid depending): Screenpipe (post-2026-06), Inngest server (SSPL), Restate runtime (BUSL-1.1). +- **Paid EULA:** official `@nut-tree/*` nut.js (use `@nut-tree-fork`). +- **Non-commercial model weights** (do not bundle): Qwen2.5-VL 3B/72B, Holo 72B, CogAgent, Ferret-UI, SeeClick, the UI-TARS-1.5 flagship, xLAM/Hammer (verify per checkpoint). +- **Mixed/enterprise:** Mastra (use Apache-2.0 core only), Zep platform, Letta. + +Everything in the "port" column is MIT / Apache-2.0 / BSD. Verify each license at the point of adoption; a couple ask for attribution (minitap/mobile-use). diff --git a/docs/R1_CHECKLIST.md b/docs/R1_CHECKLIST.md new file mode 100644 index 00000000..93871040 --- /dev/null +++ b/docs/R1_CHECKLIST.md @@ -0,0 +1,96 @@ +# R1 checklist - chat actions on the durable spine (Days 1 - 4) + +Execution checklist for R1 of `COMPUTER_USE_PLAN.md` (the build doc). The plan stays the source of truth for schedule and scope; this file only tracks R1's execution. Tick a box when its unit is landed green. + +**Rules for every box (from CLAUDE.md):** +- One box = one commit-sized unit. Land it as soon as it is green (`npx tsc --noEmit -p tsconfig.node.json && npx tsc --noEmit -p tsconfig.web.json && npm test`), then move on. Spine work commits in `../shared`. +- Tests land in the same commit as the change - one case per branch, condition, and error path. Coverage ratchet holds. +- Port before writing: the sources per component are in `PORTING_MAP.md`. Verify the license at the point of adoption. +- Any UI string follows the brand copy rules. + +**Design references:** the Action record and state machine are `ASSISTANT_ARCHITECTURE.md` Section 3; the reliability stack is Section 4; the gate contract is decision 7 (payload binding). The spine is platform-free and lives in `../shared/packages/use` (`@offgrid/use`); OGAD consumes it via `file:../shared/packages/use`. + +--- + +## Day 1 - the spine package (`@offgrid/use`, in `../shared`) + +- [x] **1. Scaffold `packages/use`** in the shared repo: tsup + node --test (the shared-repo house pattern), mirroring the sync engine layout; consumed from OGAD as `file:../shared/packages/use`. + *Done when:* the package builds, an empty test runs, and OGAD's tsc still passes with the dependency declared. +- [x] **2. The Action contract** (`packages/use/src/action.ts`): Zod schema + types for `id, type, source, intent, args, payloadHash, risk, rail, idempotencyKey, attempts, verification, state, triggerAt`, audit refs. Closed `type` enum (message / email / calendar / reminder / open / lookup / file-share / web-task). + *Done when:* schema tests cover each risk class, each type, and reject malformed input (fail closed). +- [x] **3. The state machine** (`packages/use/src/machine.ts`, XState v5): `proposed -> rejected | scheduled | resolving -> awaiting_approval | ready -> executing -> verifying -> done | executing(retry) | needs_help`, exactly as the architecture doc draws it. Persist via `getPersistedSnapshot()`; rehydrate on start. + *Done when:* every transition has a test, plus a snapshot -> restore roundtrip test (the crash-resume guarantee). +- [x] **4. The durable queue** (`packages/use/src/queue.ts`): the goqite/sqliteq pattern - lease + visibility timeout + auto-extend + attempts + `UNIQUE(idempotencyKey)` dedup - behind a small `Storage` interface (the spine stays platform-free; hosts inject the DB). + *Done when:* tested against better-sqlite3 `:memory:` - lease expiry re-queues, a duplicate enqueue dedups, attempts increment, a held lease blocks a second worker. + +## Day 2 - the guarantees + +- [x] **5. Retry policy** (`packages/use/src/retry.ts`; pure policy - the machine owns the loop, so no promise-retry dep): retry-once-with-verify for reversible actions; single-attempt-behind-the-gate for irreversible ones (decision 8.1 lean). + *Done when:* both policies are tested, including that an irreversible action never fires twice even when verify errors. +- [x] **6. The gate seam** (`packages/use/src/gate.ts`): the interrupt -> approve/edit/reject -> resume contract as a host callback; `payloadHash` computed at propose time and re-checked at execute time so the approved payload is exactly what runs. + *Done when:* tests cover approve, reject, edit-then-approve (hash changes, re-gate), and a tampered payload refusing to execute. +- [x] **7. The DeviceController port + handler registry** (`packages/use/src/device.ts`, `registry.ts`): `execute(action)` port; each action handler declares its rail, risk default, and how it verifies (read-back / status / none-fuzzy). Every attempt records the rail it ran on (the Action record is the effect journal), and escalation across rails is a re-fire governed by box 5's policy - a non-retryable action never escalates. + *Done when:* a fake DeviceController proves the seam - registering a second fake handler needs zero caller changes (the DSP test), and routing picks by declared rail. +- [x] **8. The engine facade + worker** (`packages/use/src/engine.ts`): `propose()` validates and enqueues; a worker drains the queue through machine -> gate -> execute -> verify. + *Done when:* the fake-device suite is green end to end: a routed action, the gate flow, a verify-retry scenario, crash-resume (kill mid-execute, rehydrate, no double-fire thanks to the idempotency key), exactly-once under a duplicate enqueue. **This is the engine checkpoint.** + +## Day 3 - wire into the app (macOS end to end) + +- [x] **9. The storage adapter in OGAD** (`src/main/actions/use-driver.ts` + `src/main/__tests__/use-storage.integration.dbtest.ts`): the queue/state tables live in the app's existing better-sqlite3 DB (one DB is the SSOT), with a migration. + *Done when:* an integration test runs the real engine against a temp app DB (no mocks at the DB seam). +- [x] **10. The semantic rail adapter** (`src/main/actions/semantic-rail.ts`): wrap the existing `runNativeAction` helper behind the DeviceController port; map the Action types to the helper's verbs (calendar, reminders, contacts, messages, mail, open_url). + *Done when:* each mapped type has a test through an injected helper boundary; unknown types are refused, not guessed. +- [x] **11. The gate host**: wire the existing `actions:proposeApproval` seam as the engine's gate callback; the approval card shows the resolved values from the bound payload. + *Done when:* an integration test proves approve runs exactly the approved payload and reject lands the Action in `rejected`. +- [x] **12. Emission hardening** (`src/main/actions/emit.ts`): the action tool's schema goes to llama-server as grammar-constrained `response_format`; a TS SAP coercer (ported from BAML's schema-aligned parsing, keyed to the Action schema) repairs near-misses; bounded Zod validate-and-retry feeds the error back. + *Done when:* coercion tests per branch (markdown fence, trailing prose, unquoted keys, missing optional), and a test that an unrepairable emission is rejected, never guessed. +- [x] **13. Chat tool integration**: mutations from the chat tool loop enqueue durable Actions through the engine; pure reads stay inline (decision 7.5). Existing native-tool behavior is preserved. + *Done when:* the existing native-action tests still pass, plus new tests that a mutation goes through the queue and gate while a read does not. +- [x] **14. Verification per handler**: calendar and reminders verify by read-back (list after create); messages and mail declare fuzzy -> single-attempt; open_url verifies by launch result. + *Done when:* each handler's declared verification has a test, including a failed read-back triggering the retry policy correctly. +- [x] **15. macOS checkpoint evidence** (free-build engine path; the approval-card capture lands with the pro migration): on a seeded demo profile (`npm run demo` seeding rules), a chat ask ("remind me to send the deck at 6pm") produces gate -> execute -> verified -> confirmation. Capture screenshots into `e2e/screenshots/`. + *Done when:* the flow runs clean and the screenshots show the approval card and the verified confirmation (validate the images before counting this done). + +## Day 4 - Windows + release + +- [x] **16. Windows toolchain** (pre-existing on main - build-win job, fetch-win-binaries.ps1 with llama-server.exe pinned to the mac engine ref, NSIS + auto-update, optional signing secrets; our delta: windows-build.yml gained the shared checkout, both workflows now build @offgrid/use, and run 31779453356 built this branch green with a 414MB installer artifact. Remaining as release items: the signing cert, and the model-load smoke on a real Windows machine per WINDOWS_TEST_PLAN.md) (start this in parallel as early as Day 1 - it is the schedule floor and has CI latency): electron-builder Windows target, code-signing, and the `llama-server` Windows engine build in `release.yml` with the same gates the mac build learned (deployment target / staged deps / no foreign paths, adapted to Windows). + *Done when:* CI produces a signed Windows build whose bundled engine loads a model. +- [x] **17. The Windows semantic rail** (`src/main/actions/semantic-rail-win.ts`), **local-first**: mail + calendar via local Outlook automation (COM / PowerShell) where Outlook exists - a local write that syncs later, matching the mac rail - with Microsoft Graph as the fallback for setups without local Outlook (online-only, labeled honestly, user's own sign-in); open via the Windows shell. iMessage is macOS-only in R1 (documented tier difference). + *Done when:* handler tests through an injected Graph boundary; the registry proves macOS and Windows rails swap with zero caller changes. +- [ ] **18. E2E + evidence**: a Playwright spec driving chat ask -> approval card -> done state on a fresh temp profile (`OFFGRID_PRO=0`, synthetic seed only); screenshots per surface, a short video of the golden path. + *Done when:* `npm run test:e2e` includes the new spec and passes; evidence attached to the PR per the repo's PR rules. +- [ ] **18b. Release UX notes**: Tools defaults ON (fresh installs) with native actions under the Tools category - verify in the e2e that a fresh profile can act without touching any toggle. Flag to the lead: the free-build inline-confirm question for mutate/irreversible actions (open-core line), and the R2 router retiring the per-turn toggle. +- [ ] **19. Ship it**: version bump, release via CI, checkpoint sign-off against the plan ("on both macOS and Windows, a chat ask calls the action tool and the action runs gated and verified"). Update `COMPUTER_USE_PLAN.md` if any date moved. + *Done when:* the release is out and the plan reflects reality. + +--- + +## Field verdicts from the R1 pro-path smoke test (drive R2's Approval UX v2) + +- Approving a card gives no completion feedback - the chat message stays "pending" + and nothing reports the run. The engine path reports verified outcomes; the + legacy pro path is the old system. Fixed by the pro migration (approve resolves + the engine gate) in R2-B. +- Reversible simple actions (a reminder) should not gate at all: R2-B ships the + risk-tiered policy (reversible mutations auto-run + verified confirmation + + Undo; sends keep the gate). +- Chat-originated approvals belong INLINE in the conversation; the Actions screen + is the queue for unattended actions + audit. + +## Windows follow-ups (fast-follow, recorded during box 17) + +- Windows chat-tool exposure: registerNativeActionTools stays darwin-gated; enabling a + filtered spec subset on win32 (calendar/reminders/mail/open via the engine path) + needs per-platform specs and a win inline runner for reads. +- Outlook read-back verifiers: calendar/reminder verification still speaks the mac + helper's list verbs; on Windows read_back reports unverifiable (retry policy treats + it honestly) until Outlook COM list scripts land. +- Graph OAuth wiring: the port + fallback logic are boundary-tested; production + passes no Graph port until sign-in lands. + +## Watch-list (honest risks inside R1) + +- **Box 16 is the long pole.** Windows CI signing + the engine build is net-new infra with slow feedback loops; kick it off on Day 1 and let it bake while the spine lands. +- **Box 12's SAP coercer is new surface** - err toward more coercion-branch tests, not fewer; every repair rule gets a regression case. +- **Boxes 9 - 11 touch the running app** - main-process changes need an app restart; do not over-restart during capture hours. +- If a box slips, the plan's rule applies: scope trims at the tail (Windows rail detail, evidence polish), never the released core. diff --git a/e2e/app250-chat-action-engine.spec.ts b/e2e/app250-chat-action-engine.spec.ts new file mode 100644 index 00000000..9cfbce96 --- /dev/null +++ b/e2e/app250-chat-action-engine.spec.ts @@ -0,0 +1,139 @@ +/** + * APP-250 — the R1 golden path: a chat ask becomes a durable, verified action. + * + * The rendered app, tool loop, tool-call parsing, the @offgrid/use engine + * (queue, gate, semantic rail, read-back verification), IPC, and MemoryChat + * are production code. Two fakes stand at the true boundaries: a scripted + * llama-server (emits the tool call as text, the way small local models do) + * and a scripted actions helper (records creates, answers list read-backs). + * + * Proves, on a fresh profile with Tools enabled through the real composer + * menu (default-off until R2's per-turn router; see checklist 18b): + * chat ask -> tool call -> durable Action -> semantic rail create -> + * read-back verify -> confirmed in chat. The helper log pins the order: + * exactly one create, then a list (the read-back actually ran). + */ +import { expect, test, type ElectronApplication, type Page } from '@playwright/test' +import fs from 'node:fs' +import os from 'node:os' +import path from 'node:path' +import { completeOnboarding } from './helpers/onboarding' +import { launchOffGrid, targetIsPackaged } from './helpers/launch' + +let app: ElectronApplication | null = null +let page: Page +let profileDir: string +let helperLog: string + +function stageWorld(): void { + // The model boundary: a stub gguf + the scripted server as llama-server. + const modelsDir = path.join(profileDir, 'models') + const llamaDir = path.join(profileDir, 'bin', 'llama') + fs.mkdirSync(modelsDir, { recursive: true }) + fs.mkdirSync(llamaDir, { recursive: true }) + const gguf = Buffer.alloc(2_048) + gguf.write('GGUF') + fs.writeFileSync(path.join(modelsDir, 'app250-local.gguf'), gguf) + fs.writeFileSync( + path.join(modelsDir, 'active-model.json'), + JSON.stringify({ id: 'app250-local-model', primary: 'app250-local.gguf', mmproj: null }) + ) + fs.copyFileSync( + path.join(process.cwd(), 'e2e', 'fixtures', 'app250-actions-llama-server.mjs'), + path.join(llamaDir, 'llama-server') + ) + fs.chmodSync(path.join(llamaDir, 'llama-server'), 0o755) + + // The OS boundary: the scripted helper where dev resolution looks first + // (cwd/scripts/actions-helper/actions-helper - the spec launches the app + // with cwd pointed at the profile dir). + const helperDir = path.join(profileDir, 'scripts', 'actions-helper') + fs.mkdirSync(helperDir, { recursive: true }) + fs.copyFileSync( + path.join(process.cwd(), 'e2e', 'fixtures', 'app250-actions-helper.mjs'), + path.join(helperDir, 'actions-helper') + ) + fs.chmodSync(path.join(helperDir, 'actions-helper'), 0o755) +} + +const helperCalls = (): Array<{ command: string; args: Record }> => { + if (!fs.existsSync(helperLog)) { + return [] + } + return fs + .readFileSync(helperLog, 'utf8') + .split('\n') + .filter(Boolean) + .map((line) => JSON.parse(line)) +} + +test.beforeEach(async () => { + test.skip(targetIsPackaged(), 'dev-target journey: the packaged app resolves its helper from Resources') + profileDir = fs.mkdtempSync(path.join(os.tmpdir(), 'offgrid-app250-')) + helperLog = path.join(profileDir, 'helper-log.jsonl') + stageWorld() + app = await launchOffGrid({ + cwd: profileDir, + env: { + ...process.env, + OFFGRID_USER_DATA: profileDir, + OFFGRID_BIN_DIR: path.join(profileDir, 'bin'), + OFFGRID_APP250_HELPER_LOG: helperLog, + OFFGRID_PRO: '0', + NODE_ENV: 'production' + } + }) + page = await app.firstWindow() + await page.waitForLoadState('domcontentloaded') + await completeOnboarding(page) +}) + +test.afterEach(async () => { + const running = app + app = null + if (running) { + await running.close() + } + fs.rmSync(profileDir, { recursive: true, force: true }) +}) + +test('a chat ask becomes a created, read-back-verified reminder', async () => { + await page.getByRole('button', { name: 'Chat', exact: true }).click() + const composer = page.getByPlaceholder(/ask anything/i) + await expect(composer).toBeVisible() + const captureDismiss = page.getByRole('button', { name: 'Dismiss', exact: true }) + if (await captureDismiss.isVisible().catch(() => false)) { + await captureDismiss.click() + } + + // Enable Tools the way a user does: the composer's + menu. + await page.getByRole('button', { name: 'Composer options' }).click() + await page.getByRole('menuitem', { name: /^Tools/ }).click() + await page.keyboard.press('Escape') + + await composer.fill('remind me to send the deck at 6pm today') + await composer.press('Enter') + + // The model's confirmation only streams on the SECOND turn - after the + // tool ran through the engine and reported its verified outcome. + // .last(): the conversation rail previews the same text; the transcript + // copy is the one that matters. + await expect(page.getByText('Done - the reminder is set for 6pm today.').last()).toBeVisible({ + timeout: 90_000 + }) + + // The tool activity row shows the engine's verified outcome, not a guess. + await expect(page.getByText('reminders_create → Created the reminder.')).toBeVisible() + + // The helper log pins the guarantee: exactly one create, and at least one + // list AFTER it - the read-back verification actually observed the world. + const calls = helperCalls() + const creates = calls.filter((c) => c.command === 'reminders.create') + expect(creates).toHaveLength(1) + expect(creates[0]?.args.title).toBe('Send the deck') + const createIndex = calls.findIndex((c) => c.command === 'reminders.create') + const listAfter = calls.slice(createIndex + 1).some((c) => c.command === 'reminders.list') + expect(listAfter).toBe(true) + + await page.screenshot({ path: 'e2e/screenshots/r1-chat-action-verified.png' }) +}) diff --git a/e2e/devices-sync.spec.ts b/e2e/devices-sync.spec.ts index 491d0803..a2de5808 100644 --- a/e2e/devices-sync.spec.ts +++ b/e2e/devices-sync.spec.ts @@ -31,10 +31,17 @@ import { type PendingMembershipRevocation } from '@offgrid/sync' import { NodeTcpTransport } from '@offgrid/sync/node' -import { createKnowledgeDocumentSource } from '../pro/main/sync/knowledge-document-transfer' import type { KnowledgeDocumentSnapshot } from '../src/main/sync-knowledge-document' const PRO_PRESENT = fs.existsSync(path.resolve('pro/package.json')) +// Pro implementation modules load lazily behind PRO_PRESENT: a static import +// fails spec COLLECTION in a core-only checkout, before the guard can skip. +const knowledgeDocumentTransfer = PRO_PRESENT + ? // eslint-disable-next-line @typescript-eslint/no-require-imports + (require('../pro/main/sync/knowledge-document-transfer') as { + createKnowledgeDocumentSource: (...args: never[]) => unknown + }) + : null const SYNCED_PROJECT_ID = '22222222-2222-4222-8222-222222222222' const SYNCED_CONVERSATION_ID = '33333333-3333-4333-8333-333333333333' const SYNCED_MESSAGE_ID = '44444444-4444-4444-8444-444444444444' @@ -586,7 +593,7 @@ test.describe('Devices surface — pro tier', () => { } await syntheticFiles.sendFile( desktop.localDevice.id, - createKnowledgeDocumentSource(knowledgeDocument) + knowledgeDocumentTransfer!.createKnowledgeDocumentSource(knowledgeDocument as never) ) const knowledgeOp = syntheticLog.record( 'knowledge_document', diff --git a/e2e/fixtures/app250-actions-helper.mjs b/e2e/fixtures/app250-actions-helper.mjs new file mode 100755 index 00000000..8284767d --- /dev/null +++ b/e2e/fixtures/app250-actions-helper.mjs @@ -0,0 +1,38 @@ +#!/usr/bin/env node + +// APP-250's OS boundary: a scripted actions helper. Records every command it +// receives and answers reminders.list with what actually landed, so the +// engine's read-back verification runs for real against this fake world - +// and the log proves create-then-list ordering. + +import fs from 'node:fs' + +const logFile = process.env.OFFGRID_APP250_HELPER_LOG +const raw = process.argv[2] ?? '{}' +const cmd = JSON.parse(raw) + +const record = (entry) => { + if (logFile) { + fs.appendFileSync(logFile, `${JSON.stringify(entry)}\n`) + } +} + +const reply = (payload) => { + process.stdout.write(`${JSON.stringify(payload)}\n`) + process.exit(0) +} + +record({ command: cmd.command, args: cmd.args ?? {} }) + +if (cmd.command === 'reminders.create') { + reply({ ok: true, result: { id: `e2e-${Date.now()}` } }) +} +if (cmd.command === 'reminders.list') { + const lines = logFile && fs.existsSync(logFile) ? fs.readFileSync(logFile, 'utf8').split('\n').filter(Boolean) : [] + const reminders = lines + .map((line) => JSON.parse(line)) + .filter((entry) => entry.command === 'reminders.create') + .map((entry) => ({ id: 'e2e', title: String(entry.args.title ?? '') })) + reply({ ok: true, result: { reminders } }) +} +reply({ ok: true, result: {} }) diff --git a/e2e/fixtures/app250-actions-llama-server.mjs b/e2e/fixtures/app250-actions-llama-server.mjs new file mode 100755 index 00000000..b65cd213 --- /dev/null +++ b/e2e/fixtures/app250-actions-llama-server.mjs @@ -0,0 +1,62 @@ +#!/usr/bin/env node + +// APP-250's model boundary: a scripted llama-server. The production app still +// owns model discovery, the tool loop, tool-call parsing, the @offgrid/use +// engine, the semantic rail, read-back verification, IPC, and rendering. +// Turn 1 (an agentic turn carrying the reminders_create schema): emit the +// tool call AS TEXT, exactly how small local models do. Turn 2 (the request +// carries the tool's result): confirm in plain text. + +import http from 'node:http' + +const args = process.argv.slice(2) +const portFlag = Math.max(args.indexOf('--port'), args.indexOf('-p')) +const port = portFlag >= 0 ? Number(args[portFlag + 1]) : 8439 + +const delta = (content, finishReason = null) => + `data: ${JSON.stringify({ choices: [{ delta: content ? { content } : {}, finish_reason: finishReason }] })}\n\n` + +const TOOL_CALL = + '{"name":"reminders_create","arguments":{"title":"Send the deck","due":"2026-08-14T18:00:00"}}' +const CONFIRMATION = 'Done - the reminder is set for 6pm today.' + +const server = http.createServer((request, response) => { + if (request.method === 'GET' && request.url === '/health') { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ status: 'ok' })) + return + } + if (request.method === 'GET' && request.url === '/v1/models') { + response.writeHead(200, { 'Content-Type': 'application/json' }) + response.end(JSON.stringify({ data: [{ id: 'app250-local-model' }] })) + return + } + if (request.method !== 'POST' || !String(request.url).includes('/chat/completions')) { + response.writeHead(404) + response.end() + return + } + let body = '' + request.on('data', (chunk) => { + body += chunk + }) + request.on('end', () => { + response.writeHead(200, { + 'Content-Type': 'text/event-stream', + 'Cache-Control': 'no-cache', + Connection: 'keep-alive' + }) + const isAgenticFirstTurn = body.includes('reminders_create') && !body.includes('Created the reminder.') + const text = isAgenticFirstTurn ? TOOL_CALL : CONFIRMATION + for (const piece of text.match(/.{1,24}/gs) ?? []) { + response.write(delta(piece)) + } + response.write(delta(null, 'stop')) + response.write('data: [DONE]\n\n') + response.end() + }) +}) + +server.listen(port, '127.0.0.1', () => { + console.log(`app250 fake llama-server listening on ${port}`) +}) diff --git a/e2e/helpers/launch.ts b/e2e/helpers/launch.ts index 67e157cf..0edb1859 100644 --- a/e2e/helpers/launch.ts +++ b/e2e/helpers/launch.ts @@ -114,10 +114,21 @@ export interface LaunchOptions { env?: Record /** Extra Chromium/Electron flags (e.g. fake media devices). Applied to both targets. */ extraArgs?: string[] + /** Working directory for the DEV target's app process. The native actions + * helper resolves dev candidates relative to cwd, so a spec can plant a + * fake helper in a temp dir and point the app at it. Ignored when + * packaged (resolution uses resourcesPath there). */ + cwd?: string } export const launchOffGrid = async (options: LaunchOptions = {}): Promise => { const env = withCoverage({ ...process.env, ...options.env } as Record) + // A runner that itself lives inside Electron (VS Code tasks, agent + // sandboxes) exports ELECTRON_RUN_AS_NODE=1; inherited, it turns the + // launched app into plain Node - electron.app is undefined and every spec + // dies with "Process failed to launch". The app under test must never + // run as node. + delete env.ELECTRON_RUN_AS_NODE const extraArgs = options.extraArgs ?? [] if (targetIsPackaged()) { @@ -134,5 +145,6 @@ export const launchOffGrid = async (options: LaunchOptions = {}): Promise ({ - app: { getPath: vi.fn(() => fakeUserData) } -})) - -async function freshModule(): Promise { - vi.resetModules() - return import('../device-fingerprint') -} - -beforeEach(() => { - fakeUserData = fs.mkdtempSync(path.join(os.tmpdir(), 'fp-test-')) -}) - -afterEach(() => { - fs.rmSync(fakeUserData, { recursive: true, force: true }) -}) - -describe('getPlatformTag', () => { - const realPlatform = process.platform - - function setPlatform(p: NodeJS.Platform): void { - Object.defineProperty(process, 'platform', { value: p, configurable: true }) - } - - afterEach(() => { - Object.defineProperty(process, 'platform', { value: realPlatform, configurable: true }) - }) - - it('returns "macos" on darwin', async () => { - setPlatform('darwin') - const { getPlatformTag } = await freshModule() - expect(getPlatformTag()).toBe('macos') - }) - - it('returns "windows" on win32', async () => { - setPlatform('win32') - const { getPlatformTag } = await freshModule() - expect(getPlatformTag()).toBe('windows') - }) - - it('returns "linux" on linux', async () => { - setPlatform('linux') - const { getPlatformTag } = await freshModule() - expect(getPlatformTag()).toBe('linux') - }) - - it('passes an unknown platform through unchanged', async () => { - setPlatform('freebsd' as NodeJS.Platform) - const { getPlatformTag } = await freshModule() - expect(getPlatformTag()).toBe('freebsd') - }) -}) - -describe('getDeviceFingerprint', () => { - it('generates a 32-hex-char fingerprint (16 random bytes)', async () => { - const { getDeviceFingerprint } = await freshModule() - const fp = await getDeviceFingerprint() - expect(fp).toMatch(/^[0-9a-f]{32}$/) - }) - - it('is stable across two calls within one process (in-memory cache)', async () => { - const { getDeviceFingerprint } = await freshModule() - const a = await getDeviceFingerprint() - const b = await getDeviceFingerprint() - expect(b).toBe(a) - }) - - it('persists to userData so a reinstall/reboot reuses the same id', async () => { - const first = await freshModule() - const fp1 = await first.getDeviceFingerprint() - - // Simulate a fresh process: reset the module (clears the in-memory cache) but - // keep the same userData dir. The persisted file must be read back verbatim. - const second = await freshModule() - const fp2 = await second.getDeviceFingerprint() - expect(fp2).toBe(fp1) - - const onDisk = fs.readFileSync(path.join(fakeUserData, 'device-fingerprint'), 'utf8').trim() - expect(onDisk).toBe(fp1) - }) - - it('regenerates a different fingerprint for a different install (new userData dir)', async () => { - const first = await freshModule() - const fp1 = await first.getDeviceFingerprint() - - // New "install": different userData dir + cleared module cache. - fakeUserData = fs.mkdtempSync(path.join(os.tmpdir(), 'fp-test-2-')) - const second = await freshModule() - const fp2 = await second.getDeviceFingerprint() - expect(fp2).not.toBe(fp1) - fs.rmSync(fakeUserData, { recursive: true, force: true }) - }) - - it('ignores an empty persisted file and generates a fresh id', async () => { - fs.writeFileSync(path.join(fakeUserData, 'device-fingerprint'), ' ') - const { getDeviceFingerprint } = await freshModule() - const fp = await getDeviceFingerprint() - expect(fp).toMatch(/^[0-9a-f]{32}$/) - }) - - it('writes the fingerprint file with 0600 perms', async () => { - const { getDeviceFingerprint } = await freshModule() - await getDeviceFingerprint() - const mode = fs.statSync(path.join(fakeUserData, 'device-fingerprint')).mode & 0o777 - expect(mode).toBe(0o600) - }) -}) diff --git a/pro/main/licensing/__tests__/keygen-parse.test.ts b/pro/main/licensing/__tests__/keygen-parse.test.ts deleted file mode 100644 index 6835c083..00000000 --- a/pro/main/licensing/__tests__/keygen-parse.test.ts +++ /dev/null @@ -1,160 +0,0 @@ -/** - * Keygen JSON:API response parsers — the pure functions that turn Keygen's - * validate-key / machine-activate / list-machines wire bodies into our internal - * shapes. The fetch/transport layer is untested shell; these feed it real - * JSON:API fixture objects (no network) and assert the mapping + the branch that - * detects the device-cap (422 MACHINE_LIMIT_EXCEEDED). - */ -import { describe, it, expect } from 'vitest' -import { - toLicense, - parseValidateResult, - parseActivateResult, - parseMachines -} from '../keygen-client' - -describe('toLicense', () => { - it('maps a JSON:API license resource to our KeygenLicense', () => { - const data = { - id: 'lic-1', - attributes: { expiry: '2030-01-01T00:00:00Z', metadata: { plan: 'monthly' }, name: 'Ada' } - } - expect(toLicense(data)).toEqual({ - id: 'lic-1', - expiry: '2030-01-01T00:00:00Z', - metadata: { plan: 'monthly' }, - name: 'Ada' - }) - }) - - it('defaults a lifetime (null-expiry) license and missing fields', () => { - expect(toLicense({ id: 'lic-2' })).toEqual({ - id: 'lic-2', - expiry: null, - metadata: {}, - name: null - }) - }) - - it('returns null for missing data or a resource without an id', () => { - expect(toLicense(undefined)).toBeNull() - expect(toLicense(null)).toBeNull() - expect(toLicense({ attributes: {} })).toBeNull() - }) -}) - -describe('parseValidateResult', () => { - it('parses a VALID validate response (valid=true, license present)', () => { - const body = { - meta: { valid: true, code: 'VALID' }, - data: { id: 'lic-9', attributes: { expiry: null } } - } - const r = parseValidateResult(body) - expect(r.valid).toBe(true) - expect(r.code).toBe('VALID') - expect(r.license?.id).toBe('lic-9') - expect(r.license?.expiry).toBeNull() - }) - - it('parses an EXPIRED response (valid=false, code carried, license still present)', () => { - const body = { - meta: { valid: false, code: 'EXPIRED' }, - data: { id: 'lic-9', attributes: { expiry: '2020-01-01T00:00:00Z' } } - } - const r = parseValidateResult(body) - expect(r.valid).toBe(false) - expect(r.code).toBe('EXPIRED') - expect(r.license?.expiry).toBe('2020-01-01T00:00:00Z') - }) - - it('parses a NO_MACHINE (needs-activation) response with a license to reclaim', () => { - const body = { - meta: { valid: false, code: 'NO_MACHINE' }, - data: { id: 'lic-3', attributes: {} } - } - const r = parseValidateResult(body) - expect(r.code).toBe('NO_MACHINE') - expect(r.license?.id).toBe('lic-3') - }) - - it('falls back to UNKNOWN code, valid=false, null license on a malformed/empty body', () => { - expect(parseValidateResult({})).toEqual({ valid: false, code: 'UNKNOWN', license: null }) - expect(parseValidateResult(undefined)).toEqual({ valid: false, code: 'UNKNOWN', license: null }) - }) -}) - -describe('parseActivateResult', () => { - it('reports ok on a 201 Created', () => { - expect(parseActivateResult(201, {})).toEqual({ ok: true, limitReached: false }) - }) - - it('detects the device cap: 422 with an errors[].code containing LIMIT', () => { - const body = { errors: [{ title: 'Unprocessable', code: 'MACHINE_LIMIT_EXCEEDED' }] } - expect(parseActivateResult(422, body)).toEqual({ ok: false, limitReached: true }) - }) - - it('detects the device cap: 422 with a "machine limit" detail (case-insensitive)', () => { - const body = { errors: [{ detail: 'machine LIMIT has been exceeded for this license' }] } - expect(parseActivateResult(422, body)).toEqual({ ok: false, limitReached: true }) - }) - - it('a 422 that is NOT a limit error is a plain failure, not limitReached', () => { - const body = { errors: [{ code: 'FINGERPRINT_TAKEN', detail: 'already taken' }] } - expect(parseActivateResult(422, body)).toEqual({ ok: false, limitReached: false }) - }) - - it('a non-201 non-422 (e.g. 403) is a plain failure', () => { - expect(parseActivateResult(403, { errors: [{ code: 'FORBIDDEN' }] })).toEqual({ - ok: false, - limitReached: false - }) - }) - - it('handles a 422 with no errors array without throwing', () => { - expect(parseActivateResult(422, {})).toEqual({ ok: false, limitReached: false }) - }) -}) - -describe('parseMachines', () => { - it('maps a machines list, preferring lastHeartbeat for lastSeen', () => { - const body = { - data: [ - { - id: 'm1', - attributes: { - fingerprint: 'fp-1', - platform: 'macos', - name: 'Ada MBP', - lastHeartbeat: '2026-01-01T00:00:00Z', - created: '2025-01-01T00:00:00Z' - } - } - ] - } - // The raw machine, not a UI projection: the licensed-devices list does its own mapping, including - // the heartbeat-then-created fallback for last seen (license-service.listLicensedDevicesForUi). - expect(parseMachines(body)).toEqual([ - { - id: 'm1', - fingerprint: 'fp-1', - hostname: null, - platform: 'macos', - name: 'Ada MBP', - createdAt: '2025-01-01T00:00:00Z', - updatedAt: null, - lastActiveAt: '2026-01-01T00:00:00Z' - } - ]) - }) - - it('drops a machine with no fingerprint, because nothing can be matched to a device', () => { - const body = { data: [{ id: 'm2', attributes: { created: '2025-06-01T00:00:00Z' } }] } - expect(parseMachines(body)).toEqual([]) - }) - - it('returns [] for an empty or malformed body', () => { - expect(parseMachines({})).toEqual([]) - expect(parseMachines(undefined)).toEqual([]) - expect(parseMachines({ data: [] })).toEqual([]) - }) -}) diff --git a/pro/main/licensing/__tests__/keygen-validate.integration.test.ts b/pro/main/licensing/__tests__/keygen-validate.integration.test.ts deleted file mode 100644 index 7ea9ed0e..00000000 --- a/pro/main/licensing/__tests__/keygen-validate.integration.test.ts +++ /dev/null @@ -1,25 +0,0 @@ -import { afterEach, describe, expect, it, vi } from 'vitest' -import { validateKey } from '../keygen-client' - -describe('Keygen validation service boundary', () => { - afterEach(() => vi.unstubAllGlobals()) - - it('maps an unknown third-party validation code to UNKNOWN', async () => { - const fetchBoundary = vi.fn(async () => - Response.json({ - meta: { valid: false, code: 'FUTURE_KEYGEN_CODE' }, - data: { id: 'license-1', attributes: { expiry: null } } - }) - ) - vi.stubGlobal('fetch', fetchBoundary) - - const result = await validateKey('test-license-key', 'test-device-fingerprint') - - expect(fetchBoundary).toHaveBeenCalledOnce() - expect(result).toEqual({ - valid: false, - code: 'UNKNOWN', - license: { id: 'license-1', expiry: null, metadata: {}, name: null } - }) - }) -}) diff --git a/pro/main/licensing/__tests__/license-cache.test.ts b/pro/main/licensing/__tests__/license-cache.test.ts deleted file mode 100644 index 8e1b4161..00000000 --- a/pro/main/licensing/__tests__/license-cache.test.ts +++ /dev/null @@ -1,73 +0,0 @@ -import { describe, expect, it } from 'vitest' -import { decodeLicenseCache, encodeLicenseCache, type ProLicense } from '../license-cache' - -const LICENSE: ProLicense = { - isPro: true, - key: 'KEY', - licenseId: 'license-id', - expiry: null, - verifiedAt: 123 -} - -const plaintext = (): string => JSON.stringify({ enc: false, data: JSON.stringify(LICENSE) }) - -describe('license cache trust policy', () => { - it('rejects an unsigned plaintext entitlement in a packaged build', () => { - expect(() => - decodeLicenseCache(plaintext(), { - packaged: true, - decrypt: () => { - throw new Error('must not decrypt plaintext') - } - }) - ).toThrow('packaged builds reject plaintext license caches') - }) - - it('allows the development-only plaintext fallback outside a packaged build', () => { - expect( - decodeLicenseCache(plaintext(), { - packaged: false, - decrypt: () => { - throw new Error('must not decrypt plaintext') - } - }) - ).toEqual(LICENSE) - }) - - it('round-trips an encrypted cache without trusting the wrapper as entitlement', () => { - const wrapper = encodeLicenseCache(LICENSE, { - packaged: true, - encryptionAvailable: true, - encrypt: (value) => Buffer.from(`sealed:${value}`) - }) - expect(wrapper?.enc).toBe(true) - expect( - decodeLicenseCache(JSON.stringify(wrapper), { - packaged: true, - decrypt: (value) => value.toString().replace(/^sealed:/, '') - }) - ).toEqual(LICENSE) - }) - - it('refuses to persist packaged entitlement when OS encryption is unavailable', () => { - expect( - encodeLicenseCache(LICENSE, { - packaged: true, - encryptionAvailable: false, - encrypt: () => { - throw new Error('must not encrypt') - } - }) - ).toBeNull() - }) - - it('rejects malformed entitlement fields instead of coercing them', () => { - const malformed = JSON.stringify({ - enc: false, - data: JSON.stringify({ ...LICENSE, isPro: 'true' }) - }) - expect(() => decodeLicenseCache(malformed, { packaged: false, decrypt: () => '' })).toThrow( - 'license cache isPro is malformed' - ) - }) -}) diff --git a/pro/main/licensing/__tests__/license-logic.test.ts b/pro/main/licensing/__tests__/license-logic.test.ts deleted file mode 100644 index abbcd99b..00000000 --- a/pro/main/licensing/__tests__/license-logic.test.ts +++ /dev/null @@ -1,147 +0,0 @@ -/** - * Pro-gate licensing logic — the pure entitlement decisions that drive the whole - * app's pro gate (isProEntitled → isProActive) and the Settings status UI (toInfo). - * - * High blast radius: a wrong `isProActive` either locks out a paying user or hands - * Pro to a lapsed/revoked one. These exercise the real exported functions against - * hand-built license shapes — no Electron, no disk, no network. Time-relative cases - * are computed from Date.now() so they stay correct regardless of when they run. - * - * The revoked/needs-activation code lists are imported from the source (single - * source of truth) rather than re-hardcoded here. - */ -import { describe, it, expect } from 'vitest' -import { PRO_PURCHASE_URL } from '@offgrid/core/shared/product-links' -import { - isProActive, - toInfo, - REVOKED_CODES, - NEEDS_ACTIVATION, - PRO_PAY_PAGE_URL, - type ProLicense -} from '../license-service' - -const HOUR = 3600_000 - -function lic(over: Partial = {}): ProLicense { - return { isPro: true, key: 'K', licenseId: 'L', expiry: null, verifiedAt: 123, ...over } -} - -describe('isProActive', () => { - it('grants Pro for a lifetime key (isPro, null expiry)', () => { - expect(isProActive(lic({ expiry: null }))).toBe(true) - }) - - it('grants Pro for an active monthly key (expiry in the future)', () => { - const future = new Date(Date.now() + 24 * HOUR).toISOString() - expect(isProActive(lic({ expiry: future }))).toBe(true) - }) - - it('denies Pro for an expired monthly key (expiry in the past)', () => { - const past = new Date(Date.now() - HOUR).toISOString() - expect(isProActive(lic({ expiry: past }))).toBe(false) - }) - - it('denies Pro when the expiry is exactly now (<= boundary)', () => { - // isProActive uses `<= Date.now()`, so an instant that has just passed is denied. - const past = new Date(Date.now() - 1).toISOString() - expect(isProActive(lic({ expiry: past }))).toBe(false) - }) - - it('denies Pro when isPro is false even with a future expiry', () => { - const future = new Date(Date.now() + 24 * HOUR).toISOString() - expect(isProActive(lic({ isPro: false, expiry: future }))).toBe(false) - }) - - it('denies Pro when isPro is false and expiry is null', () => { - expect(isProActive(lic({ isPro: false, expiry: null }))).toBe(false) - }) - - it('denies Pro for the EMPTY-style license (no key, not pro)', () => { - const empty: ProLicense = { - isPro: false, - key: null, - licenseId: null, - expiry: null, - verifiedAt: 0 - } - expect(isProActive(empty)).toBe(false) - }) - - it('denies Pro for a revoked license the service marks isPro=false but keeps expiry', () => { - // Mirrors revalidatePro's REVOKED branch: isPro flipped false, stale future expiry left in place. - const future = new Date(Date.now() + 30 * 24 * HOUR).toISOString() - expect(isProActive(lic({ isPro: false, expiry: future }))).toBe(false) - }) - - it('denies Pro when the cached expiry is unparseable', () => { - expect(isProActive(lic({ expiry: 'not-a-date' }))).toBe(false) - }) - - it('denies a claimed entitlement without a key and license id', () => { - expect(isProActive(lic({ key: null }))).toBe(false) - expect(isProActive(lic({ licenseId: null }))).toBe(false) - }) -}) - -describe('toInfo', () => { - it('reports tier=lifetime for an active key with null expiry', () => { - expect(toInfo(lic({ expiry: null }))).toEqual({ - isPro: true, - tier: 'lifetime', - expiry: null, - verifiedAt: 123 - }) - }) - - it('reports tier=monthly for an active key with a future expiry', () => { - const future = new Date(Date.now() + 24 * HOUR).toISOString() - expect(toInfo(lic({ expiry: future }))).toEqual({ - isPro: true, - tier: 'monthly', - expiry: future, - verifiedAt: 123 - }) - }) - - it('reports isPro=false and tier=null for an expired key (still echoes the expiry)', () => { - const past = new Date(Date.now() - HOUR).toISOString() - expect(toInfo(lic({ expiry: past }))).toEqual({ - isPro: false, - tier: null, - expiry: past, - verifiedAt: 123 - }) - }) - - it('reports isPro=false and tier=null when not entitled', () => { - const info = toInfo(lic({ isPro: false })) - expect(info.isPro).toBe(false) - expect(info.tier).toBeNull() - }) - - it('carries verifiedAt through unchanged', () => { - expect(toInfo(lic({ verifiedAt: 987654 })).verifiedAt).toBe(987654) - }) -}) - -describe('validation-code classifiers (single source of truth)', () => { - it('REVOKED_CODES cover the lock-out states', () => { - expect(REVOKED_CODES).toEqual(['EXPIRED', 'SUSPENDED', 'BANNED', 'OVERDUE', 'NOT_FOUND']) - }) - - it('NEEDS_ACTIVATION cover the reclaim-slot states', () => { - expect(NEEDS_ACTIVATION).toEqual(['NO_MACHINE', 'NO_MACHINES', 'FINGERPRINT_SCOPE_MISMATCH']) - }) - - it('the two lists are disjoint — no code both revokes and reactivates', () => { - const overlap = REVOKED_CODES.filter((c) => NEEDS_ACTIVATION.includes(c)) - expect(overlap).toEqual([]) - }) -}) - -describe('purchase destination', () => { - it('uses the canonical shared Pro purchase URL', () => { - expect(PRO_PAY_PAGE_URL).toBe(PRO_PURCHASE_URL) - }) -}) diff --git a/pro/main/licensing/__tests__/license-seat-replacement.integration.test.ts b/pro/main/licensing/__tests__/license-seat-replacement.integration.test.ts deleted file mode 100644 index fadb6bd2..00000000 --- a/pro/main/licensing/__tests__/license-seat-replacement.integration.test.ts +++ /dev/null @@ -1,128 +0,0 @@ -/** - * License activation through the real service and Keygen client. Only the third-party HTTP - * boundary and Electron's OS storage boundary are replaced. - */ -import fs from 'node:fs' -import path from 'node:path' -import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest' - -const h = vi.hoisted(() => ({ - userData: `/tmp/offgrid-license-seat-${process.pid}-${process.env.VITEST_POOL_ID ?? '0'}`, - fingerprint: 'current-device-fingerprint' -})) - -vi.mock('electron', () => ({ - app: { - getPath: () => h.userData, - isPackaged: false - }, - safeStorage: { - isEncryptionAvailable: () => true, - encryptString: (value: string) => Buffer.from(value), - decryptString: (value: Buffer) => value.toString() - } -})) - -import { - activateProByKey, - getProLicenseInfo, - setDirectEntitlementActivationOwner -} from '../license-service' -import { installEntitlementActivationFake } from '../../__tests__/helpers/entitlementActivationFake' - -const machine = (id: string, fingerprint: string, lastSeen: string): Record => ({ - type: 'machines', - id, - attributes: { - fingerprint, - platform: 'macos', - name: id, - lastHeartbeat: lastSeen - } -}) - -beforeAll(() => { - fs.mkdirSync(h.userData, { recursive: true }) - fs.writeFileSync(path.join(h.userData, 'device-fingerprint'), h.fingerprint) -}) - -// Activation goes through the personal-mesh registry owner the sync layer registers in production. -// Without it every activation waits for an owner that never arrives and reports network_unavailable. -let activation: ReturnType - -beforeEach(() => { - activation = installEntitlementActivationFake(setDirectEntitlementActivationOwner) -}) - -afterEach(() => { - activation.stop() - vi.unstubAllGlobals() -}) - -afterAll(() => { - fs.rmSync(h.userData, { recursive: true, force: true }) -}) - -describe('Pro activation at the device limit', () => { - it('activates through the registry owner when the licence reports its machine cap', async () => { - const requests: Array<{ method: string; path: string }> = [] - const fetchBoundary = vi.fn(async (input: string | URL | Request, init?: RequestInit) => { - const url = new URL(input instanceof Request ? input.url : input.toString()) - const method = init?.method ?? (input instanceof Request ? input.method : 'GET') - requests.push({ method, path: url.pathname }) - - if (url.pathname.endsWith('/licenses/actions/validate-key')) { - return Response.json({ - meta: { valid: false, code: 'TOO_MANY_MACHINES' }, - data: { - type: 'licenses', - id: 'license-1', - attributes: { expiry: null, metadata: {}, name: 'Pro' } - } - }) - } - if (url.pathname.endsWith('/licenses/license-1/machines')) { - return Response.json({ - data: [ - machine('machine-current', h.fingerprint, '2020-01-01T00:00:00Z'), - machine('machine-oldest', 'oldest-device', '2024-01-01T00:00:00Z'), - machine('machine-newer-1', 'newer-1', '2025-01-01T00:00:00Z'), - machine('machine-newer-2', 'newer-2', '2025-02-01T00:00:00Z'), - machine('machine-newest', 'newest', '2025-03-01T00:00:00Z') - ] - }) - } - if (url.pathname.endsWith('/machines/machine-oldest') && method === 'DELETE') { - return new Response(null, { status: 204 }) - } - if (url.pathname.endsWith('/machines') && method === 'POST') { - return new Response(null, { status: 201 }) - } - return new Response(null, { status: 500 }) - }) - vi.stubGlobal('fetch', fetchBoundary) - - await expect(activateProByKey('license-key')).resolves.toEqual({ ok: true }) - - // A licence at its cap still activates - and which device gives up its seat is the personal-mesh - // registry's decision, not a DELETE this service issues. So the licence service asks Keygen one - // question, then hands the activation to the owner as a transaction. - expect( - requests.map(({ method, path: requestPath }) => ({ - method, - path: requestPath.replace(/^\/v1\/accounts\/[^/]+/, '') - })) - ).toEqual([{ method: 'POST', path: '/licenses/actions/validate-key' }]) - expect(activation.prepared).toEqual([ - { - key: 'license-key', - licenseId: 'license-1', - expiresAt: null, - fingerprint: h.fingerprint, - platform: expect.any(String) - } - ]) - expect(activation.committed).toHaveLength(1) - expect(getProLicenseInfo()).toMatchObject({ isPro: true, tier: 'lifetime' }) - }) -}) diff --git a/scripts/actions-helper/main.swift b/scripts/actions-helper/main.swift new file mode 100644 index 00000000..c4d66e2f --- /dev/null +++ b/scripts/actions-helper/main.swift @@ -0,0 +1,328 @@ +import Foundation +import EventKit +import Contacts +import AppKit + +// Off Grid AI Desktop - native actions helper (macOS), the backend of the computer-use +// semantic rail. One-shot CLI: reads a single JSON command argument, performs one +// scoped native action (EventKit today; Reminders / Contacts / Photos next), prints +// ONE compact JSON line to stdout, and exits 0. +// +// Handled errors are reported as {"ok":false,"error":...} inside that JSON, not via +// the exit code, so the Node invoker always reads the result from stdout and a +// permission denial is a normal result rather than a crash. Invoked as a child of the +// signed .app, the helper inherits the app's TCC identity, so the Info.plist usage +// strings (NSCalendarsFullAccessUsageDescription and friends) drive the OS prompts. + +func emit(_ object: [String: Any]) -> Never { + if let data = try? JSONSerialization.data(withJSONObject: object), + let json = String(data: data, encoding: .utf8) { + print(json) + } else { + print("{\"ok\":false,\"error\":\"failed to serialize response\"}") + } + exit(0) +} + +func fail(_ message: String) -> Never { emit(["ok": false, "error": message]) } +func ok(_ result: [String: Any]) -> Never { emit(["ok": true, "result": result]) } + +let iso = ISO8601DateFormatter() + +// Accept a full ISO 8601 string (with timezone) first, then fall back to the +// timezone-less local forms a model commonly emits (2026-08-13T15:00:00, +// 2026-08-13T15:00, 2026-08-13) interpreted in the user's local timezone. +func parseDate(_ value: Any?) -> Date? { + guard let raw = value as? String else { return nil } + if let date = iso.date(from: raw) { return date } + let formatter = DateFormatter() + formatter.locale = Locale(identifier: "en_US_POSIX") + formatter.timeZone = TimeZone.current + for pattern in ["yyyy-MM-dd'T'HH:mm:ss", "yyyy-MM-dd'T'HH:mm", "yyyy-MM-dd"] { + formatter.dateFormat = pattern + if let date = formatter.date(from: raw) { return date } + } + return nil +} + +// Request EventKit access synchronously. The completion handler runs off the calling +// thread, so block on it - this one-shot tool must have a decision before it can act. +func requestEventAccess(_ store: EKEventStore) -> (granted: Bool, error: String?) { + let semaphore = DispatchSemaphore(value: 0) + var granted = false + var errorMessage: String? + let handler: (Bool, Error?) -> Void = { allowed, err in + granted = allowed + if let err = err { errorMessage = err.localizedDescription } + semaphore.signal() + } + if #available(macOS 14.0, *) { + store.requestFullAccessToEvents(completion: handler) + } else { + store.requestAccess(to: .event, completion: handler) + } + semaphore.wait() + return (granted, errorMessage) +} + +func createEvent(_ args: [String: Any]) -> Never { + guard let title = args["title"] as? String, !title.isEmpty else { + fail("createEvent requires a non-empty title") + } + guard let start = parseDate(args["start"]) else { + fail("createEvent requires an ISO8601 start date") + } + let allDay = (args["allDay"] as? Bool) ?? false + let end = parseDate(args["end"]) ?? start.addingTimeInterval(3600) + + let store = EKEventStore() + let access = requestEventAccess(store) + if !access.granted { fail(access.error ?? "calendar access was not granted") } + + let event = EKEvent(eventStore: store) + event.title = title + event.startDate = start + event.endDate = end + event.isAllDay = allDay + if let notes = args["notes"] as? String { event.notes = notes } + if let calName = args["calendar"] as? String, + let cal = store.calendars(for: .event).first(where: { $0.title == calName }) { + event.calendar = cal + } else { + event.calendar = store.defaultCalendarForNewEvents + } + do { + try store.save(event, span: .thisEvent) + ok(["id": event.eventIdentifier ?? ""]) + } catch { + fail("failed to save event: \(error.localizedDescription)") + } +} + +// Reminders share EKEventStore with calendar but need their own access grant. +func requestReminderAccess(_ store: EKEventStore) -> (granted: Bool, error: String?) { + let semaphore = DispatchSemaphore(value: 0) + var granted = false + var errorMessage: String? + let handler: (Bool, Error?) -> Void = { allowed, err in + granted = allowed + if let err = err { errorMessage = err.localizedDescription } + semaphore.signal() + } + if #available(macOS 14.0, *) { + store.requestFullAccessToReminders(completion: handler) + } else { + store.requestAccess(to: .reminder, completion: handler) + } + semaphore.wait() + return (granted, errorMessage) +} + +func createReminder(_ args: [String: Any]) -> Never { + guard let title = args["title"] as? String, !title.isEmpty else { + fail("createReminder requires a non-empty title") + } + let store = EKEventStore() + let access = requestReminderAccess(store) + if !access.granted { fail(access.error ?? "reminders access was not granted") } + + let reminder = EKReminder(eventStore: store) + reminder.title = title + reminder.calendar = store.defaultCalendarForNewReminders() + if let notes = args["notes"] as? String { reminder.notes = notes } + if let due = parseDate(args["due"]) { + reminder.dueDateComponents = Calendar.current.dateComponents( + [.year, .month, .day, .hour, .minute], from: due) + } + do { + try store.save(reminder, commit: true) + ok(["id": reminder.calendarItemIdentifier]) + } catch { + fail("failed to save reminder: \(error.localizedDescription)") + } +} + +func listReminders(_ args: [String: Any]) -> Never { + let store = EKEventStore() + let access = requestReminderAccess(store) + if !access.granted { fail(access.error ?? "reminders access was not granted") } + + let predicate = store.predicateForIncompleteReminders( + withDueDateStarting: nil, ending: nil, calendars: nil) + let semaphore = DispatchSemaphore(value: 0) + var out: [[String: Any]] = [] + store.fetchReminders(matching: predicate) { reminders in + for reminder in reminders ?? [] { + var item: [String: Any] = ["id": reminder.calendarItemIdentifier, "title": reminder.title ?? ""] + if let due = reminder.dueDateComponents, let date = Calendar.current.date(from: due) { + item["due"] = iso.string(from: date) + } + out.append(item) + } + semaphore.signal() + } + semaphore.wait() + ok(["reminders": out]) +} + +func listEvents(_ args: [String: Any]) -> Never { + guard let start = parseDate(args["start"]), let end = parseDate(args["end"]) else { + fail("listEvents requires ISO8601 start and end dates") + } + let store = EKEventStore() + let access = requestEventAccess(store) + if !access.granted { fail(access.error ?? "calendar access was not granted") } + + let predicate = store.predicateForEvents(withStart: start, end: end, calendars: nil) + let events = store.events(matching: predicate).map { event -> [String: Any] in + [ + "id": event.eventIdentifier ?? "", + "title": event.title ?? "", + "start": iso.string(from: event.startDate), + "end": iso.string(from: event.endDate), + "allDay": event.isAllDay, + "calendar": event.calendar?.title ?? "" + ] + } + ok(["events": events]) +} + +func requestContactsAccess(_ store: CNContactStore) -> (granted: Bool, error: String?) { + let semaphore = DispatchSemaphore(value: 0) + var granted = false + var errorMessage: String? + store.requestAccess(for: .contacts) { allowed, err in + granted = allowed + if let err = err { errorMessage = err.localizedDescription } + semaphore.signal() + } + semaphore.wait() + return (granted, errorMessage) +} + +func searchContacts(_ args: [String: Any]) -> Never { + guard let query = args["query"] as? String, !query.isEmpty else { + fail("searchContacts requires a non-empty query") + } + let store = CNContactStore() + let access = requestContactsAccess(store) + if !access.granted { fail(access.error ?? "contacts access was not granted") } + + let keys: [CNKeyDescriptor] = [ + CNContactGivenNameKey as CNKeyDescriptor, + CNContactFamilyNameKey as CNKeyDescriptor, + CNContactPhoneNumbersKey as CNKeyDescriptor, + CNContactEmailAddressesKey as CNKeyDescriptor + ] + let predicate = CNContact.predicateForContacts(matchingName: query) + do { + let contacts = try store.unifiedContacts(matching: predicate, keysToFetch: keys) + let out = contacts.map { contact -> [String: Any] in + let name = CNContactFormatter.string(from: contact, style: .fullName) + ?? "\(contact.givenName) \(contact.familyName)" + return [ + "name": name, + "phones": contact.phoneNumbers.map { $0.value.stringValue }, + "emails": contact.emailAddresses.map { $0.value as String } + ] + } + ok(["contacts": out]) + } catch { + fail("failed to search contacts: \(error.localizedDescription)") + } +} + +// AppleScript backs the send actions (Messages, Mail). User-supplied values are +// escaped before interpolation so a quote or backslash cannot break the script or +// inject extra statements. +func escapeForAppleScript(_ value: String) -> String { + return value + .replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "\"", with: "\\\"") +} + +func runAppleScript(_ source: String) -> String? { + var errorDict: NSDictionary? + let script = NSAppleScript(source: source) + _ = script?.executeAndReturnError(&errorDict) + if let errorDict = errorDict { + return (errorDict[NSAppleScript.errorMessage] as? String) ?? "AppleScript error" + } + return nil +} + +func sendMessage(_ args: [String: Any]) -> Never { + guard let to = args["to"] as? String, !to.isEmpty else { + fail("sendMessage requires a 'to' recipient") + } + guard let text = args["text"] as? String, !text.isEmpty else { + fail("sendMessage requires non-empty 'text'") + } + let script = """ + tell application "Messages" + send "\(escapeForAppleScript(text))" to participant "\(escapeForAppleScript(to))" of (1st account whose service type = iMessage) + end tell + """ + if let err = runAppleScript(script) { fail("failed to send message: \(err)") } + ok(["sent": true]) +} + +func sendMail(_ args: [String: Any]) -> Never { + guard let to = args["to"] as? String, !to.isEmpty else { + fail("sendMail requires a 'to' recipient") + } + let subject = (args["subject"] as? String) ?? "" + let body = (args["body"] as? String) ?? "" + let script = """ + tell application "Mail" + set newMessage to make new outgoing message with properties {subject:"\(escapeForAppleScript(subject))", content:"\(escapeForAppleScript(body))", visible:false} + tell newMessage + make new to recipient at end of to recipients with properties {address:"\(escapeForAppleScript(to))"} + send + end tell + end tell + """ + if let err = runAppleScript(script) { fail("failed to send mail: \(err)") } + ok(["sent": true]) +} + +func openURL(_ args: [String: Any]) -> Never { + guard let urlString = args["url"] as? String, let url = URL(string: urlString) else { + fail("openURL requires a valid 'url'") + } + if NSWorkspace.shared.open(url) { + ok(["opened": true]) + } else { + fail("failed to open URL: \(urlString)") + } +} + +let arguments = CommandLine.arguments +guard arguments.count >= 2 else { fail("no command provided") } +guard let data = arguments[1].data(using: .utf8), + let payload = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let command = payload["command"] as? String else { + fail("invalid command JSON") +} +let commandArgs = (payload["args"] as? [String: Any]) ?? [:] + +switch command { +case "calendar.createEvent": + createEvent(commandArgs) +case "calendar.listEvents": + listEvents(commandArgs) +case "reminders.create": + createReminder(commandArgs) +case "reminders.list": + listReminders(commandArgs) +case "contacts.search": + searchContacts(commandArgs) +case "messages.send": + sendMessage(commandArgs) +case "mail.send": + sendMail(commandArgs) +case "system.openURL": + openURL(commandArgs) +default: + fail("unknown command: \(command)") +} diff --git a/scripts/build-actions-helper.sh b/scripts/build-actions-helper.sh new file mode 100755 index 00000000..79879fa4 --- /dev/null +++ b/scripts/build-actions-helper.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Compile the native actions helper (EventKit / Reminders / Contacts / Photos), the +# backend of the computer-use semantic rail. Output lands next to the source so dev +# mode finds it; CI copies it into resources/bin so extraResources bundles it at +# Contents/Resources/bin. Pinned to the same deployment target as every other bundled +# native binary (macOS 13) so it launches on the versions the app advertises. +set -euo pipefail +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SRC="$ROOT_DIR/actions-helper/main.swift" +OUT="$ROOT_DIR/actions-helper/actions-helper" +swiftc -O -target arm64-apple-macos13.0 -emit-executable "$SRC" -o "$OUT" +echo "built $OUT" diff --git a/scripts/build-mac-local.sh b/scripts/build-mac-local.sh index dc3e71a9..e64d7984 100755 --- a/scripts/build-mac-local.sh +++ b/scripts/build-mac-local.sh @@ -56,10 +56,12 @@ stage_native_helpers() { MACOS_DEPLOYMENT_TARGET=13.0 WHISPER_REF=v1.7.4 bash scripts/build-whisper-cli.sh bash scripts/build-meeting-recorder.sh bash scripts/build-dictation-hotkey.sh + bash scripts/build-actions-helper.sh mkdir -p resources/bin cp scripts/meeting-recorder/meeting-recorder resources/bin/meeting-recorder cp scripts/dictation-hotkey/dictation-hotkey resources/bin/dictation-hotkey - chmod +x resources/bin/meeting-recorder resources/bin/dictation-hotkey + cp scripts/actions-helper/actions-helper resources/bin/actions-helper + chmod +x resources/bin/meeting-recorder resources/bin/dictation-hotkey resources/bin/actions-helper bash scripts/fetch-parakeet.sh } diff --git a/src/main/__tests__/computer-use-entitlements.test.ts b/src/main/__tests__/computer-use-entitlements.test.ts new file mode 100644 index 00000000..418a9169 --- /dev/null +++ b/src/main/__tests__/computer-use-entitlements.test.ts @@ -0,0 +1,46 @@ +/** + * Packaging contract for computer use's semantic action rail. Each TCC usage + * string must survive in electron-builder.yml and the apple-events entitlement in + * the plist: a hardened-runtime build is refused the capability BEFORE any prompt + * when its Info.plist key is missing, so a dropped key is a silent, ship-breaking + * regression (the exact "half-built in the safe direction" failure the computer-use + * plan warns about). Guarded by reading the source, per CLAUDE.md contract guards. + */ +import fs from 'node:fs' +import path from 'node:path' +import { describe, expect, it } from 'vitest' + +const root = path.resolve(import.meta.dirname, '../../..') +const builder = fs.readFileSync(path.join(root, 'electron-builder.yml'), 'utf8') +const entitlements = fs.readFileSync(path.join(root, 'build/entitlements.mac.plist'), 'utf8') + +// The usage-description keys the semantic rail needs. Calendars and Reminders +// carry BOTH the macOS 14+ FullAccess key and the pre-14 legacy key, because the +// build advertises minimumSystemVersion 13.0. +const REQUIRED_USAGE_KEYS = [ + 'NSAppleEventsUsageDescription', + 'NSCalendarsFullAccessUsageDescription', + 'NSCalendarsUsageDescription', + 'NSRemindersFullAccessUsageDescription', + 'NSContactsUsageDescription', + 'NSPhotoLibraryUsageDescription' +] + +describe('computer-use packaging entitlements', () => { + it('declares the apple-events entitlement AppleScript needs under hardened runtime', () => { + expect(entitlements).toContain('com.apple.security.automation.apple-events') + }) + + it.each(REQUIRED_USAGE_KEYS)('carries a non-empty %s usage string', (key) => { + const match = builder.match(new RegExp(`${key}:\\s*(\\S.*)$`, 'm')) + expect(match, `${key} missing from electron-builder.yml extendInfo`).not.toBeNull() + expect(match?.[1]?.trim().length ?? 0).toBeGreaterThan(0) + }) + + it('keeps the computer-use usage strings free of em dashes (brand rule)', () => { + for (const key of REQUIRED_USAGE_KEYS) { + const line = builder.match(new RegExp(`${key}:.*$`, 'm'))?.[0] ?? '' + expect(line, `${key} uses an em dash; the brand voice bans it (use " - ")`).not.toContain('—') + } + }) +}) diff --git a/src/main/__tests__/gate-host.integration.dbtest.ts b/src/main/__tests__/gate-host.integration.dbtest.ts new file mode 100644 index 00000000..39ed1743 --- /dev/null +++ b/src/main/__tests__/gate-host.integration.dbtest.ts @@ -0,0 +1,200 @@ +/** + * Box 11's done-when: the real engine on a real DB, gated through the real + * hook registry via the gate host. Proves approve runs exactly the approved + * payload (binding held end to end), reject lands the Action in rejected + * with the device never fired, and an edit re-binds before running. + */ +import fs from 'fs' +import os from 'os' +import path from 'path' +import Database from 'better-sqlite3-multiple-ciphers' +import { afterEach, describe, expect, it } from 'vitest' +import { HandlerRegistry, UseEngine, type ActionRecord } from '@offgrid/use' +import { makeUseDriver } from '../actions/use-driver' +import { gateHost, resolveActionGate } from '../actions/gate-host' +import { HOOKS, registerHook, unregisterHook } from '../bootstrap/hookRegistry' + +const tempDirs: string[] = [] +const openDbs: Database.Database[] = [] + +afterEach(() => { + unregisterHook(HOOKS.actionsProposeApproval) + for (const db of openDbs.splice(0)) { + try { + db.close() + } catch { + /* closed */ + } + } + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }) + } +}) + +function makeWorld() { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ogad-gate-host-')) + tempDirs.push(dir) + const db = new Database(path.join(dir, 'app.db')) + openDbs.push(db) + db.exec(`CREATE TABLE test_reminders (title TEXT NOT NULL)`) + + const registry = new HandlerRegistry() + registry.register({ + type: 'reminder', + rail: 'semantic', + defaultRisk: 'mutate', + verification: 'read_back', + verify: async (action) => { + const row = db + .prepare(`SELECT COUNT(*) AS n FROM test_reminders WHERE title = ?`) + .get(String(action.args.title)) as { n: number } + return row.n > 0 + } + }) + + const executed: Record[] = [] + const device = { + async execute(action: ActionRecord) { + executed.push({ ...action.args }) + db.prepare(`INSERT INTO test_reminders (title) VALUES (?)`).run(String(action.args.title)) + return { ok: true } + } + } + + const clock = { t: 1_000_000 } + let n = 0 + const engine = new UseEngine({ + driver: makeUseDriver(db), + registry, + device, + gate: gateHost, + now: () => clock.t, + newId: () => `act_${++n}`, + attemptTimeoutMs: 500, + visibilityMs: 60_000 + }) + return { engine, executed, db } +} + +/** Wait until the approval hook has captured the request for an id. */ +async function until(condition: () => boolean): Promise { + for (let i = 0; i < 200 && !condition(); i++) { + await new Promise((resolve) => setTimeout(resolve, 5)) + } + expect(condition()).toBe(true) +} + +/** Narrow a tick outcome to the record-carrying variants, or fail the test. */ +function recordOutcome(result: Awaited>) { + if (!result || result.outcome === 'poisoned') { + throw new Error(`unexpected tick outcome: ${JSON.stringify(result)}`) + } + return result +} + +/** The captured approval request at an index, or fail the test. */ +function requestAt(requests: Record[], index: number): Record { + const request = requests[index] + if (!request) { + throw new Error(`no approval request captured at index ${index}`) + } + return request +} + +const proposal = { + type: 'reminder', + intent: 'remind me to send the deck', + args: { title: 'Send the deck' }, + risk: 'mutate' +} + +describe('the engine gated through the real approval seam', () => { + it('approve runs exactly the approved payload', async () => { + const { engine, executed } = makeWorld() + await engine.init() + const requests: Record[] = [] + registerHook(HOOKS.actionsProposeApproval, (req: Record) => { + requests.push(req) + return true + }) + + await engine.propose(proposal, { source: 'chat' }) + const running = engine.tick() + await until(() => requests.length === 1) + + const request = requestAt(requests, 0) + expect(request).toMatchObject({ + kind: 'native', + risk: 'mutate', + actionType: 'reminder', + args: { title: 'Send the deck' } + }) + resolveActionGate(String(request.actionId), { kind: 'approve' }) + + const result = recordOutcome(await running) + expect(result.outcome).toBe('done') + expect(executed).toEqual([{ title: 'Send the deck' }]) + // The payload that ran is the payload the card showed, byte for byte. + expect(result.record.payloadHash).toBe(request.payloadHash) + }) + + it('reject lands the Action in rejected and the device never fires', async () => { + const { engine, executed } = makeWorld() + await engine.init() + const requests: Record[] = [] + registerHook(HOOKS.actionsProposeApproval, (req: Record) => { + requests.push(req) + return true + }) + + await engine.propose(proposal, { source: 'chat' }) + const running = engine.tick() + await until(() => requests.length === 1) + resolveActionGate(String(requestAt(requests, 0).actionId), { kind: 'reject', reason: 'not now' }) + + const result = recordOutcome(await running) + expect(result.outcome).toBe('rejected') + expect(result.record.state).toBe('rejected') + expect(executed).toEqual([]) + }) + + it('an edit at the card re-binds and the edited payload is what runs', async () => { + const { engine, executed } = makeWorld() + await engine.init() + const requests: Record[] = [] + registerHook(HOOKS.actionsProposeApproval, (req: Record) => { + requests.push(req) + return true + }) + + await engine.propose(proposal, { source: 'chat' }) + const first = engine.tick() + await until(() => requests.length === 1) + resolveActionGate(String(requestAt(requests, 0).actionId), { + kind: 'edit', + args: { title: 'Send the v2 deck' } + }) + expect((await first)?.outcome).toBe('edited') + + // The edited record re-gates on the next tick, with a new hash. + const second = engine.tick() + await until(() => requests.length === 2) + const regated = requestAt(requests, 1) + expect(regated.payloadHash).not.toBe(requestAt(requests, 0).payloadHash) + expect(regated.args).toEqual({ title: 'Send the v2 deck' }) + resolveActionGate(String(regated.actionId), { kind: 'approve' }) + + const result = await second + expect(result?.outcome).toBe('done') + expect(executed).toEqual([{ title: 'Send the v2 deck' }]) + }) + + it('free build (no hook registered): the mutation runs and verifies, unchanged behaviour', async () => { + const { engine, executed } = makeWorld() + await engine.init() + await engine.propose(proposal, { source: 'chat' }) + const result = await engine.tick() + expect(result?.outcome).toBe('done') + expect(executed).toEqual([{ title: 'Send the deck' }]) + }) +}) diff --git a/src/main/__tests__/image-runtime-reliability.integration.dbtest.ts b/src/main/__tests__/image-runtime-reliability.integration.dbtest.ts index affa767a..31a01650 100644 --- a/src/main/__tests__/image-runtime-reliability.integration.dbtest.ts +++ b/src/main/__tests__/image-runtime-reliability.integration.dbtest.ts @@ -338,7 +338,7 @@ describe('multimodal runtime reliability', () => { }, 20_000) it('keeps local chat usable when external network reachability is unavailable', async () => { - startModelServer(gatewayPort) + await startModelServer(gatewayPort) await expect(fetch('https://example.invalid/health')).rejects.toThrow( 'network unavailable in offline integration fixture: https://example.invalid' ) @@ -437,7 +437,7 @@ describe('multimodal runtime reliability', () => { expect(llm.isReady()).toBe(true) expect(lineCount(fixture.llamaLog) - startsBefore).toBe(1) - startModelServer(gatewayPort) + await startModelServer(gatewayPort) const health = await fetch(`http://127.0.0.1:${String(gatewayPort)}/v1`) expect(health.status).toBe(200) }) diff --git a/src/main/__tests__/license-gate-smoke.integration.test.ts b/src/main/__tests__/license-gate-smoke.integration.test.ts index 0eac07fd..5f097405 100644 --- a/src/main/__tests__/license-gate-smoke.integration.test.ts +++ b/src/main/__tests__/license-gate-smoke.integration.test.ts @@ -33,7 +33,10 @@ function runRunner( return spawnSync(process.execPath, args, { cwd: REPO_ROOT, encoding: 'utf8', - env: { ...process.env, ...extraEnvironment }, + // A runner living inside Electron (VS Code tasks, agent sandboxes) exports + // ELECTRON_RUN_AS_NODE=1; inherited, it turns the Electron under test into + // plain Node and the launch dies before the gate can be observed. + env: { ...process.env, ELECTRON_RUN_AS_NODE: undefined, ...extraEnvironment }, timeout: REAL_APP_TIMEOUT_MS }) } diff --git a/src/main/__tests__/mcp-connector-tool-extension.dbtest.ts b/src/main/__tests__/mcp-connector-tool-extension.dbtest.ts index 82b02338..82c2b800 100644 --- a/src/main/__tests__/mcp-connector-tool-extension.dbtest.ts +++ b/src/main/__tests__/mcp-connector-tool-extension.dbtest.ts @@ -25,6 +25,7 @@ import { type McpConnectorToolBoundary } from '../tools/mcpConnectorToolExtension' import type { ConnectorToolDefinition } from '../tools/mcpConnectorToolExtension-logic' +import type { ActionApprovalRequest } from '../actions/approval' interface ToolExecution { connectorId: number @@ -38,7 +39,7 @@ class FakeMcpBoundary implements McpConnectorToolBoundary { readonly tools = new Map() readonly results = new Map() readonly executions: ToolExecution[] = [] - readonly approvals: Record[] = [] + readonly approvals: ActionApprovalRequest[] = [] approveWrites = false async fetchTools(connectorId: number): Promise { @@ -62,7 +63,7 @@ class FakeMcpBoundary implements McpConnectorToolBoundary { return result } - proposeApproval(request: Record): boolean { + proposeApproval(request: ActionApprovalRequest): boolean { this.approvals.push(request) return this.approveWrites } @@ -149,6 +150,8 @@ describe('McpConnectorToolExtension with real connector state', () => { expect(output).toContain('Queued for the user') expect(boundary.approvals).toEqual([ expect.objectContaining({ + kind: 'mcp', + risk: 'mutate', connectorId, tool: 'send_message', connector: 'Slack', diff --git a/src/main/__tests__/use-runtime.integration.dbtest.ts b/src/main/__tests__/use-runtime.integration.dbtest.ts new file mode 100644 index 00000000..c34951e6 --- /dev/null +++ b/src/main/__tests__/use-runtime.integration.dbtest.ts @@ -0,0 +1,86 @@ +/** + * The actions runtime composition, on a real DB with only its true + * boundaries mocked: electron (paths) and the native helper (the OS). Covers + * what the pure suites cannot - the lazy singleton, the device's rail guard, + * propose/waitForOutcome through the real worker, and the approval-hook + * probe - so the wiring the app actually ships is measured, not assumed. + */ +import fs from 'fs' +import os from 'os' +import path from 'path' +import { afterAll, describe, expect, it, vi } from 'vitest' +import { HOOKS, registerHook, unregisterHook } from '../bootstrap/hookRegistry' + +const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ogad-use-runtime-')) +process.env.OFFGRID_USER_DATA = tempDir + +vi.mock('electron', () => ({ + app: { + isPackaged: false, + getPath: () => tempDir, + getAppPath: () => tempDir + } +})) + +// The OS boundary: reminders land in memory; lists read them back. +const landed: string[] = [] +vi.mock('../actions/native-helper', () => ({ + runNativeAction: vi.fn(async (cmd: { command: string; args: Record }) => { + if (cmd.command === 'reminders.create') { + landed.push(String(cmd.args.title)) + return { ok: true, result: { id: 'rt1' } } + } + if (cmd.command === 'reminders.list') { + return { ok: true, result: { reminders: landed.map((title) => ({ title })) } } + } + return { ok: false, error: `unhandled ${cmd.command}` } + }) +})) + +afterAll(() => { + fs.rmSync(tempDir, { recursive: true, force: true }) +}) + +describe('getActionsRuntime', () => { + it('composes once (lazy singleton) and drives a real action end to end', async () => { + const { getActionsRuntime } = await import('../actions/use-runtime') + const runtime = getActionsRuntime() + expect(getActionsRuntime()).toBe(runtime) + + const proposed = await runtime.propose( + { + type: 'reminder', + intent: 'remind me to send the deck', + args: { title: 'Send the deck' }, + risk: 'mutate' + }, + { source: 'chat' } + ) + expect(proposed.accepted).toBe(true) + if (!proposed.accepted) { + return + } + runtime.kick() + const outcome = await runtime.waitForOutcome(proposed.id, 10_000) + expect(outcome?.outcome).toBe('done') + expect(landed).toEqual(['Send the deck']) + }) + + it('waitForOutcome times out to undefined for an unknown action', async () => { + const { getActionsRuntime } = await import('../actions/use-runtime') + const outcome = await getActionsRuntime().waitForOutcome('act_ghost', 50) + expect(outcome).toBeUndefined() + }) + + it('approvalHookActive reflects both hook registrations', async () => { + const { getActionsRuntime } = await import('../actions/use-runtime') + const runtime = getActionsRuntime() + expect(runtime.approvalHookActive()).toBe(false) + registerHook(HOOKS.actionsProposeApproval, () => true) + expect(runtime.approvalHookActive()).toBe(true) + unregisterHook(HOOKS.actionsProposeApproval) + registerHook(HOOKS.legacyMcpProposeApproval, () => true) + expect(runtime.approvalHookActive()).toBe(true) + unregisterHook(HOOKS.legacyMcpProposeApproval) + }) +}) diff --git a/src/main/__tests__/use-storage.integration.dbtest.ts b/src/main/__tests__/use-storage.integration.dbtest.ts new file mode 100644 index 00000000..786ab14c --- /dev/null +++ b/src/main/__tests__/use-storage.integration.dbtest.ts @@ -0,0 +1,188 @@ +/** + * Integration tests at the real DB seam: the actual @offgrid/use engine + * running against a real better-sqlite3 file in a temp dir - no mocks + * between the engine and SQLite. Fakes exist only at the true boundaries + * (the device = the OS surface, the gate = a human). The device writes its + * effect into a table in the SAME database, and the handler verifies by + * reading it back - one DB as the source of truth, end to end. + */ +import fs from 'fs' +import os from 'os' +import path from 'path' +// The app's own SQLite build (drop-in better-sqlite3 superset); the db suite +// swaps its native ABI to the test runner's node for the run. +import Database from 'better-sqlite3-multiple-ciphers' +import { afterEach, describe, expect, it } from 'vitest' +import { HandlerRegistry, UseEngine, type ActionRecord, type GateDecision } from '@offgrid/use' +import { makeUseDriver } from '../actions/use-driver' + +const tempDirs: string[] = [] +const openDbs: Database.Database[] = [] + +afterEach(() => { + for (const db of openDbs.splice(0)) { + try { + db.close() + } catch { + /* already closed */ + } + } + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }) + } +}) + +function makeAppDb(): { db: Database.Database; dbPath: string } { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ogad-use-storage-')) + tempDirs.push(dir) + const dbPath = path.join(dir, 'app.db') + const db = new Database(dbPath) + openDbs.push(db) + // The app's own world: an existing table the engine must coexist with, + // and the table the semantic rail's effects land in. + db.exec(`CREATE TABLE app_settings (key TEXT PRIMARY KEY, value TEXT)`) + db.exec(`CREATE TABLE test_reminders (title TEXT NOT NULL)`) + db.prepare(`INSERT INTO app_settings (key, value) VALUES (?, ?)`).run('theme', 'dark') + return { db, dbPath } +} + +function makeEngine( + db: Database.Database, + clock: { t: number }, + options: { gate?: (record: ActionRecord) => GateDecision; ids?: string } = {} +) { + const registry = new HandlerRegistry() + registry.register({ + type: 'reminder', + rail: 'semantic', + defaultRisk: 'mutate', + verification: 'read_back', + verify: async (action) => { + const row = db + .prepare(`SELECT COUNT(*) AS n FROM test_reminders WHERE title = ?`) + .get(String(action.args.title)) as { n: number } + return row.n > 0 + } + }) + const device = { + calls: 0, + async execute(action: ActionRecord) { + device.calls += 1 + db.prepare(`INSERT INTO test_reminders (title) VALUES (?)`).run(String(action.args.title)) + return { ok: true } + } + } + let n = 0 + const engine = new UseEngine({ + driver: makeUseDriver(db), + registry, + device, + gate: async ({ action }) => options.gate?.(action) ?? { kind: 'approve' as const }, + now: () => clock.t, + newId: () => `${options.ids ?? 'act'}_${++n}`, + attemptTimeoutMs: 500, + visibilityMs: 1000 + }) + return { engine, device } +} + +const proposal = (title: string, triggerAt?: number) => ({ + type: 'reminder', + intent: `remind me: ${title}`, + args: { title }, + risk: 'mutate', + ...(triggerAt ? { triggerAt } : {}) +}) + +describe('the engine on the app database', () => { + it('migrates its tables into the app DB and coexists with app tables', async () => { + const { db } = makeAppDb() + const clock = { t: 1_000_000 } + const { engine } = makeEngine(db, clock) + await engine.init() + await engine.init() // idempotent + + const tables = db + .prepare(`SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name`) + .all() + .map((r) => (r as { name: string }).name) + expect(tables).toContain('use_queue') + expect(tables).toContain('app_settings') + const setting = db.prepare(`SELECT value FROM app_settings WHERE key = 'theme'`).get() as { + value: string + } + expect(setting.value).toBe('dark') + }) + + it('walks a real action end to end: the effect lands in the same DB and read-back verifies it', async () => { + const { db } = makeAppDb() + const clock = { t: 1_000_000 } + const { engine, device } = makeEngine(db, clock) + await engine.init() + + const proposed = await engine.propose(proposal('Send the deck'), { source: 'chat' }) + expect(proposed.accepted).toBe(true) + + const result = await engine.tick() + expect(result?.outcome).toBe('done') + expect(device.calls).toBe(1) + const rows = db.prepare(`SELECT title FROM test_reminders`).all() as { title: string }[] + expect(rows.map((r) => r.title)).toEqual(['Send the deck']) + expect(db.prepare(`SELECT COUNT(*) AS n FROM use_queue`).get()).toEqual({ n: 0 }) + }) + + it('a scheduled action survives a full engine restart over the same file', async () => { + const { db, dbPath } = makeAppDb() + const clock = { t: 1_000_000 } + const first = makeEngine(db, clock, { ids: 'a' }) + await first.engine.init() + await first.engine.propose(proposal('later', clock.t + 60_000), { source: 'schedule' }) + expect(await first.engine.tick()).toBeUndefined() + db.close() // the app quits + + const reopened = new Database(dbPath) + openDbs.push(reopened) + reopened.exec(`CREATE TABLE IF NOT EXISTS test_reminders (title TEXT NOT NULL)`) + clock.t += 60_000 + const second = makeEngine(reopened, clock, { ids: 'b' }) + await second.engine.init() + const result = await second.engine.tick() + expect(result?.outcome).toBe('done') + const rows = reopened.prepare(`SELECT title FROM test_reminders`).all() as { title: string }[] + expect(rows.map((r) => r.title)).toEqual(['later']) + }) + + it('a lease held by one engine blocks a second engine on the same DB until it expires', async () => { + const { db } = makeAppDb() + const clock = { t: 1_000_000 } + const a = makeEngine(db, clock, { ids: 'a' }) + const b = makeEngine(db, clock, { ids: 'b' }) + await a.engine.init() + await a.engine.propose(proposal('exclusive'), { source: 'chat' }) + + // Worker A leases the message directly (simulating a worker that died + // mid-run without completing). + const leased = await a.engine.queue.receive() + expect(leased?.id).toBeDefined() + + expect(await b.engine.tick()).toBeUndefined() // blocked by the live lease + clock.t += 1001 // the dead worker's lease expires + const result = await b.engine.tick() + expect(result?.outcome).toBe('done') + expect(b.device.calls + a.device.calls).toBe(1) + }) + + it('dedup holds across engine instances sharing the DB', async () => { + const { db } = makeAppDb() + const clock = { t: 1_000_000 } + const a = makeEngine(db, clock, { ids: 'a' }) + const b = makeEngine(db, clock, { ids: 'b' }) + await a.engine.init() + + const first = await a.engine.propose(proposal('once'), { source: 'chat' }) + const second = await b.engine.propose(proposal('once'), { source: 'chat' }) + expect(first).toMatchObject({ accepted: true, deduped: false }) + expect(second).toMatchObject({ accepted: true, deduped: true }) + expect(db.prepare(`SELECT COUNT(*) AS n FROM use_queue`).get()).toEqual({ n: 1 }) + }) +}) diff --git a/src/main/__tests__/verification.integration.dbtest.ts b/src/main/__tests__/verification.integration.dbtest.ts new file mode 100644 index 00000000..b6ebe520 --- /dev/null +++ b/src/main/__tests__/verification.integration.dbtest.ts @@ -0,0 +1,128 @@ +/** + * Box 14's done-when: a failed read-back drives the retry policy correctly, + * proven on the real engine + real DB + the REAL registry the app ships + * (buildRegistry), with only the helper boundary scripted. The same + * scripted helper serves both the rail (create) and the verifiers (list), + * exactly as production shares runNativeAction. + */ +import fs from 'fs' +import os from 'os' +import path from 'path' +import Database from 'better-sqlite3-multiple-ciphers' +import { afterEach, describe, expect, it } from 'vitest' +import { UseEngine, type ActionRecord, type Rail } from '@offgrid/use' +import { makeUseDriver } from '../actions/use-driver' +import { makeSemanticRailExecutor } from '../actions/semantic-rail' +import { buildRegistry } from '../actions/use-runtime' +import type { NativeActionCommand, NativeActionResponse } from '../actions/native-helper-logic' + +const tempDirs: string[] = [] +const openDbs: Database.Database[] = [] + +afterEach(() => { + for (const db of openDbs.splice(0)) { + try { + db.close() + } catch { + /* closed */ + } + } + for (const dir of tempDirs.splice(0)) { + fs.rmSync(dir, { recursive: true, force: true }) + } +}) + +/** + * A scripted Reminders world: creates succeed or silently drop (the classic + * false-ok), lists report what actually landed. + */ +function makeWorld({ dropFirstCreates = 0 } = {}) { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'ogad-verify-')) + tempDirs.push(dir) + const db = new Database(path.join(dir, 'app.db')) + openDbs.push(db) + + const landed: string[] = [] + let drops = dropFirstCreates + let creates = 0 + const run = async (cmd: NativeActionCommand): Promise => { + if (cmd.command === 'reminders.create') { + creates += 1 + if (drops > 0) { + drops -= 1 + return { ok: true, result: { id: 'ghost' } } // claims ok, never lands + } + landed.push(String(cmd.args.title)) + return { ok: true, result: { id: `r${creates}` } } + } + if (cmd.command === 'reminders.list') { + return { ok: true, result: { reminders: landed.map((title) => ({ title })) } } + } + return { ok: false, error: `unexpected command ${cmd.command}` } + } + + const semanticExecute = makeSemanticRailExecutor(run) + const clock = { t: 1_000_000 } + let n = 0 + const engine = new UseEngine({ + driver: makeUseDriver(db), + registry: buildRegistry(run), + device: { + async execute(action: ActionRecord, rail: Rail) { + if (rail !== 'semantic') { + return { ok: false, detail: 'wrong rail' } + } + return semanticExecute(action) + } + }, + gate: async () => ({ kind: 'approve' as const }), + now: () => clock.t, + newId: () => `act_${++n}`, + attemptTimeoutMs: 500, + visibilityMs: 60_000 + }) + return { engine, landed, creates: () => creates } +} + +const proposal = { + type: 'reminder', + intent: 'remind me to send the deck', + args: { title: 'Send the deck' }, + risk: 'mutate' +} + +describe('read-back verification driving the retry policy (real registry, real DB)', () => { + it('a clean create verifies by read-back and is done in one attempt', async () => { + const { engine, landed, creates } = makeWorld() + await engine.init() + await engine.propose(proposal, { source: 'chat' }) + const result = await engine.tick() + expect(result?.outcome).toBe('done') + expect(landed).toEqual(['Send the deck']) + expect(creates()).toBe(1) + }) + + it('a false-ok create is caught by read-back and retried exactly once to success', async () => { + const { engine, landed, creates } = makeWorld({ dropFirstCreates: 1 }) + await engine.init() + await engine.propose(proposal, { source: 'chat' }) + const result = await engine.tick() + expect(result?.outcome).toBe('done') + expect(creates()).toBe(2) + expect(landed).toEqual(['Send the deck']) + if (result && result.outcome !== 'poisoned') { + expect(result.record.attempts).toBe(2) + expect(result.record.attemptLog.map((a) => a.outcome)).toEqual(['ok', 'ok']) + } + }) + + it('a write that never lands exhausts retry-once and asks instead of looping', async () => { + const { engine, landed, creates } = makeWorld({ dropFirstCreates: 99 }) + await engine.init() + await engine.propose(proposal, { source: 'chat' }) + const result = await engine.tick() + expect(result?.outcome).toBe('needs_help') + expect(creates()).toBe(2) + expect(landed).toEqual([]) + }) +}) diff --git a/src/main/actions/__tests__/approval.test.ts b/src/main/actions/__tests__/approval.test.ts new file mode 100644 index 00000000..fee1f236 --- /dev/null +++ b/src/main/actions/__tests__/approval.test.ts @@ -0,0 +1,78 @@ +/** + * Unit tests for the transport-agnostic action-approval seam. High blast radius: + * every executor that can act on the user's behalf (MCP connectors today, computer + * and browser actions next) gates through shouldGate + proposeActionApproval, and + * the free/pro split hinges on whether a hook is registered. + */ +import { afterEach, describe, expect, it } from 'vitest' +import { + shouldGate, + proposeActionApproval, + type ActionApprovalRequest, + type ActionRisk +} from '../approval' +import { registerHook, unregisterHook, HOOKS } from '../../bootstrap/hookRegistry' + +const NEW = HOOKS.actionsProposeApproval +const LEGACY = HOOKS.legacyMcpProposeApproval + +function request(risk: ActionRisk): ActionApprovalRequest { + return { kind: 'mcp', title: 't', detail: 'd', risk, args: {}, source: 'chat' } +} + +afterEach(() => { + unregisterHook(NEW) + unregisterHook(LEGACY) +}) + +describe('shouldGate', () => { + it('gates mutate and irreversible, runs read and navigate freely', () => { + expect(shouldGate('mutate')).toBe(true) + expect(shouldGate('irreversible')).toBe(true) + expect(shouldGate('read')).toBe(false) + expect(shouldGate('navigate')).toBe(false) + }) +}) + +describe('proposeActionApproval', () => { + it('returns undefined when nothing is listening (free build runs the action)', () => { + expect(proposeActionApproval(request('mutate'))).toBeUndefined() + }) + + it('routes to the new hook and forwards its verdict', () => { + registerHook(NEW, () => true) + expect(proposeActionApproval(request('mutate'))).toBe(true) + }) + + it('honours a registered new hook that declined to queue (returns false)', () => { + registerHook(NEW, () => false) + expect(proposeActionApproval(request('mutate'))).toBe(false) + }) + + it('trusts a registered new hook even when it returns undefined — no legacy fallback', () => { + let legacyCalled = false + registerHook(NEW, () => undefined) + registerHook(LEGACY, () => { + legacyCalled = true + return true + }) + expect(proposeActionApproval(request('mutate'))).toBeUndefined() + expect(legacyCalled).toBe(false) + }) + + it('falls back to the legacy hook when the new name is unregistered', () => { + registerHook(LEGACY, () => true) + expect(proposeActionApproval(request('mutate'))).toBe(true) + }) + + it('passes the full request through to the handler', () => { + let seen: ActionApprovalRequest | undefined + registerHook(NEW, (req: ActionApprovalRequest) => { + seen = req + return true + }) + const req = request('irreversible') + proposeActionApproval(req) + expect(seen).toEqual(req) + }) +}) diff --git a/src/main/actions/__tests__/emit.test.ts b/src/main/actions/__tests__/emit.test.ts new file mode 100644 index 00000000..f6547901 --- /dev/null +++ b/src/main/actions/__tests__/emit.test.ts @@ -0,0 +1,139 @@ +/** + * Emission hardening: one case per repair branch, and the discipline that + * an unrepairable emission is rejected, never guessed. + */ +import { describe, expect, it, vi } from 'vitest' +import { + actionProposalJsonSchema, + emitActionProposal, + extractBalancedObject, + parseEmission +} from '../emit' + +const valid = { + type: 'reminder', + intent: 'remind me to send the deck at 6pm', + args: { title: 'Send the deck' }, + risk: 'mutate' +} +const validJson = JSON.stringify(valid) + +describe('actionProposalJsonSchema', () => { + it('constrains type to the registered handlers, not the full vocabulary', () => { + const schema = actionProposalJsonSchema(['reminder', 'open']) + const properties = schema.properties as Record + expect(properties.type?.enum).toEqual(['reminder', 'open']) + }) + + it('requires the proposal fields and forbids extras', () => { + const schema = actionProposalJsonSchema(['reminder']) + expect(schema.required).toEqual(['type', 'intent', 'args', 'risk']) + expect(schema.additionalProperties).toBe(false) + }) +}) + +describe('extractBalancedObject', () => { + it('finds the object inside prose and respects braces in strings', () => { + const text = 'Sure! Here it is: {"a": "curly } inside", "b": {"c": 1}} - hope that helps' + expect(extractBalancedObject(text)).toBe('{"a": "curly } inside", "b": {"c": 1}}') + }) + + it('handles escaped quotes inside strings', () => { + const text = 'prefix {"a": "say \\"hi\\" loudly", "b": 1} suffix' + expect(extractBalancedObject(text)).toBe('{"a": "say \\"hi\\" loudly", "b": 1}') + }) + + it('returns undefined when no object closes', () => { + expect(extractBalancedObject('nothing here')).toBeUndefined() + expect(extractBalancedObject('{"never": "closes"')).toBeUndefined() + }) +}) + +describe('parseEmission - one case per repair branch', () => { + it('clean JSON parses as-is', () => { + expect(parseEmission(validJson)).toEqual({ ok: true, proposal: valid }) + }) + + it('a markdown fence is stripped', () => { + const result = parseEmission('```json\n' + validJson + '\n```') + expect(result.ok).toBe(true) + }) + + it('surrounding prose is cut away', () => { + const result = parseEmission(`Sure, here's the action you asked for:\n${validJson}\nLet me know!`) + expect(result.ok).toBe(true) + }) + + it('a trailing comma is repaired', () => { + const raw = `{"type": "reminder", "intent": "x", "args": {"title": "y",}, "risk": "mutate",}` + const result = parseEmission(raw) + expect(result.ok).toBe(true) + }) + + it('bare keys are quoted', () => { + const raw = `{type: "reminder", intent: "x", args: {title: "y"}, risk: "mutate"}` + const result = parseEmission(raw) + expect(result.ok).toBe(true) + if (result.ok) { + expect(result.proposal.args).toEqual({ title: 'y' }) + } + }) + + it('a missing optional args falls back to the schema default', () => { + const raw = `{"type": "lookup", "intent": "what is on my calendar", "risk": "read"}` + const result = parseEmission(raw) + expect(result.ok).toBe(true) + if (result.ok) { + expect(result.proposal.args).toEqual({}) + } + }) + + it('an unrepairable emission is rejected, never guessed', () => { + const result = parseEmission('I am sorry, I cannot create reminders.') + expect(result.ok).toBe(false) + }) + + it('a repaired but invalid proposal still fails closed, with the reason', () => { + const raw = `Here: {"type": "teleport", "intent": "x", "args": {}, "risk": "mutate"}` + const result = parseEmission(raw) + expect(result.ok).toBe(false) + if (!result.ok) { + expect(result.error).toMatch(/type/) + } + }) + + it('an engine-owned field on the proposal is a rejection (strict schema)', () => { + const raw = JSON.stringify({ ...valid, id: 'act_1', state: 'ready' }) + expect(parseEmission(raw).ok).toBe(false) + }) +}) + +describe('emitActionProposal - bounded retry with the error fed back', () => { + it('a clean first answer needs no retry', async () => { + const ask = vi.fn(async () => validJson) + const result = await emitActionProposal(ask) + expect(result.ok).toBe(true) + expect(ask).toHaveBeenCalledTimes(1) + expect(ask).toHaveBeenCalledWith(undefined) + }) + + it('a bad first answer retries once with the validation error in the feedback', async () => { + const ask = vi + .fn() + .mockResolvedValueOnce('cannot do') + .mockResolvedValueOnce(validJson) + const result = await emitActionProposal(ask) + expect(result.ok).toBe(true) + expect(ask).toHaveBeenCalledTimes(2) + const feedback = ask.mock.calls[1]?.[0] as string + expect(feedback).toMatch(/not a valid action/) + expect(feedback).toMatch(/ONLY the corrected JSON/) + }) + + it('exhausted attempts reject with the last error - never a guess', async () => { + const ask = vi.fn(async () => 'still nonsense') + const result = await emitActionProposal(ask, { maxAttempts: 3 }) + expect(result.ok).toBe(false) + expect(ask).toHaveBeenCalledTimes(3) + }) +}) diff --git a/src/main/actions/__tests__/gate-host.test.ts b/src/main/actions/__tests__/gate-host.test.ts new file mode 100644 index 00000000..b3daf57f --- /dev/null +++ b/src/main/actions/__tests__/gate-host.test.ts @@ -0,0 +1,168 @@ +/** + * The gate host bridging the engine's awaitable gate to the app's + * fire-and-queue approval seam. Guards: the free build keeps its unchanged + * run-now behaviour, a queued action parks until the approval surface + * resolves it, and the request the surface receives carries everything the + * card needs (id, type, payload hash, mapped kind). + */ +import { afterEach, describe, expect, it, vi } from 'vitest' +import type { ActionRecord } from '@offgrid/use' +import { HOOKS, registerHook, unregisterHook } from '../../bootstrap/hookRegistry' +import { + abandonActionGate, + gateHost, + onGateParked, + pendingActionGateCount, + railToKind, + resolveActionGate, + whenActionParked +} from '../gate-host' + +const record = (overrides: Partial = {}): ActionRecord => + ({ + type: 'reminder', + intent: 'remind me to send the deck', + args: { title: 'Send the deck' }, + risk: 'mutate', + id: 'act_1', + source: 'chat', + payloadHash: 'a'.repeat(64), + rail: 'semantic', + idempotencyKey: 'k', + attempts: 0, + attemptLog: [], + state: 'awaiting_approval', + createdAt: 1, + updatedAt: 1, + ...overrides + }) as ActionRecord + +afterEach(() => { + unregisterHook(HOOKS.actionsProposeApproval) + unregisterHook(HOOKS.legacyMcpProposeApproval) + abandonActionGate('act_1') + abandonActionGate('act_2') +}) + +describe('railToKind', () => { + it('maps the engine rails onto the approval kinds', () => { + expect(railToKind('semantic')).toBe('native') + expect(railToKind('browser')).toBe('browser') + expect(railToKind('accessibility')).toBe('computer') + expect(railToKind('vision')).toBe('computer') + expect(railToKind(undefined)).toBe('native') + }) +}) + +describe('gateHost', () => { + it('free build (nothing listening): approves immediately - unchanged behaviour', async () => { + const decision = await gateHost({ action: record() }) + expect(decision).toEqual({ kind: 'approve' }) + expect(pendingActionGateCount()).toBe(0) + }) + + it('a handler that declines to queue also lets the action run', async () => { + registerHook(HOOKS.actionsProposeApproval, () => false) + const decision = await gateHost({ action: record() }) + expect(decision).toEqual({ kind: 'approve' }) + }) + + it('a queued action parks until the approval surface resolves it', async () => { + const seen = vi.fn(() => true) + registerHook(HOOKS.actionsProposeApproval, seen) + + const parked = gateHost({ action: record() }) + expect(pendingActionGateCount()).toBe(1) + + expect(resolveActionGate('act_1', { kind: 'approve' })).toBe(true) + await expect(parked).resolves.toEqual({ kind: 'approve' }) + expect(pendingActionGateCount()).toBe(0) + }) + + it('the request carries what the card needs: id, type, hash, mapped kind, args', async () => { + let request: Record = {} + registerHook(HOOKS.actionsProposeApproval, (req: Record) => { + request = req + return true + }) + const parked = gateHost({ action: record({ rail: 'browser', risk: 'irreversible' }) }) + expect(request).toMatchObject({ + kind: 'browser', + risk: 'irreversible', + actionId: 'act_1', + actionType: 'reminder', + payloadHash: 'a'.repeat(64), + title: 'remind me to send the deck', + args: { title: 'Send the deck' }, + source: 'chat' + }) + resolveActionGate('act_1', { kind: 'reject', reason: 'no' }) + await expect(parked).resolves.toEqual({ kind: 'reject', reason: 'no' }) + }) + + it('reject and edit decisions pass through untouched', async () => { + registerHook(HOOKS.actionsProposeApproval, () => true) + const first = gateHost({ action: record() }) + resolveActionGate('act_1', { kind: 'edit', args: { title: 'Send the v2 deck' } }) + await expect(first).resolves.toEqual({ kind: 'edit', args: { title: 'Send the v2 deck' } }) + }) + + it('resolving an unknown action reports false instead of throwing', () => { + expect(resolveActionGate('act_ghost', { kind: 'approve' })).toBe(false) + }) + + it('falls back to the legacy mcp hook when the new one is unregistered', async () => { + const legacy = vi.fn(() => true) + registerHook(HOOKS.legacyMcpProposeApproval, legacy) + const parked = gateHost({ action: record() }) + expect(legacy).toHaveBeenCalled() + resolveActionGate('act_1', { kind: 'approve' }) + await parked + }) +}) + +describe('the park signals', () => { + it('whenActionParked resolves immediately for an already-parked action', async () => { + registerHook(HOOKS.actionsProposeApproval, () => true) + const parked = gateHost({ action: record() }) + await whenActionParked('act_1') // already pending: resolves now + resolveActionGate('act_1', { kind: 'approve' }) + await parked + }) + + it('whenActionParked resolves when the park happens later', async () => { + registerHook(HOOKS.actionsProposeApproval, () => true) + const waiting = whenActionParked('act_1') + const parked = gateHost({ action: record() }) + await waiting + resolveActionGate('act_1', { kind: 'approve' }) + await parked + }) + + it('onGateParked notifies global listeners and unsubscribe stops them', async () => { + registerHook(HOOKS.actionsProposeApproval, () => true) + let fired = 0 + const unsubscribe = onGateParked(() => { + fired += 1 + }) + const first = gateHost({ action: record() }) + expect(fired).toBe(1) + resolveActionGate('act_1', { kind: 'approve' }) + await first + + unsubscribe() + const second = gateHost({ action: record({ id: 'act_2' }) }) + expect(fired).toBe(1) + resolveActionGate('act_2', { kind: 'approve' }) + await second + }) + + it('pendingActionGateCount tracks parks and abandonActionGate drops one', () => { + registerHook(HOOKS.actionsProposeApproval, () => true) + void gateHost({ action: record() }) + expect(pendingActionGateCount()).toBe(1) + expect(abandonActionGate('act_1')).toBe(true) + expect(abandonActionGate('act_1')).toBe(false) + expect(pendingActionGateCount()).toBe(0) + }) +}) diff --git a/src/main/actions/__tests__/native-helper-logic.test.ts b/src/main/actions/__tests__/native-helper-logic.test.ts new file mode 100644 index 00000000..ae2e032d --- /dev/null +++ b/src/main/actions/__tests__/native-helper-logic.test.ts @@ -0,0 +1,117 @@ +/** + * Unit tests for the native-helper invoker's pure logic. Guards the command/response + * contract the Swift helper (scripts/actions-helper/main.swift) and every semantic + * tool share, plus the packaged-vs-dev binary resolution that mirrors ocr.ts. The + * response parser must degrade every malformed shape to a reported { ok: false } so a + * broken helper never throws into the tool loop. + */ +import path from 'path' +import { describe, expect, it } from 'vitest' +import { serializeCommand, helperBinCandidates, parseHelperResponse } from '../native-helper-logic' + +describe('parseHelperResponse truncation', () => { + it('truncates a long invalid line in the reported error', () => { + const long = 'x'.repeat(250) + const res = parseHelperResponse(long) + expect(res.ok).toBe(false) + if (!res.ok) { + expect(res.error.length).toBeLessThan(260) + expect(res.error).toContain('invalid JSON') + } + }) +}) + +describe('serializeCommand', () => { + it('encodes the command and args as a single JSON string', () => { + expect(serializeCommand({ command: 'calendar.createEvent', args: { title: 'Sync' } })).toBe( + '{"command":"calendar.createEvent","args":{"title":"Sync"}}' + ) + }) +}) + +describe('helperBinCandidates', () => { + it('prefers the bundled bin path in a packaged build', () => { + expect( + helperBinCandidates({ + isPackaged: true, + resourcesPath: '/App/Contents/Resources', + cwd: '/ignored', + appPath: '/ignored' + }) + ).toEqual([ + path.join('/App/Contents/Resources', 'bin', 'actions-helper'), + path.join('/App/Contents/Resources', 'actions-helper') + ]) + }) + + it('resolves next to the source in a dev build', () => { + expect( + helperBinCandidates({ + isPackaged: false, + resourcesPath: '/ignored', + cwd: '/repo', + appPath: '/app' + }) + ).toEqual([ + path.join('/repo', 'scripts', 'actions-helper', 'actions-helper'), + path.join('/app', 'scripts', 'actions-helper', 'actions-helper') + ]) + }) +}) + +describe('parseHelperResponse', () => { + it('parses a success response and preserves the result', () => { + expect(parseHelperResponse('{"ok":true,"result":{"id":"E1"}}')).toEqual({ + ok: true, + result: { id: 'E1' } + }) + }) + + it('parses an in-band error response', () => { + expect(parseHelperResponse('{"ok":false,"error":"calendar access was not granted"}')).toEqual({ + ok: false, + error: 'calendar access was not granted' + }) + }) + + it('reads the last non-empty line so a stray leading line does not break parsing', () => { + expect(parseHelperResponse('warming up\n\n{"ok":true,"result":null}\n')).toEqual({ + ok: true, + result: null + }) + }) + + it('reports empty output as an error rather than throwing', () => { + expect(parseHelperResponse(' \n ')).toEqual({ + ok: false, + error: 'actions helper returned no output' + }) + }) + + it('reports invalid JSON as an error and truncates the echoed text', () => { + const res = parseHelperResponse('not json at all') + expect(res.ok).toBe(false) + expect(res).toMatchObject({ error: expect.stringContaining('invalid JSON') }) + }) + + it('rejects a non-object JSON payload', () => { + expect(parseHelperResponse('42')).toEqual({ + ok: false, + error: 'actions helper returned a non-object response' + }) + }) + + it('rejects a recognized-shape-but-missing-ok payload', () => { + expect(parseHelperResponse('{"result":{"id":"E1"}}')).toEqual({ + ok: false, + error: 'actions helper returned an unrecognized response' + }) + }) + + it('substitutes a generic message when ok:false carries no error string', () => { + expect(parseHelperResponse('{"ok":false}')).toEqual({ + ok: false, + error: 'actions helper reported an error' + }) + }) +}) diff --git a/src/main/actions/__tests__/native-helper.test.ts b/src/main/actions/__tests__/native-helper.test.ts new file mode 100644 index 00000000..6dd79b9e --- /dev/null +++ b/src/main/actions/__tests__/native-helper.test.ts @@ -0,0 +1,76 @@ +/** + * The Electron-bound helper invoker, with its two true boundaries mocked: + * electron (packaging context) and child_process (the spawned helper). + * Covers what the dev/e2e paths cannot: candidate resolution misses, the + * non-zero-exit-with-stdout salvage, and the spawn failure - each of which + * must degrade to a reported { ok: false }, never a throw into the loop. + */ +import { afterEach, describe, expect, it, vi } from 'vitest' + +const execFileMock = vi.hoisted(() => vi.fn()) +vi.mock('electron', () => ({ + app: { isPackaged: false, getAppPath: () => '/fake/app' } +})) +vi.mock('child_process', () => ({ execFile: execFileMock })) +vi.mock('fs', async (importOriginal) => { + const real = (await importOriginal()) as typeof import('fs') + return { ...real, default: { ...real, existsSync: (p: string) => existingPaths.has(p) } } +}) + +let existingPaths = new Set() + +// promisify(execFile) consumes the callback-style mock; script it per-case. +type ExecCallback = (error: Error | null, result: { stdout: string; stderr: string }) => void +function scriptExec(behavior: (cmd: string) => { error?: Error & { stdout?: string }; stdout?: string }): void { + execFileMock.mockImplementation( + (bin: string, _args: string[], _opts: unknown, callback: ExecCallback) => { + const out = behavior(bin) + if (out.error) { + callback(out.error, { stdout: out.error.stdout ?? '', stderr: '' }) + return + } + callback(null, { stdout: out.stdout ?? '', stderr: '' }) + } + ) +} + +afterEach(() => { + execFileMock.mockReset() + existingPaths = new Set() +}) + +describe('runNativeAction', () => { + it('reports helper-not-available when no candidate exists, without spawning', async () => { + const { runNativeAction } = await import('../native-helper') + const res = await runNativeAction({ command: 'reminders.list', args: {} }) + expect(res).toEqual({ ok: false, error: 'the native actions helper is not available in this build' }) + expect(execFileMock).not.toHaveBeenCalled() + }) + + it('runs the first existing candidate and parses its response', async () => { + const { runNativeAction } = await import('../native-helper') + existingPaths = new Set([`${process.cwd()}/scripts/actions-helper/actions-helper`]) + scriptExec(() => ({ stdout: '{"ok":true,"result":{"id":"r1"}}\n' })) + const res = await runNativeAction({ command: 'reminders.create', args: { title: 'x' } }) + expect(res).toEqual({ ok: true, result: { id: 'r1' } }) + }) + + it('salvages the response a dying helper printed before its non-zero exit', async () => { + const { runNativeAction } = await import('../native-helper') + existingPaths = new Set([`${process.cwd()}/scripts/actions-helper/actions-helper`]) + const error = Object.assign(new Error('exited 1'), { + stdout: '{"ok":false,"error":"Reminders access denied"}\n' + }) + scriptExec(() => ({ error })) + const res = await runNativeAction({ command: 'reminders.list', args: {} }) + expect(res).toEqual({ ok: false, error: 'Reminders access denied' }) + }) + + it('a spawn failure with no output degrades to the error message', async () => { + const { runNativeAction } = await import('../native-helper') + existingPaths = new Set([`${process.cwd()}/scripts/actions-helper/actions-helper`]) + scriptExec(() => ({ error: Object.assign(new Error('spawn EPERM'), { stdout: '' }) })) + const res = await runNativeAction({ command: 'reminders.list', args: {} }) + expect(res).toEqual({ ok: false, error: 'spawn EPERM' }) + }) +}) diff --git a/src/main/actions/__tests__/semantic-rail-win.test.ts b/src/main/actions/__tests__/semantic-rail-win.test.ts new file mode 100644 index 00000000..904d5dfa --- /dev/null +++ b/src/main/actions/__tests__/semantic-rail-win.test.ts @@ -0,0 +1,223 @@ +/** + * The Windows semantic rail through injected boundaries: the PowerShell + * runner, the opener, and the Graph fallback port. Guards the local-first + * contract (Outlook COM first; Graph only when Outlook is genuinely absent + * AND the port says it is signed in), the honest refusals, and - with the + * mac rail beside it - the DeviceController swap with zero caller changes. + */ +import { describe, expect, it, vi } from 'vitest' +import type { ActionRecord } from '@offgrid/use' +import { + buildOutlookScript, + isOutlookUnavailable, + makeWindowsSemanticRailExecutor, + psQuote, + type GraphPort +} from '../semantic-rail-win' +import { makeSemanticRailExecutor } from '../semantic-rail' + +const action = (type: string, args: Record = {}) => + ({ type, args }) as ActionRecord + +const ok = { ok: true as const, result: {} } +const graphPort = (available = true): GraphPort & { calls: string[] } => { + const calls: string[] = [] + return { + calls, + available: () => available, + async createEvent() { + calls.push('createEvent') + return ok + }, + async createTask() { + calls.push('createTask') + return ok + }, + async sendMail() { + calls.push('sendMail') + return ok + } + } +} + +describe('psQuote', () => { + it('single-quotes and doubles embedded quotes', () => { + expect(psQuote("Ali's deck")).toBe("'Ali''s deck'") + expect(psQuote(undefined)).toBe("''") + }) +}) + +describe('buildOutlookScript', () => { + it('calendar: appointment with explicit end and notes', () => { + const script = buildOutlookScript('calendar', { + title: "Q3 'final' sync", + start: '2026-08-15T09:00:00', + end: '2026-08-15T10:00:00', + notes: 'bring the deck' + }) + expect(script).toContain('CreateItem(1)') + expect(script).toContain("$i.Subject = 'Q3 ''final'' sync'") + expect(script).toContain("[datetime]'2026-08-15T09:00:00'") + expect(script).toContain("$i.End = [datetime]'2026-08-15T10:00:00'") + expect(script).toContain("$i.Body = 'bring the deck'") + expect(script).toContain('ConvertTo-Json -Compress') + expect(script).toContain('catch') + }) + + it('calendar: a missing end defaults to one hour (the helper convention)', () => { + const script = buildOutlookScript('calendar', { title: 'x', start: '2026-08-15T09:00:00' }) + expect(script).toContain('$i.End = $i.Start.AddHours(1)') + }) + + it('reminder: a task with optional due', () => { + const script = buildOutlookScript('reminder', { title: 'Send the deck', due: '2026-08-15T18:00:00' }) + expect(script).toContain('CreateItem(3)') + expect(script).toContain("$i.DueDate = [datetime]'2026-08-15T18:00:00'") + const noDue = buildOutlookScript('reminder', { title: 'Send the deck' }) + expect(noDue).not.toContain('DueDate') + }) + + it('email: a mail item that Sends (lands in the local outbox, syncs later)', () => { + const script = buildOutlookScript('email', { to: 'ali@x.test', subject: 's', body: 'b' }) + expect(script).toContain('CreateItem(0)') + expect(script).toContain("$i.To = 'ali@x.test'") + expect(script).toContain('$i.Send()') + }) +}) + +describe('isOutlookUnavailable', () => { + it('matches the COM-not-registered shapes and nothing else', () => { + expect(isOutlookUnavailable('80040154 Class not registered')).toBe(true) + expect(isOutlookUnavailable('Cannot create a COM object')).toBe(true) + expect(isOutlookUnavailable("Retrieving the COM class factory for Outlook.Application failed")).toBe(true) + expect(isOutlookUnavailable('The operation was cancelled by the user')).toBe(false) + }) +}) + +describe('makeWindowsSemanticRailExecutor', () => { + it('open goes through the opener', async () => { + const openUrl = vi.fn(async () => ok) + const execute = makeWindowsSemanticRailExecutor({ runPs: vi.fn(), openUrl }) + expect(await execute(action('open', { url: 'https://x.test' }))).toEqual({ ok: true }) + expect(openUrl).toHaveBeenCalledWith('https://x.test') + }) + + it('message is refused honestly - macOS-only in this release', async () => { + const execute = makeWindowsSemanticRailExecutor({ runPs: vi.fn(), openUrl: vi.fn() }) + const result = await execute(action('message', { to: 'x', text: 'hi' })) + expect(result.ok).toBe(false) + expect(result.detail).toMatch(/macOS-only/) + }) + + it.each([['lookup'], ['file_share'], ['web_task']])('%s has no Windows mapping', async (type) => { + const runPs = vi.fn() + const execute = makeWindowsSemanticRailExecutor({ runPs, openUrl: vi.fn() }) + const result = await execute(action(type)) + expect(result.ok).toBe(false) + expect(runPs).not.toHaveBeenCalled() + }) + + it('a local Outlook success is the happy path - Graph is never consulted', async () => { + const graph = graphPort() + const execute = makeWindowsSemanticRailExecutor({ + runPs: vi.fn(async () => ok), + openUrl: vi.fn(), + graph + }) + expect(await execute(action('calendar', { title: 'x', start: 's' }))).toEqual({ ok: true }) + expect(graph.calls).toEqual([]) + }) + + it('an ordinary Outlook error passes through without touching Graph', async () => { + const graph = graphPort() + const execute = makeWindowsSemanticRailExecutor({ + runPs: vi.fn(async () => ({ ok: false as const, error: 'The item could not be saved' })), + openUrl: vi.fn(), + graph + }) + const result = await execute(action('reminder', { title: 'x' })) + expect(result).toEqual({ ok: false, detail: 'The item could not be saved' }) + expect(graph.calls).toEqual([]) + }) + + it('Outlook absent + no Graph port: the honest failure names both', async () => { + const execute = makeWindowsSemanticRailExecutor({ + runPs: vi.fn(async () => ({ ok: false as const, error: '80040154 Class not registered' })), + openUrl: vi.fn() + }) + const result = await execute(action('email', { to: 'a@b.c' })) + expect(result.ok).toBe(false) + expect(result.detail).toMatch(/local Outlook is not available/) + }) + + it('Outlook absent + Graph signed out: Graph is not called', async () => { + const graph = graphPort(false) + const execute = makeWindowsSemanticRailExecutor({ + runPs: vi.fn(async () => ({ ok: false as const, error: '80040154' })), + openUrl: vi.fn(), + graph + }) + const result = await execute(action('email', { to: 'a@b.c' })) + expect(result.ok).toBe(false) + expect(graph.calls).toEqual([]) + }) + + it.each([ + ['calendar', 'createEvent'], + ['reminder', 'createTask'], + ['email', 'sendMail'] + ])('Outlook absent + Graph available: %s falls back to %s', async (type, method) => { + const graph = graphPort() + const execute = makeWindowsSemanticRailExecutor({ + runPs: vi.fn(async () => ({ ok: false as const, error: '80040154' })), + openUrl: vi.fn(), + graph + }) + expect(await execute(action(type, { title: 'x', start: 's', to: 't' }))).toEqual({ ok: true }) + expect(graph.calls).toEqual([method]) + }) + + it('a Graph failure is labeled as the online path failing', async () => { + const graph = graphPort() + graph.sendMail = async () => ({ ok: false as const, error: '401 unauthorized' }) + const execute = makeWindowsSemanticRailExecutor({ + runPs: vi.fn(async () => ({ ok: false as const, error: '80040154' })), + openUrl: vi.fn(), + graph + }) + const result = await execute(action('email', { to: 'a@b.c' })) + expect(result.detail).toMatch(/Microsoft Graph \(online\) failed: 401/) + }) + + it('a throwing boundary is caught - the executor never throws', async () => { + const execute = makeWindowsSemanticRailExecutor({ + runPs: vi.fn(async () => { + throw new Error('powershell missing') + }), + openUrl: vi.fn() + }) + const result = await execute(action('calendar', { title: 'x', start: 's' })) + expect(result.ok).toBe(false) + expect(result.detail).toMatch(/powershell missing/) + }) +}) + +describe('the DeviceController swap (DSP)', () => { + it('one dispatch drives either platform rail with zero caller changes', async () => { + const macExecute = makeSemanticRailExecutor(async () => ({ ok: true, result: {} })) + const winExecute = makeWindowsSemanticRailExecutor({ + runPs: async () => ok, + openUrl: async () => ok + }) + // Written once; never mentions a platform. Swapping the rail is a + // constructor argument, not a code change - the seam under test. + const dispatch = async ( + execute: (a: ActionRecord) => Promise<{ ok: boolean; detail?: string }>, + a: ActionRecord + ) => execute(a) + + const reminder = action('reminder', { title: 'Send the deck' }) + expect((await dispatch(macExecute, reminder)).ok).toBe(true) + expect((await dispatch(winExecute, reminder)).ok).toBe(true) + }) +}) diff --git a/src/main/actions/__tests__/semantic-rail.test.ts b/src/main/actions/__tests__/semantic-rail.test.ts new file mode 100644 index 00000000..c7eb97c2 --- /dev/null +++ b/src/main/actions/__tests__/semantic-rail.test.ts @@ -0,0 +1,94 @@ +/** + * The semantic rail's mapping and executor, through an injected runner. + * Guards the Action-type -> helper-verb contract: every mapped type reaches + * exactly its verb with args passed through, and everything unmapped is + * refused before the helper is ever invoked. + */ +import { describe, expect, it, vi } from 'vitest' +import { mapActionToCommand, makeSemanticRailExecutor } from '../semantic-rail' +import type { NativeActionCommand } from '../native-helper-logic' + +const action = (type: string, args: Record = {}) => + ({ type, args }) as Parameters[0] + +describe('mapActionToCommand', () => { + it.each([ + ['calendar', 'calendar.createEvent', { title: 'Sync', start: 's', end: 'e' }], + ['reminder', 'reminders.create', { title: 'Send the deck', due: '18:00' }], + ['message', 'messages.send', { to: 'Ali', text: 'hi' }], + ['email', 'mail.send', { to: 'ali@example.com', subject: 's', body: 'b' }], + ['open', 'open_url', { url: 'https://example.com' }] + ] as const)('maps %s to %s with args passed through', (type, command, args) => { + const mapped = mapActionToCommand(action(type, { ...args })) + expect(mapped).toEqual({ ok: true, command: { command, args } }) + }) + + it.each([ + ['contacts', 'contacts.search'], + ['calendar', 'calendar.listEvents'], + ['reminders', 'reminders.list'] + ])('maps lookup kind %s to %s and drops the discriminator', (kind, command) => { + const mapped = mapActionToCommand(action('lookup', { kind, query: 'ali' })) + expect(mapped).toEqual({ ok: true, command: { command, args: { query: 'ali' } } }) + }) + + it('refuses a lookup with no kind at all', () => { + const mapped = mapActionToCommand(action('lookup', { query: 'x' })) + expect(mapped.ok).toBe(false) + }) + + it('refuses a lookup with an unknown kind', () => { + const mapped = mapActionToCommand(action('lookup', { kind: 'photos' })) + expect(mapped.ok).toBe(false) + if (!mapped.ok) { + expect(mapped.error).toMatch(/photos/) + } + }) + + it.each([['file_share'], ['web_task']])('refuses %s - it belongs to another rail', (type) => { + const mapped = mapActionToCommand(action(type)) + expect(mapped.ok).toBe(false) + if (!mapped.ok) { + expect(mapped.error).toMatch(/no mapping/) + } + }) +}) + +describe('makeSemanticRailExecutor', () => { + const record = (type: string, args: Record = {}) => + ({ type, args }) as Parameters>[0] + + it('executes a mapped action through the runner and reports ok', async () => { + const run = vi.fn(async (_cmd: NativeActionCommand) => ({ ok: true as const, result: null })) + const execute = makeSemanticRailExecutor(run) + const result = await execute(record('reminder', { title: 'x' })) + expect(result).toEqual({ ok: true }) + expect(run).toHaveBeenCalledWith({ command: 'reminders.create', args: { title: 'x' } }) + }) + + it('a refused mapping never reaches the helper', async () => { + const run = vi.fn() + const execute = makeSemanticRailExecutor(run) + const result = await execute(record('web_task')) + expect(result.ok).toBe(false) + expect(run).not.toHaveBeenCalled() + }) + + it('a helper-reported failure becomes a result with its detail', async () => { + const execute = makeSemanticRailExecutor(async () => ({ + ok: false as const, + error: 'Calendar access denied' + })) + const result = await execute(record('calendar', { title: 'x' })) + expect(result).toEqual({ ok: false, detail: 'Calendar access denied' }) + }) + + it('a throwing runner is caught - execute never throws', async () => { + const execute = makeSemanticRailExecutor(async () => { + throw new Error('spawn failed') + }) + const result = await execute(record('open', { url: 'x' })) + expect(result.ok).toBe(false) + expect(result.detail).toMatch(/spawn failed/) + }) +}) diff --git a/src/main/actions/__tests__/use-driver.test.ts b/src/main/actions/__tests__/use-driver.test.ts new file mode 100644 index 00000000..a0bc47b4 --- /dev/null +++ b/src/main/actions/__tests__/use-driver.test.ts @@ -0,0 +1,75 @@ +/** + * The SqlDriver adapter's routing logic, against a structural fake. The + * real-SQLite behaviour is proven in the dbtest suite; these cover the + * branch matrix purely: reader vs non-reader statements through run/get/all. + */ +import { describe, expect, it } from 'vitest' +import { makeUseDriver, type DatabaseLike, type StatementLike } from '../use-driver' + +function fakeDb(reader: boolean, rows: unknown[] = [{ id: 1 }, { id: 2 }]): { + db: DatabaseLike + calls: string[] +} { + const calls: string[] = [] + const statement: StatementLike = { + reader, + run: (...params: unknown[]) => { + calls.push(`run:${params.length}`) + return { changes: 7 } + }, + get: (...params: unknown[]) => { + calls.push(`get:${params.length}`) + return rows[0] + }, + all: (...params: unknown[]) => { + calls.push(`all:${params.length}`) + return rows + } + } + return { db: { prepare: () => statement }, calls } +} + +describe('makeUseDriver', () => { + it('run on a non-reader statement reports the write count', async () => { + const { db, calls } = fakeDb(false) + const driver = makeUseDriver(db) + expect(await driver.run('UPDATE x SET y = ?', [1])).toEqual({ changes: 7 }) + expect(calls).toEqual(['run:1']) + }) + + it('run on a reader statement (UPDATE ... RETURNING) counts returned rows as changes', async () => { + const { db, calls } = fakeDb(true) + const driver = makeUseDriver(db) + expect(await driver.run('UPDATE x ... RETURNING *', [])).toEqual({ changes: 2 }) + expect(calls).toEqual(['all:0']) + }) + + it('get on a reader statement returns the row', async () => { + const { db } = fakeDb(true, [{ n: 42 }]) + const driver = makeUseDriver(db) + expect(await driver.get('SELECT n FROM x')).toEqual({ n: 42 }) + }) + + it('get on a non-reader statement executes it and returns undefined', async () => { + const { db, calls } = fakeDb(false) + const driver = makeUseDriver(db) + expect(await driver.get('DELETE FROM x WHERE id = ?', [9])).toBeUndefined() + expect(calls).toEqual(['run:1']) + }) + + it('all returns every row with params applied', async () => { + const { db, calls } = fakeDb(true, [{ a: 1 }, { a: 2 }, { a: 3 }]) + const driver = makeUseDriver(db) + expect(await driver.all('SELECT * FROM x WHERE a > ?', [0])).toHaveLength(3) + expect(calls).toEqual(['all:1']) + }) + + it('defaults params to empty across all three methods', async () => { + const { db, calls } = fakeDb(true) + const driver = makeUseDriver(db) + await driver.run('SELECT 1') + await driver.get('SELECT 1') + await driver.all('SELECT 1') + expect(calls).toEqual(['all:0', 'get:0', 'all:0']) + }) +}) diff --git a/src/main/actions/__tests__/use-worker.test.ts b/src/main/actions/__tests__/use-worker.test.ts new file mode 100644 index 00000000..627278d5 --- /dev/null +++ b/src/main/actions/__tests__/use-worker.test.ts @@ -0,0 +1,107 @@ +/** + * The park-aware drain loop, against scripted engine and park-signal fakes. + * The property under test: an action waiting on a human never blocks the + * queue - the loop moves on, and the parked tick's outcome still reaches + * its waiter when the gate finally resolves. + */ +import { describe, expect, it } from 'vitest' +import type { TickOutcome } from '@offgrid/use' +import { createActionWorker, type EngineLike, type ParkSignal } from '../use-worker' + +const done = (id: string): TickOutcome => + ({ id, outcome: 'done', record: { id } as never }) as TickOutcome + +function makePark() { + const listeners = new Set<() => void>() + const signal: ParkSignal = { + onParked(listener) { + listeners.add(listener) + return () => listeners.delete(listener) + } + } + return { signal, fire: () => listeners.forEach((l) => l()) } +} + +const flush = () => new Promise((resolve) => setTimeout(resolve, 10)) + +describe('createActionWorker', () => { + it('drains outcomes to their waiters and stops when nothing is due', async () => { + const script: Array = [done('a1'), done('a2'), undefined] + const engine: EngineLike = { tick: async () => script.shift() } + const { signal } = makePark() + const worker = createActionWorker(engine, signal) + + const w1 = worker.waitForOutcome('a1', 1000) + const w2 = worker.waitForOutcome('a2', 1000) + worker.kick() + + expect((await w1)?.id).toBe('a1') + expect((await w2)?.id).toBe('a2') + await flush() + expect(worker.draining()).toBe(false) + }) + + it('a parked tick does not block the loop, and its outcome still lands later', async () => { + const { signal, fire } = makePark() + let resolveParkedTick: ((o: TickOutcome) => void) | undefined + let call = 0 + const engine: EngineLike = { + tick: async () => { + call += 1 + if (call === 1) { + // This action reaches the gate and waits on a human. + return new Promise((resolve) => { + resolveParkedTick = resolve + queueMicrotask(fire) // the gate host announces the park + }) + } + if (call === 2) { + return done('quick') + } + return undefined + } + } + const worker = createActionWorker(engine, signal) + const parked = worker.waitForOutcome('parked', 1000) + const quick = worker.waitForOutcome('quick', 1000) + worker.kick() + + expect((await quick)?.id).toBe('quick') + // The human decides much later; the parked outcome still arrives. + resolveParkedTick?.(done('parked')) + expect((await parked)?.id).toBe('parked') + }) + + it('waitForOutcome times out to undefined and drops its waiter', async () => { + const engine: EngineLike = { tick: async () => undefined } + const worker = createActionWorker(engine, makePark().signal) + const result = await worker.waitForOutcome('ghost', 20) + expect(result).toBeUndefined() + }) + + it('kick while draining does not start a second drain', async () => { + let ticks = 0 + let release: (() => void) | undefined + const engine: EngineLike = { + tick: async () => { + ticks += 1 + if (ticks === 1) { + await new Promise((resolve) => { + release = resolve + }) + return done('slow') + } + return undefined + } + } + const worker = createActionWorker(engine, makePark().signal) + worker.kick() + worker.kick() + worker.kick() + await flush() + expect(ticks).toBe(1) + release?.() + await flush() + expect(worker.draining()).toBe(false) + }) +}) diff --git a/src/main/actions/__tests__/verification.test.ts b/src/main/actions/__tests__/verification.test.ts new file mode 100644 index 00000000..a2c6365d --- /dev/null +++ b/src/main/actions/__tests__/verification.test.ts @@ -0,0 +1,103 @@ +/** + * Read-back verification, through a scripted helper boundary. Everything + * fails closed: helper errors, malformed results, and missing args verify + * as false so the retry policy - not optimism - decides what happens next. + */ +import { describe, expect, it, vi } from 'vitest' +import type { ActionRecord } from '@offgrid/use' +import { calendarVerifyWindow, listContainsTitle, makeReadBackVerifiers } from '../verification' +import type { NativeActionCommand } from '../native-helper-logic' + +const action = (type: string, args: Record) => ({ type, args }) as ActionRecord + +describe('listContainsTitle', () => { + it('matches an exact title in the helper shape', () => { + const result = { reminders: [{ id: 'r1', title: 'Send the deck' }] } + expect(listContainsTitle(result, 'reminders', 'Send the deck')).toBe(true) + expect(listContainsTitle(result, 'reminders', 'send the deck')).toBe(false) + }) + + it('fails closed on malformed shapes', () => { + expect(listContainsTitle(null, 'reminders', 'x')).toBe(false) + expect(listContainsTitle({ reminders: 'nope' }, 'reminders', 'x')).toBe(false) + expect(listContainsTitle({ events: [{ title: 'x' }] }, 'reminders', 'x')).toBe(false) + expect(listContainsTitle({ reminders: [null, 42] }, 'reminders', 'x')).toBe(false) + }) +}) + +describe('calendarVerifyWindow', () => { + it('pads the event range by a minute on both sides', () => { + const window = calendarVerifyWindow({ + start: '2026-08-14T09:30:00.000Z', + end: '2026-08-14T10:00:00.000Z' + }) + expect(window).toEqual({ + start: '2026-08-14T09:29:00.000Z', + end: '2026-08-14T10:01:00.000Z' + }) + }) + + it('defaults a missing end to one hour after start (the helper default)', () => { + const window = calendarVerifyWindow({ start: '2026-08-14T09:30:00.000Z' }) + expect(window?.end).toBe('2026-08-14T10:31:00.000Z') + }) + + it('an unparseable start means nothing sane to verify against', () => { + expect(calendarVerifyWindow({ start: 'whenever' })).toBeUndefined() + expect(calendarVerifyWindow({})).toBeUndefined() + }) +}) + +describe('makeReadBackVerifiers', () => { + it('a reminder verifies true when the list shows it, false when absent', async () => { + const run = vi.fn(async () => ({ + ok: true as const, + result: { reminders: [{ title: 'Send the deck' }] } + })) + const verifiers = makeReadBackVerifiers(run) + expect(await verifiers.reminder(action('reminder', { title: 'Send the deck' }))).toBe(true) + expect(await verifiers.reminder(action('reminder', { title: 'Something else' }))).toBe(false) + expect(run).toHaveBeenCalledWith({ command: 'reminders.list', args: {} }) + }) + + it('a calendar event verifies inside its padded window', async () => { + const seen: NativeActionCommand[] = [] + const run = vi.fn(async (cmd: NativeActionCommand) => { + seen.push(cmd) + return { ok: true as const, result: { events: [{ title: 'Standup' }] } } + }) + const verifiers = makeReadBackVerifiers(run) + const verified = await verifiers.calendar( + action('calendar', { title: 'Standup', start: '2026-08-14T09:30:00.000Z' }) + ) + expect(verified).toBe(true) + expect(seen[0]?.command).toBe('calendar.listEvents') + expect(seen[0]?.args).toEqual({ + start: '2026-08-14T09:29:00.000Z', + end: '2026-08-14T10:31:00.000Z' + }) + }) + + it('a calendar event with an unparseable start verifies false without listing', async () => { + const run = vi.fn() + const verifiers = makeReadBackVerifiers(run) + expect(await verifiers.calendar(action('calendar', { title: 'x', start: 'whenever' }))).toBe(false) + expect(run).not.toHaveBeenCalled() + }) + + it('a helper failure verifies false, never optimistic', async () => { + const run = vi.fn(async () => ({ ok: false as const, error: 'Reminders access denied' })) + const verifiers = makeReadBackVerifiers(run) + expect(await verifiers.reminder(action('reminder', { title: 'x' }))).toBe(false) + }) + + it('a missing or empty title fails closed without calling the helper', async () => { + const run = vi.fn() + const verifiers = makeReadBackVerifiers(run) + expect(await verifiers.reminder(action('reminder', {}))).toBe(false) + expect(await verifiers.calendar(action('calendar', { start: '2026-08-14T09:30:00.000Z' }))).toBe( + false + ) + expect(run).not.toHaveBeenCalled() + }) +}) diff --git a/src/main/actions/approval.ts b/src/main/actions/approval.ts new file mode 100644 index 00000000..c7af5d0b --- /dev/null +++ b/src/main/actions/approval.ts @@ -0,0 +1,69 @@ +// Transport-agnostic action-approval seam (core). Every executor that acts on the +// user's behalf — MCP connectors today, computer/GUI actions and the agent browser +// next — classifies each action's risk and, when it is consequential, offers it to +// the approval hook BEFORE doing it. Pro registers the hook to route the action +// through its approval queue + audit log; the free build registers nothing, so the +// action just runs (unchanged free behaviour). +// +// This replaces the MCP-specific `mcp:proposeApproval` hook: the old one carried a +// connector-shaped payload and derived risk from a tool-name regex, neither of +// which generalises to a GUI click (a click is always a write; a screenshot never +// is). Risk is classified per executor via its own riskOf(); the shape below is the +// one thing every executor shares. + +import { callHook, hasHook, HOOKS } from '../bootstrap/hookRegistry' + +/** How consequential an action is, independent of which executor produced it. + * - read: observes only, never changes the world (a screenshot, a list call) + * - navigate: moves focus/location without committing (open a URL, scroll) + * - mutate: changes state, usually recoverable (send a message, create an event) + * - irreversible: cannot be undone (delete, pay, submit, create an account) + * read/navigate run freely; mutate/irreversible are offered for approval. */ +export type ActionRisk = 'read' | 'navigate' | 'mutate' | 'irreversible' + +/** Which executor raised the action — lets the approval UI and audit log group and + * label without branching on executor-specific fields. + * - mcp: a connector tool call + * - native: a semantic OS action (EventKit, AppleScript, Shortcuts) — the rail-1 path + * - browser: the embedded agent browser + * - computer: GUI automation (accessibility tree + synthetic input) */ +export type ActionKind = 'mcp' | 'native' | 'browser' | 'computer' + +export interface ActionApprovalRequest { + kind: ActionKind + /** One-line, user-facing summary of what will happen. */ + title: string + /** Longer context for the approval card (arguments, source surface). */ + detail: string + risk: ActionRisk + /** Structured arguments, passed through to the executor on approval. */ + args: Record + /** Where the action originated (e.g. 'chat', a skill id). */ + source: string + /** Executor-specific fields (connectorId/tool for mcp, selector for browser). + * Left open so the seam never needs to know each executor's payload shape. */ + [extra: string]: unknown +} + +/** mutate and irreversible actions gate; read and navigate run freely. The single + * source of truth for the gating rule — executors and tests both call this rather + * than re-encoding the set. */ +export function shouldGate(risk: ActionRisk): boolean { + return risk === 'mutate' || risk === 'irreversible' +} + +/** Offer an action to the approval hook. Returns true when it was queued (the + * caller must NOT execute), false when a handler ran but did not queue it, and + * undefined when nothing is listening (free build — execute now). + * + * Falls back to the legacy `mcp:proposeApproval` hook so a pro build that has not + * yet migrated keeps gating MCP writes instead of silently running them. hasHook + * distinguishes "new handler present" from "new handler returned undefined", so a + * registered new handler is always authoritative and the legacy path is only used + * when the new name is genuinely unregistered. */ +export function proposeActionApproval(request: ActionApprovalRequest): boolean | undefined { + if (hasHook(HOOKS.actionsProposeApproval)) { + return callHook(HOOKS.actionsProposeApproval, request) + } + return callHook(HOOKS.legacyMcpProposeApproval, request) +} diff --git a/src/main/actions/emit.ts b/src/main/actions/emit.ts new file mode 100644 index 00000000..5eaa55cc --- /dev/null +++ b/src/main/actions/emit.ts @@ -0,0 +1,149 @@ +/** + * Emission hardening (R1 box 12) - how a weak local model reliably produces + * a valid ActionProposal. + * + * Three layers, per the porting research: + * 1. Constrain: actionProposalJsonSchema() goes to llama-server as + * grammar-constrained response_format, so a conforming decode CANNOT be + * shaped wrong. (The schema is not injected into the prompt - the prompt + * builder must still describe the action types.) + * 2. Coerce (SAP, ported idea from BAML's schema-aligned parsing): when raw + * output arrives anyway - fenced, wrapped in prose, trailing commas, + * unquoted keys - deterministic repairs produce candidates and the first + * one that passes the fail-closed schema wins. Repairs only ever ADD a + * candidate; they never mutate the original, so a bad repair cannot turn + * a valid emission into a different one. + * 3. Retry (Instructor pattern): emitActionProposal asks again with the + * validation error fed back, bounded. An unrepairable emission is + * rejected, never guessed. + * + * Pure module: no Electron, the asker is injected. + */ +import { parseActionProposal, RISK_CLASSES, type ActionProposal, type ActionType } from '@offgrid/use' + +/** + * The wire schema for llama-server's response_format. `type` is constrained + * to the HANDLERS ACTUALLY REGISTERED, not the full vocabulary - the model + * cannot propose an action this build cannot execute. + */ +export function actionProposalJsonSchema(types: readonly ActionType[]): Record { + return { + type: 'object', + properties: { + type: { type: 'string', enum: [...types] }, + intent: { type: 'string', minLength: 1 }, + args: { type: 'object', additionalProperties: true }, + risk: { type: 'string', enum: [...RISK_CLASSES] }, + triggerAt: { type: 'integer', minimum: 1 } + }, + required: ['type', 'intent', 'args', 'risk'], + additionalProperties: false + } +} + +/** The first balanced {...} in the text, respecting strings and escapes. */ +export function extractBalancedObject(text: string): string | undefined { + const start = text.indexOf('{') + if (start === -1) { + return undefined + } + let depth = 0 + let inString = false + let escaped = false + for (let i = start; i < text.length; i++) { + const ch = text[i] + if (inString) { + if (escaped) { + escaped = false + } else if (ch === '\\') { + escaped = true + } else if (ch === '"') { + inString = false + } + continue + } + if (ch === '"') { + inString = true + } else if (ch === '{') { + depth += 1 + } else if (ch === '}') { + depth -= 1 + if (depth === 0) { + return text.slice(start, i + 1) + } + } + } + return undefined +} + +const stripFences = (text: string): string => + text.replace(/```[a-zA-Z]*\n?/g, '').replace(/```/g, '') + +const dropTrailingCommas = (text: string): string => text.replace(/,(\s*[}\]])/g, '$1') + +/** Quote bare object keys - a heuristic repair, only ever an extra candidate. */ +const quoteBareKeys = (text: string): string => + text.replace(/([{,]\s*)([A-Za-z_][A-Za-z0-9_]*)(\s*:)/g, '$1"$2"$3') + +/** + * Repair candidates in trust order: the raw text first, then progressively + * repaired variants. Deduped; each is tried against the fail-closed schema. + */ +export function extractCandidates(raw: string): string[] { + const candidates: string[] = [raw.trim()] + const unfenced = stripFences(raw).trim() + candidates.push(unfenced) + const balanced = extractBalancedObject(unfenced) + if (balanced) { + candidates.push(balanced) + candidates.push(dropTrailingCommas(balanced)) + candidates.push(quoteBareKeys(dropTrailingCommas(balanced))) + } + return [...new Set(candidates)].filter((c) => c.length > 0) +} + +export type EmissionResult = + | { ok: true; proposal: ActionProposal } + | { ok: false; error: string } + +/** Parse one raw emission through the repair ladder. Fail closed. */ +export function parseEmission(raw: string): EmissionResult { + let lastError = 'no JSON object found in the output' + for (const candidate of extractCandidates(raw)) { + let value: unknown + try { + value = JSON.parse(candidate) + } catch { + continue + } + const parsed = parseActionProposal(value) + if (parsed.ok) { + return { ok: true, proposal: parsed.value } + } + lastError = parsed.error + } + return { ok: false, error: lastError } +} + +/** + * Ask, parse, and on failure ask again with the error fed back - bounded. + * The asker owns the model call (and the response_format constraint); this + * owns the loop and the discipline that exhaustion means rejection. + */ +export async function emitActionProposal( + ask: (feedback?: string) => Promise, + options: { maxAttempts?: number } = {} +): Promise { + const maxAttempts = options.maxAttempts ?? 2 + let feedback: string | undefined + let last: EmissionResult = { ok: false, error: 'no attempts were made' } + for (let attempt = 0; attempt < maxAttempts; attempt++) { + const raw = await ask(feedback) + last = parseEmission(raw) + if (last.ok) { + return last + } + feedback = `The last output was not a valid action: ${last.error}. Reply with ONLY the corrected JSON object, nothing else.` + } + return last +} diff --git a/src/main/actions/gate-host.ts b/src/main/actions/gate-host.ts new file mode 100644 index 00000000..4c9bcb93 --- /dev/null +++ b/src/main/actions/gate-host.ts @@ -0,0 +1,129 @@ +/** + * The gate host - the engine's approval callback, wired to the existing + * actions:proposeApproval seam (R1 box 11). + * + * Two contracts meet here. The engine's gate AWAITS a decision (approve / + * edit / reject) bound to the exact payload. The app's approval seam is + * fire-and-queue: proposeActionApproval offers the action to the pro + * approval queue and reports queued / not-queued / nobody-listening. The + * bridge: propose with the action's id and payload hash on the request, + * then park the decision in a pending registry that the approval UI (pro's + * queue, or core's card) resolves via resolveActionGate(actionId, decision). + * + * Free build: nothing listens, so mutations keep the unchanged free + * behaviour and run (the engine still verifies and journals them). + * + * Note on leases: tick() holds the queue lease while awaiting a human. On a + * single-worker desktop that is safe - and if the app quits first, the + * pending map dies with the process while the Action survives in the DB at + * awaiting_approval, so the next launch re-offers it. Nothing is lost. + */ +import type { ActionRecord, GateDecision, Rail } from '@offgrid/use' +import { proposeActionApproval, type ActionKind } from './approval' + +/** The engine's rails, translated to the approval UI's executor kinds. */ +export function railToKind(rail: Rail | undefined): ActionKind { + switch (rail) { + case 'browser': + return 'browser' + case 'accessibility': + case 'vision': + return 'computer' + case 'semantic': + default: + return 'native' + } +} + +const pending = new Map void>() +const parkedWaiters = new Map void>>() + +/** + * Resolves as soon as the action parks at the gate (immediately when it is + * already parked). The chat tool races this against the action's outcome to + * answer "pending approval" instead of blocking on a human. + */ +export function whenActionParked(actionId: string): Promise { + if (pending.has(actionId)) { + return Promise.resolve() + } + return new Promise((resolve) => { + const waiters = parkedWaiters.get(actionId) ?? [] + waiters.push(resolve) + parkedWaiters.set(actionId, waiters) + }) +} + +const parkListeners = new Set<() => void>() + +/** Global "an action just parked at the gate" signal - the worker's cue to + * move on to the next due message instead of blocking on a human. */ +export function onGateParked(listener: () => void): () => void { + parkListeners.add(listener) + return () => parkListeners.delete(listener) +} + +function notifyParked(actionId: string): void { + const waiters = parkedWaiters.get(actionId) + if (waiters) { + parkedWaiters.delete(actionId) + for (const resolve of waiters) { + resolve() + } + } + for (const listener of parkListeners) { + listener() + } +} + +/** + * Called by the approval surface (IPC from the card, or pro's queue) with + * the human's verdict. False when the id is unknown - the decision may have + * arrived after a restart cleared the in-memory registry; the action will + * be re-offered on its next tick. + */ +export function resolveActionGate(actionId: string, decision: GateDecision): boolean { + const resolve = pending.get(actionId) + if (!resolve) { + return false + } + pending.delete(actionId) + resolve(decision) + return true +} + +/** How many actions are parked waiting on a human - a health surface. */ +export function pendingActionGateCount(): number { + return pending.size +} + +/** Drop a parked decision (tests, and future cancel-from-UI). */ +export function abandonActionGate(actionId: string): boolean { + return pending.delete(actionId) +} + +/** The GateCallback the engine host is constructed with. */ +export async function gateHost({ action }: { action: ActionRecord }): Promise { + const queued = proposeActionApproval({ + kind: railToKind(action.rail), + title: action.intent, + detail: JSON.stringify(action.args, null, 2), + risk: action.risk, + args: action.args, + source: action.source, + // Engine-specific fields the approval card needs to resolve the gate + // and to show exactly what was bound. + actionId: action.id, + actionType: action.type, + payloadHash: action.payloadHash + }) + if (queued !== true) { + // Nothing listening (free build), or the handler chose not to queue it: + // the unchanged behaviour is to run. The engine still verifies. + return { kind: 'approve' } + } + return new Promise((resolve) => { + pending.set(action.id, resolve) + notifyParked(action.id) + }) +} diff --git a/src/main/actions/native-helper-logic.ts b/src/main/actions/native-helper-logic.ts new file mode 100644 index 00000000..b05f4635 --- /dev/null +++ b/src/main/actions/native-helper-logic.ts @@ -0,0 +1,85 @@ +// Pure logic for the native actions helper invoker (no Electron, so it is unit +// testable). The Electron-bound wrapper in native-helper.ts resolves the binary and +// runs it; everything that can be reasoned about without spawning a process lives +// here: the command/response contract, binary-path candidates, and response parsing. + +import path from 'path' + +/** A command sent to the native helper: one namespaced action plus its arguments, + * e.g. { command: 'calendar.createEvent', args: { title, start, end } }. */ +export interface NativeActionCommand { + command: string + args: Record +} + +/** The helper's reply. It always exits 0 and reports handled failures in-band, so a + * denied permission or a bad argument is a normal { ok: false } result, not a throw. */ +export type NativeActionResponse = { ok: true; result: unknown } | { ok: false; error: string } + +export function serializeCommand(cmd: NativeActionCommand): string { + return JSON.stringify(cmd) +} + +export interface HelperPathContext { + isPackaged: boolean + resourcesPath: string + cwd: string + appPath: string +} + +/** Where the compiled helper can live, most-specific first. Packaged: bundled under + * Contents/Resources/bin (extraResources maps resources/ -> .). Dev: next to its + * source where build-actions-helper.sh emits it. Mirrors ocr.ts's resolution. */ +export function helperBinCandidates(ctx: HelperPathContext): string[] { + if (ctx.isPackaged) { + return [ + path.join(ctx.resourcesPath, 'bin', 'actions-helper'), + path.join(ctx.resourcesPath, 'actions-helper') + ] + } + return [ + path.join(ctx.cwd, 'scripts', 'actions-helper', 'actions-helper'), + path.join(ctx.appPath, 'scripts', 'actions-helper', 'actions-helper') + ] +} + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null +} + +function truncate(text: string): string { + return text.length > 200 ? `${text.slice(0, 200)}…` : text +} + +/** Parse the helper's stdout into a typed response. The helper prints one compact + * JSON line; we read the last non-empty line so a stray leading log line cannot + * break parsing. Any shape we do not recognize becomes an { ok: false } error + * rather than a throw, so a malformed helper degrades to a reported failure. */ +export function parseHelperResponse(stdout: string): NativeActionResponse { + const lines = stdout + .split('\n') + .map((l) => l.trim()) + .filter((l) => l.length > 0) + const last = lines[lines.length - 1] + if (!last) { + return { ok: false, error: 'actions helper returned no output' } + } + let parsed: unknown + try { + parsed = JSON.parse(last) + } catch { + return { ok: false, error: `actions helper returned invalid JSON: ${truncate(last)}` } + } + if (!isRecord(parsed)) { + return { ok: false, error: 'actions helper returned a non-object response' } + } + if (parsed.ok === true) { + return { ok: true, result: parsed.result } + } + if (parsed.ok === false) { + const error = + typeof parsed.error === 'string' ? parsed.error : 'actions helper reported an error' + return { ok: false, error } + } + return { ok: false, error: 'actions helper returned an unrecognized response' } +} diff --git a/src/main/actions/native-helper.ts b/src/main/actions/native-helper.ts new file mode 100644 index 00000000..d5365055 --- /dev/null +++ b/src/main/actions/native-helper.ts @@ -0,0 +1,64 @@ +// Electron-bound invoker for the native actions helper (macOS). Resolves the compiled +// helper binary and runs it as a one-shot child process, handing it one JSON command +// and parsing the one JSON line it prints back. This is the single seam every semantic +// native capability (calendar, reminders, contacts, photos) goes through, so the +// process/permission handling lives in one place. Mirrors ocr.ts. + +import { execFile } from 'child_process' +import { promisify } from 'util' +import fs from 'fs' +import { app } from 'electron' +import { + helperBinCandidates, + parseHelperResponse, + serializeCommand, + type NativeActionCommand, + type NativeActionResponse +} from './native-helper-logic' + +const execFileAsync = promisify(execFile) + +function helperBin(): string | null { + const candidates = helperBinCandidates({ + isPackaged: app.isPackaged, + resourcesPath: process.resourcesPath, + cwd: process.cwd(), + appPath: app.getAppPath() + }) + for (const candidate of candidates) { + try { + if (fs.existsSync(candidate)) { + return candidate + } + } catch { + /* ignore */ + } + } + return null +} + +/** Run one native action. Never throws: a missing helper, a spawn failure, a timeout, + * or a handled in-band error all resolve to an { ok: false } response so callers + * (tools, the approval executor) have a single shape to report. */ +export async function runNativeAction(cmd: NativeActionCommand): Promise { + const bin = helperBin() + if (!bin) { + return { ok: false, error: 'the native actions helper is not available in this build' } + } + try { + const { stdout } = await execFileAsync(bin, [serializeCommand(cmd)], { + maxBuffer: 8 * 1024 * 1024, + timeout: 20_000 + }) + return parseHelperResponse(stdout) + } catch (e) { + // execFile rejects on a non-zero exit, a timeout, or a spawn failure. The helper + // exits 0 even on handled errors, so reaching here means the process itself failed + // - but it may still have printed a response before dying, so prefer that. + const stdout = (e as { stdout?: string }).stdout + if (typeof stdout === 'string' && stdout.trim().length > 0) { + return parseHelperResponse(stdout) + } + return { ok: false, error: (e as Error).message } + } +} diff --git a/src/main/actions/semantic-rail-win.ts b/src/main/actions/semantic-rail-win.ts new file mode 100644 index 00000000..b975873f --- /dev/null +++ b/src/main/actions/semantic-rail-win.ts @@ -0,0 +1,156 @@ +/** + * The Windows semantic rail (R1 box 17) - local-first, like the mac rail. + * + * Calendar, reminders (tasks), and mail go through LOCAL Outlook COM + * automation via PowerShell: the write lands in Outlook's local store and + * syncs when the network returns, matching the macOS EventKit/Mail + * behaviour instead of failing offline the way a cloud API would. open goes + * through the injected opener (Electron's shell at wiring time). iMessage + * has no Windows equivalent - message is refused honestly, macOS-only in R1. + * + * The scripts print ONE compact JSON line ({ok, result|error}) - the exact + * contract the mac helper speaks - so parseHelperResponse is shared, not + * duplicated. Pure module: the PowerShell runner, the opener, and the + * optional Graph fallback port are injected; nothing here touches Electron. + * + * Graph (online-only, the user's own sign-in) is the fallback for setups + * without local Outlook. R1 ships the PORT and the fallback logic, + * boundary-tested; the OAuth wiring lands with the fast-follow, so + * production passes no Graph port yet and the failure stays honest. + */ +import type { ActionRecord } from '@offgrid/use' +import type { NativeActionResponse } from './native-helper-logic' + +export type RunPowerShell = (script: string) => Promise + +export interface GraphPort { + /** True only when the user has signed in and the network is reachable. */ + available(): boolean + createEvent(args: Record): Promise + createTask(args: Record): Promise + sendMail(args: Record): Promise +} + +export interface WindowsRailDeps { + runPs: RunPowerShell + openUrl: (url: string) => Promise + graph?: GraphPort +} + +export interface WinExecuteResult { + ok: boolean + detail?: string +} + +/** Single-quote a value for PowerShell: embedded quotes double, newlines stay. */ +export function psQuote(value: unknown): string { + return `'${String(value ?? '').replace(/'/g, "''")}'` +} + +const RESULT_TAIL = `| ConvertTo-Json -Compress` +const CATCH = `} catch { @{ ok = $false; error = $_.Exception.Message } ${RESULT_TAIL} }` + +/** + * The COM scripts. Outlook item types: 0 = MailItem, 1 = AppointmentItem, + * 3 = TaskItem. Each script is self-contained and reports the one JSON line. + */ +export function buildOutlookScript( + type: 'calendar' | 'reminder' | 'email', + args: Record +): string { + if (type === 'calendar') { + const lines = [ + `try {`, + `$o = New-Object -ComObject Outlook.Application`, + `$i = $o.CreateItem(1)`, + `$i.Subject = ${psQuote(args.title)}`, + `$i.Start = [datetime]${psQuote(args.start)}`, + args.end ? `$i.End = [datetime]${psQuote(args.end)}` : `$i.End = $i.Start.AddHours(1)`, + args.notes ? `$i.Body = ${psQuote(args.notes)}` : '', + `$i.Save()`, + `@{ ok = $true; result = @{ id = $i.EntryID } } ${RESULT_TAIL}`, + CATCH + ] + return lines.filter(Boolean).join('\n') + } + if (type === 'reminder') { + const lines = [ + `try {`, + `$o = New-Object -ComObject Outlook.Application`, + `$i = $o.CreateItem(3)`, + `$i.Subject = ${psQuote(args.title)}`, + args.due ? `$i.DueDate = [datetime]${psQuote(args.due)}` : '', + args.notes ? `$i.Body = ${psQuote(args.notes)}` : '', + `$i.Save()`, + `@{ ok = $true; result = @{ id = $i.EntryID } } ${RESULT_TAIL}`, + CATCH + ] + return lines.filter(Boolean).join('\n') + } + const lines = [ + `try {`, + `$o = New-Object -ComObject Outlook.Application`, + `$i = $o.CreateItem(0)`, + `$i.To = ${psQuote(args.to)}`, + `$i.Subject = ${psQuote(args.subject)}`, + `$i.Body = ${psQuote(args.body)}`, + `$i.Send()`, + `@{ ok = $true; result = @{ queued = $true } } ${RESULT_TAIL}`, + CATCH + ] + return lines.filter(Boolean).join('\n') +} + +/** COM error shapes that mean "Outlook is not installed / not registered". */ +export function isOutlookUnavailable(error: string): boolean { + return /80040154|REGDB_E_CLASSNOTREG|Outlook\.Application|cannot create.*COM/i.test(error) +} + +const GRAPH_BY_TYPE = { + calendar: 'createEvent', + reminder: 'createTask', + email: 'sendMail' +} as const + +/** One attempt on the Windows semantic rail. Never throws. */ +export function makeWindowsSemanticRailExecutor(deps: WindowsRailDeps) { + return async (action: ActionRecord): Promise => { + try { + if (action.type === 'open') { + const res = await deps.openUrl(String(action.args.url ?? '')) + return res.ok ? { ok: true } : { ok: false, detail: res.error } + } + if (action.type === 'message') { + return { + ok: false, + detail: 'iMessage is macOS-only; there is no Windows message rail in this release' + } + } + if (action.type !== 'calendar' && action.type !== 'reminder' && action.type !== 'email') { + return { ok: false, detail: `the Windows semantic rail has no mapping for '${action.type}'` } + } + + const local = await deps.runPs(buildOutlookScript(action.type, action.args)) + if (local.ok) { + return { ok: true } + } + if (isOutlookUnavailable(local.error) && deps.graph?.available()) { + // Online-only fallback, on the user's own sign-in - labeled so. + const remote = await deps.graph[GRAPH_BY_TYPE[action.type]](action.args) + return remote.ok + ? { ok: true } + : { ok: false, detail: `Microsoft Graph (online) failed: ${remote.error}` } + } + if (isOutlookUnavailable(local.error)) { + return { + ok: false, + detail: + 'local Outlook is not available on this PC, and the online Microsoft fallback is not set up' + } + } + return { ok: false, detail: local.error } + } catch (error) { + return { ok: false, detail: `windows semantic rail failed: ${(error as Error).message}` } + } + } +} diff --git a/src/main/actions/semantic-rail.ts b/src/main/actions/semantic-rail.ts new file mode 100644 index 00000000..fdcd1119 --- /dev/null +++ b/src/main/actions/semantic-rail.ts @@ -0,0 +1,79 @@ +/** + * The semantic rail - the existing native actions helper behind the + * DeviceController port (R1 box 10). + * + * Maps the engine's closed Action types onto the Swift helper's verbs and + * nothing else: an unknown type is refused, never guessed (file_share and + * web_task belong to other rails). Pure module - the runner is injected, so + * tests exercise every mapping through a fake boundary and the Electron- + * bound runNativeAction is only attached at wiring time. + */ +import type { ActionRecord } from '@offgrid/use' +import type { NativeActionCommand, NativeActionResponse } from './native-helper-logic' + +export type RunNativeAction = (cmd: NativeActionCommand) => Promise + +export interface SemanticExecuteResult { + ok: boolean + detail?: string +} + +type MapResult = { ok: true; command: NativeActionCommand } | { ok: false; error: string } + +const LOOKUP_COMMANDS: Record = { + contacts: 'contacts.search', + calendar: 'calendar.listEvents', + reminders: 'reminders.list' +} + +/** + * Action type -> helper verb. Args pass through: the emission layer (box 12) + * constrains their shape to what the helper expects per verb. + */ +export function mapActionToCommand(action: Pick): MapResult { + switch (action.type) { + case 'calendar': + return { ok: true, command: { command: 'calendar.createEvent', args: action.args } } + case 'reminder': + return { ok: true, command: { command: 'reminders.create', args: action.args } } + case 'message': + return { ok: true, command: { command: 'messages.send', args: action.args } } + case 'email': + return { ok: true, command: { command: 'mail.send', args: action.args } } + case 'open': + return { ok: true, command: { command: 'open_url', args: action.args } } + case 'lookup': { + const kind = String(action.args.kind ?? '') + const command = LOOKUP_COMMANDS[kind] + if (!command) { + return { + ok: false, + error: `lookup kind '${kind}' is not one of ${Object.keys(LOOKUP_COMMANDS).join(', ')}` + } + } + const { kind: _dropped, ...args } = action.args + return { ok: true, command: { command, args } } + } + default: + return { ok: false, error: `the semantic rail has no mapping for '${action.type}'` } + } +} + +/** One attempt on the semantic rail. Never throws - failure is a result. */ +export function makeSemanticRailExecutor(run: RunNativeAction) { + return async (action: ActionRecord): Promise => { + const mapped = mapActionToCommand(action) + if (!mapped.ok) { + return { ok: false, detail: mapped.error } + } + try { + const response = await run(mapped.command) + if (response.ok) { + return { ok: true } + } + return { ok: false, detail: response.error } + } catch (error) { + return { ok: false, detail: `semantic rail failed: ${(error as Error).message}` } + } + } +} diff --git a/src/main/actions/use-driver.ts b/src/main/actions/use-driver.ts new file mode 100644 index 00000000..e4bd415b --- /dev/null +++ b/src/main/actions/use-driver.ts @@ -0,0 +1,52 @@ +/** + * The storage adapter between the app's SQLite and the @offgrid/use engine. + * + * One DB is the source of truth: the engine's queue/state tables live in the + * SAME better-sqlite3 database the app already owns (getDB), not a second + * store that could disagree with it. This module is deliberately pure - it + * takes any better-sqlite3-shaped handle by structure (the app's + * better-sqlite3-multiple-ciphers instance and plain better-sqlite3 in tests + * both satisfy it), imports nothing from Electron, and is fully testable + * against a temp DB. + * + * better-sqlite3 is synchronous; the engine's SqlDriver is async so the same + * spine runs over mobile's async SQLite later. Wrapping sync in resolved + * promises costs nothing here. + */ +import type { SqlDriver } from '@offgrid/use' + +export interface StatementLike { + /** true when the statement returns rows (SELECT, or UPDATE ... RETURNING). */ + reader: boolean + run(...params: unknown[]): { changes: number } + get(...params: unknown[]): unknown + all(...params: unknown[]): unknown[] +} + +export interface DatabaseLike { + prepare(sql: string): StatementLike +} + +export function makeUseDriver(db: DatabaseLike): SqlDriver { + return { + async run(sql, params = []) { + const stmt = db.prepare(sql) + if (stmt.reader) { + // A returning statement still mutates; report how many rows it touched. + return { changes: stmt.all(...params).length } + } + return { changes: stmt.run(...params).changes } + }, + async get(sql: string, params: unknown[] = []) { + const stmt = db.prepare(sql) + if (stmt.reader) { + return stmt.get(...params) as T | undefined + } + stmt.run(...params) + return undefined + }, + async all(sql: string, params: unknown[] = []) { + return db.prepare(sql).all(...params) as T[] + } + } +} diff --git a/src/main/actions/use-runtime.ts b/src/main/actions/use-runtime.ts new file mode 100644 index 00000000..c1d595cd --- /dev/null +++ b/src/main/actions/use-runtime.ts @@ -0,0 +1,155 @@ +/** + * The actions runtime - the app's one composition of the @offgrid/use engine + * (R1 box 13). Electron-bound wiring only; every part it assembles is a + * tested, injectable module: the app DB via makeUseDriver, the semantic rail + * over runNativeAction, the gate host on the approval seam, and the park- + * aware worker. + * + * Lease policy: ticks hold their queue lease while an action waits at the + * gate, so visibility is set LONG (a day) and provably-stale leases from a + * previous process are cleared at startup instead (releaseAll - safe because + * the app is single-instance, so there is never a second live worker). + */ +import { + HandlerRegistry, + UseEngine, + type ActionSource, + type ProposeOutcome, + type Rail, + type TickOutcome, + type ActionRecord +} from '@offgrid/use' +import { getDB } from '../database' +import { hasHook, HOOKS } from '../bootstrap/hookRegistry' +import { shell } from 'electron' +import { makeUseDriver } from './use-driver' +import { makeSemanticRailExecutor } from './semantic-rail' +import { makeWindowsSemanticRailExecutor } from './semantic-rail-win' +import { runPowerShell } from './win-powershell' +import { makeReadBackVerifiers } from './verification' +import { runNativeAction } from './native-helper' +import { gateHost, onGateParked, whenActionParked } from './gate-host' +import { createActionWorker, type ActionWorker } from './use-worker' + +export interface ActionsRuntime { + propose( + input: unknown, + meta: { source: ActionSource; sourceRef?: string } + ): Promise + waitForOutcome(actionId: string, timeoutMs: number): Promise + whenParked(actionId: string): Promise + kick(): void + /** True when a pro approval queue is listening - the chat tool keeps the + * legacy path then, so an unmigrated pro build behaves exactly as today. */ + approvalHookActive(): boolean +} + +export function buildRegistry(run: typeof runNativeAction): HandlerRegistry { + const registry = new HandlerRegistry() + const verifiers = makeReadBackVerifiers(run) + // Calendar and reminders are observable: read back after create, so a + // failed write retries once and "done" means the item is really there. + registry.register({ + type: 'calendar', + rail: 'semantic', + defaultRisk: 'mutate', + verification: 'read_back', + verify: verifiers.calendar + }) + registry.register({ + type: 'reminder', + rail: 'semantic', + defaultRisk: 'mutate', + verification: 'read_back', + verify: verifiers.reminder + }) + // Sends have no reliable read-back ("did it send?"), so they are fuzzy + // and single-attempt behind the gate - a wrong verify can never double- + // send. open_url's launch result IS its verdict; lookups are reads. + for (const handler of [ + { type: 'message', defaultRisk: 'mutate' }, + { type: 'email', defaultRisk: 'mutate' }, + { type: 'open', defaultRisk: 'navigate' }, + { type: 'lookup', defaultRisk: 'read' } + ] as const) { + registry.register({ + type: handler.type, + rail: 'semantic', + defaultRisk: handler.defaultRisk, + verification: 'none_fuzzy' + }) + } + return registry +} + +let runtime: ActionsRuntime | null = null + +/** Lazy singleton: built on first use so the DB and helper exist by then. */ +export function getActionsRuntime(): ActionsRuntime { + if (runtime) { + return runtime + } + + // The platform decides which semantic rail implements the port - the one + // concrete choice, made once here; nothing above it branches on an OS. + // Windows note: read-back verification still speaks the mac helper's list + // verbs, so calendar/reminder read_back reports unverifiable there until + // the Outlook read-back lands (fast-follow) - the retry policy treats that + // as fuzzy-failure honestly rather than double-firing. + const semanticExecute = + process.platform === 'win32' + ? makeWindowsSemanticRailExecutor({ + runPs: runPowerShell, + openUrl: async (url: string) => { + await shell.openExternal(url) + return { ok: true as const, result: {} } + } + }) + : makeSemanticRailExecutor(runNativeAction) + const engine = new UseEngine({ + driver: makeUseDriver(getDB()), + registry: buildRegistry(runNativeAction), + device: { + async execute(action: ActionRecord, rail: Rail) { + if (rail !== 'semantic') { + return { ok: false, detail: `the '${rail}' rail is not built yet (R1 ships semantic)` } + } + return semanticExecute(action) + } + }, + gate: gateHost, + attemptTimeoutMs: 30_000, // the helper's own timeout is 20s + visibilityMs: 24 * 60 * 60 * 1000 + }) + + const worker: ActionWorker = createActionWorker(engine, { onParked: onGateParked }) + + const ready = (async () => { + await engine.init() + await engine.queue.releaseAll() // stale leases from the previous process + worker.kick() // resume anything the last session left behind + })() + + // Scheduled actions become due while the app idles; a slow heartbeat + // re-kicks the drain. unref'd so it never holds the process open. + const heartbeat = setInterval(() => worker.kick(), 30_000) + heartbeat.unref?.() + + runtime = { + async propose(input, meta) { + await ready + const outcome = await engine.propose(input, meta) + worker.kick() + return outcome + }, + async waitForOutcome(actionId, timeoutMs) { + await ready + return worker.waitForOutcome(actionId, timeoutMs) + }, + whenParked: whenActionParked, + kick: () => worker.kick(), + approvalHookActive: () => + hasHook(HOOKS.actionsProposeApproval) || hasHook(HOOKS.legacyMcpProposeApproval) + } + return runtime +} diff --git a/src/main/actions/use-worker.ts b/src/main/actions/use-worker.ts new file mode 100644 index 00000000..677ca383 --- /dev/null +++ b/src/main/actions/use-worker.ts @@ -0,0 +1,117 @@ +/** + * The action worker - drains the engine's queue and routes each outcome to + * whoever is waiting on it (R1 box 13). + * + * A tick that reaches the gate holds its promise open until a human + * decides, so the drain loop cannot simply await every tick: it races each + * tick against the park signal, and when a tick parks it is left running in + * the background (its outcome still lands with waiters when the human + * eventually resolves the gate) while the loop moves on to the next due + * message. The queue's lease keeps concurrent in-flight ticks safe. + * + * Pure orchestration over two injected ports (an engine-shaped tick and the + * park signal), so it is testable with scripted fakes; use-runtime.ts wires + * the real UseEngine and gate host. + */ +import type { TickOutcome } from '@offgrid/use' + +export interface EngineLike { + tick(): Promise +} + +export interface ParkSignal { + /** Subscribe to "an action just parked at the gate"; returns unsubscribe. */ + onParked(listener: () => void): () => void +} + +export interface ActionWorker { + /** Start (or continue) draining until the queue reports nothing due. */ + kick(): void + /** The outcome for one action id, or undefined when the wait times out + * (parked at the gate, or scheduled for later). */ + waitForOutcome(actionId: string, timeoutMs: number): Promise + /** Whether a drain pass is currently running (health surface, tests). */ + draining(): boolean +} + +export function createActionWorker(engine: EngineLike, park: ParkSignal): ActionWorker { + const waiters = new Map void>>() + let running = false + + const notify = (outcome: TickOutcome) => { + const list = waiters.get(outcome.id) + if (list) { + waiters.delete(outcome.id) + for (const resolve of list) { + resolve(outcome) + } + } + } + + const drain = async () => { + running = true + try { + for (;;) { + let parkedResolve: (() => void) | undefined + const parked = new Promise<'parked'>((resolve) => { + parkedResolve = () => resolve('parked') + }) + const unsubscribe = park.onParked(() => parkedResolve?.()) + const tickPromise = engine.tick() + try { + const first = await Promise.race([ + tickPromise.then((outcome) => ({ kind: 'tick' as const, outcome })), + parked.then(() => ({ kind: 'parked' as const })) + ]) + if (first.kind === 'parked') { + // The tick is waiting on a human. Leave it in flight - its + // outcome still reaches waiters when the gate resolves - and + // move on to the next due message. + void tickPromise.then((outcome) => outcome && notify(outcome)) + continue + } + if (!first.outcome) { + return // nothing due + } + notify(first.outcome) + } finally { + unsubscribe() + } + } + } finally { + running = false + } + } + + return { + kick() { + if (!running) { + void drain() + } + }, + draining() { + return running + }, + waitForOutcome(actionId, timeoutMs) { + return new Promise((resolve) => { + const timer = setTimeout(() => { + const list = waiters.get(actionId) + if (list) { + waiters.set( + actionId, + list.filter((w) => w !== wrapped) + ) + } + resolve(undefined) + }, timeoutMs) + const wrapped = (outcome: TickOutcome) => { + clearTimeout(timer) + resolve(outcome) + } + const list = waiters.get(actionId) ?? [] + list.push(wrapped) + waiters.set(actionId, list) + }) + } + } +} diff --git a/src/main/actions/verification.ts b/src/main/actions/verification.ts new file mode 100644 index 00000000..da52dd2e --- /dev/null +++ b/src/main/actions/verification.ts @@ -0,0 +1,91 @@ +/** + * Read-back verification for the semantic rail (R1 box 14). + * + * "Done" must mean the effect is OBSERVABLE, not that the helper returned + * ok - the field's number-one trust failure is an agent reporting success + * on a write that never landed. Calendar and reminders can actually be + * read back (list after create), so their handlers declare read_back and + * verify here. Messages and mail cannot ("did it send?" has no reliable + * read-back), so they stay none_fuzzy and single-attempt behind the gate. + * open_url's launch result IS its verdict. + * + * Everything fails closed: a helper error, a malformed result, or missing + * args verify as false - the retry policy takes it from there. + */ +import type { ActionRecord } from '@offgrid/use' +import type { NativeActionCommand, NativeActionResponse } from './native-helper-logic' + +export type RunNative = (cmd: NativeActionCommand) => Promise + +/** Does a helper list result contain an item with this exact title? */ +export function listContainsTitle( + result: unknown, + key: 'reminders' | 'events', + title: string +): boolean { + if (typeof result !== 'object' || result === null) { + return false + } + const items = (result as Record)[key] + if (!Array.isArray(items)) { + return false + } + return items.some( + (item) => + typeof item === 'object' && + item !== null && + (item as Record).title === title + ) +} + +const HOUR_MS = 60 * 60 * 1000 +const PAD_MS = 60 * 1000 + +/** + * The list window for a created event: its own start/end padded by a + * minute (the helper defaults a missing end to start plus one hour). + * Undefined when the start is unparseable - nothing sane to verify against. + */ +export function calendarVerifyWindow(args: Record): + | { start: string; end: string } + | undefined { + const startMs = Date.parse(String(args.start ?? '')) + if (Number.isNaN(startMs)) { + return undefined + } + const endParsed = Date.parse(String(args.end ?? '')) + const endMs = Number.isNaN(endParsed) ? startMs + HOUR_MS : endParsed + return { + start: new Date(startMs - PAD_MS).toISOString(), + end: new Date(endMs + PAD_MS).toISOString() + } +} + +/** The read-back verifiers, over the same helper boundary the rail uses. */ +export function makeReadBackVerifiers(run: RunNative): { + reminder: (action: ActionRecord) => Promise + calendar: (action: ActionRecord) => Promise +} { + return { + async reminder(action) { + const title = action.args.title + if (typeof title !== 'string' || title.length === 0) { + return false + } + const res = await run({ command: 'reminders.list', args: {} }) + return res.ok && listContainsTitle(res.result, 'reminders', title) + }, + async calendar(action) { + const title = action.args.title + if (typeof title !== 'string' || title.length === 0) { + return false + } + const window = calendarVerifyWindow(action.args) + if (!window) { + return false + } + const res = await run({ command: 'calendar.listEvents', args: window }) + return res.ok && listContainsTitle(res.result, 'events', title) + } + } +} diff --git a/src/main/actions/win-powershell.ts b/src/main/actions/win-powershell.ts new file mode 100644 index 00000000..d8aa232d --- /dev/null +++ b/src/main/actions/win-powershell.ts @@ -0,0 +1,30 @@ +/** + * Electron-bound PowerShell runner for the Windows semantic rail. The one + * seam every Outlook COM script goes through - mirrors native-helper.ts on + * macOS, and speaks the same one-JSON-line contract, parsed by the same + * parseHelperResponse. Never throws: a spawn failure, a timeout, or a + * script error all resolve to a reported { ok: false }. + */ +import { execFile } from 'child_process' +import { promisify } from 'util' +import { parseHelperResponse, type NativeActionResponse } from './native-helper-logic' + +const execFileAsync = promisify(execFile) + +export async function runPowerShell(script: string): Promise { + try { + const { stdout } = await execFileAsync( + 'powershell.exe', + ['-NoProfile', '-NonInteractive', '-ExecutionPolicy', 'Bypass', '-Command', script], + { maxBuffer: 8 * 1024 * 1024, timeout: 20_000, windowsHide: true } + ) + return parseHelperResponse(stdout) + } catch (e) { + // A non-zero exit may still have printed a response line - prefer it. + const stdout = (e as { stdout?: string }).stdout + if (typeof stdout === 'string' && stdout.trim().length > 0) { + return parseHelperResponse(stdout) + } + return { ok: false, error: (e as Error).message } + } +} diff --git a/src/main/bootstrap/__tests__/hookRegistry.test.ts b/src/main/bootstrap/__tests__/hookRegistry.test.ts index 1730ceeb..60b7d57a 100644 --- a/src/main/bootstrap/__tests__/hookRegistry.test.ts +++ b/src/main/bootstrap/__tests__/hookRegistry.test.ts @@ -5,7 +5,14 @@ * and universal-search sources both route through it. */ import { describe, it, expect } from 'vitest' -import { registerHook, callHook, callHookAsync, HOOKS } from '../hookRegistry' +import { + registerHook, + unregisterHook, + hasHook, + callHook, + callHookAsync, + HOOKS +} from '../hookRegistry' describe('hookRegistry', () => { it('registers a hook and callHook returns its result', () => { @@ -48,8 +55,30 @@ describe('hookRegistry', () => { await expect(callHookAsync('t.sync-via-async')).resolves.toBe('plain') }) + it('hasHook reports registration and unregisterHook removes it', () => { + expect(hasHook('t.presence')).toBe(false) + registerHook('t.presence', () => 1) + expect(hasHook('t.presence')).toBe(true) + unregisterHook('t.presence') + expect(hasHook('t.presence')).toBe(false) + expect(callHook('t.presence')).toBeUndefined() + }) + + it('hasHook is true even for a hook that returns undefined', () => { + registerHook('t.returns-undefined', () => undefined) + expect(hasHook('t.returns-undefined')).toBe(true) + expect(callHook('t.returns-undefined')).toBeUndefined() + unregisterHook('t.returns-undefined') + }) + + it('unregisterHook is a no-op for an unknown key', () => { + expect(() => unregisterHook('t.never')).not.toThrow() + }) + it('exposes the known hook-name constants core and pro share', () => { expect(HOOKS.chatAugmentContext).toBe('chat.augmentContext') expect(HOOKS.searchExtraSources).toBe('search.extraSources') + expect(HOOKS.actionsProposeApproval).toBe('actions:proposeApproval') + expect(HOOKS.legacyMcpProposeApproval).toBe('mcp:proposeApproval') }) }) diff --git a/src/main/bootstrap/hookRegistry.ts b/src/main/bootstrap/hookRegistry.ts index de4e7f79..37839ab2 100644 --- a/src/main/bootstrap/hookRegistry.ts +++ b/src/main/bootstrap/hookRegistry.ts @@ -16,6 +16,19 @@ export function registerHook(name: string, fn: HookFn): void { hooks[name] = fn } +/** Remove a registered hook. No-op when absent. Mainly for test isolation and + * for retiring a legacy hook name once its replacement is registered. */ +export function unregisterHook(name: string): void { + delete hooks[name] +} + +/** Whether a hook is currently registered. Lets a caller distinguish "no handler" + * from "handler ran and returned undefined" — needed when falling back from a new + * hook name to a legacy one. */ +export function hasHook(name: string): boolean { + return name in hooks +} + /** Call a hook if registered; returns its result, or undefined when absent. */ export function callHook(name: string, ...args: unknown[]): R | undefined { const fn = hooks[name] @@ -56,5 +69,13 @@ export const HOOKS = { * generating, or null when it is generating nothing. Pro streams it live to paired devices; free * builds leave it inert. A SNAPSHOT rather than a delta, so a consumer cannot miss the end. */ - syncStreamingState: 'sync.streamingState' + syncStreamingState: 'sync.streamingState', + /** (request: ActionApprovalRequest) => boolean — offer a consequential action + * for approval; returns true when queued (caller must not execute). Pro + * registers it to route the action through its approval queue + audit log. */ + actionsProposeApproval: 'actions:proposeApproval', + /** Legacy MCP-only predecessor of actionsProposeApproval. Kept so a pro build + * that has not yet migrated still gates connector writes; remove once + * desktop-pro registers actionsProposeApproval. */ + legacyMcpProposeApproval: 'mcp:proposeApproval' } as const diff --git a/src/main/index.ts b/src/main/index.ts index 929bfe4c..5c58e0d4 100644 --- a/src/main/index.ts +++ b/src/main/index.ts @@ -19,6 +19,8 @@ import icon from '../../resources/icon.png?asset' import { setupIPC } from './ipc' // IMPORT FROM IPC ONLY import { setupRagIPC } from './rag-ipc' import { setupMcpIpc } from './mcp-ipc' +import { registerToolExtension } from './tools' +import { registerNativeActionTools } from './tools/nativeActionToolExtension' import { setupDesktopBackupIPC } from './backup/ipc' import { preloadPath } from './preload-path' import { rendererHtmlPath } from './renderer-path' @@ -383,6 +385,7 @@ app.whenReady().then(async () => { setupIPC() setupRagIPC() setupMcpIpc() // basic MCP connectors (management + chat tool extension) + registerNativeActionTools(registerToolExtension) // computer use: semantic rail (macOS-only) setupDesktopBackupIPC() // one OpenAI-compatible local gateway (LLM + STT); auto-picks a free port. Async, so handle a // rejection on the promise (a try/catch around a fire-and-forget async call can't catch it). diff --git a/src/main/tools.ts b/src/main/tools.ts index 477d086e..0c5b90a9 100644 --- a/src/main/tools.ts +++ b/src/main/tools.ts @@ -16,6 +16,7 @@ import { buildUserContent } from './tool-content' import { stripTags, htmlToText, decodeDdgHref } from './tools-parsers' import { mimeFromExt } from './model-server/data-url' import { evaluateArithmetic } from './calculator' +import { selectToolExtensions } from './tools/extension-select' // Per-tool enable/disable, persisted as a list of disabled tool names. function disabledSet(): Set { @@ -386,6 +387,12 @@ async function runTool( // Mirrors mobile/src/services/tools/extensions.ts. export interface ToolExtension { id: string + /** What kind of capability this is. 'tool' = the assistant's own on-device + * abilities (native actions) - included in every agentic turn. 'connector' + * = external service accounts (MCP) - included only when the user turns + * Connectors on. Defaults to 'connector' (fail closed for anything that + * might touch an external service). */ + category?: 'tool' | 'connector' /** OpenAI tool schemas to add when extensions are enabled. Built once per turn; * the extension may cache any per-turn state it needs for execute(). */ schemas(): Promise | unknown[] @@ -470,7 +477,7 @@ export async function toolChat( // alongside the built-ins. Schemas are built once per turn; each extension // caches whatever per-turn state it needs for execute(). Free build registers // no extensions, so this is just the built-ins. - const exts = opts.connectors ? getToolExtensions() : [] + const exts = selectToolExtensions(getToolExtensions(), { connectors: !!opts.connectors }) const extSchemas: unknown[] = [] const hints: string[] = [] for (const e of exts) { diff --git a/src/main/tools/__tests__/extension-select.test.ts b/src/main/tools/__tests__/extension-select.test.ts new file mode 100644 index 00000000..1d688997 --- /dev/null +++ b/src/main/tools/__tests__/extension-select.test.ts @@ -0,0 +1,42 @@ +/** + * The one rule for which extensions join an agentic turn: the assistant's + * own tools always ride; connectors only when the user turned them on; an + * undeclared category fails closed as a connector. + */ +import { describe, expect, it } from 'vitest' +import { selectToolExtensions } from '../extension-select' +import { nativeActionToolExtension } from '../nativeActionToolExtension' +import type { ToolExtension } from '../../tools' + +const ext = (id: string, category?: 'tool' | 'connector'): ToolExtension => ({ + id, + category, + schemas: () => [], + canHandle: () => false, + execute: () => 'x' +}) + +describe('selectToolExtensions', () => { + it('the assistant\'s own tools ride every agentic turn', () => { + const picked = selectToolExtensions([ext('native', 'tool'), ext('mcp', 'connector')], { + connectors: false + }) + expect(picked.map((e) => e.id)).toEqual(['native']) + }) + + it('connectors join only when turned on', () => { + const picked = selectToolExtensions([ext('native', 'tool'), ext('mcp', 'connector')], { + connectors: true + }) + expect(picked.map((e) => e.id)).toEqual(['native', 'mcp']) + }) + + it('an undeclared category fails closed as a connector', () => { + const picked = selectToolExtensions([ext('legacy')], { connectors: false }) + expect(picked).toEqual([]) + }) + + it('the native actions extension declares itself a tool', () => { + expect(nativeActionToolExtension.category).toBe('tool') + }) +}) diff --git a/src/main/tools/__tests__/mcpConnectorToolExtension.test.ts b/src/main/tools/__tests__/mcpConnectorToolExtension.test.ts index 5e3a7591..63e43685 100644 --- a/src/main/tools/__tests__/mcpConnectorToolExtension.test.ts +++ b/src/main/tools/__tests__/mcpConnectorToolExtension.test.ts @@ -2,8 +2,10 @@ import { describe, expect, it } from 'vitest' import { buildConnectorToolSchema, formatConnectorToolResult, - isActionTool + isActionTool, + riskOf } from '../mcpConnectorToolExtension-logic' +import { shouldGate } from '../../actions/approval' describe('isActionTool', () => { it.each([ @@ -37,6 +39,28 @@ describe('isActionTool', () => { }) }) +describe('riskOf', () => { + it('maps read-verb tools to a non-gating read risk', () => { + for (const tool of ['list_channels', 'get_user', 'search_docs', 'read_file']) { + expect(riskOf(tool)).toBe('read') + expect(shouldGate(riskOf(tool))).toBe(false) + } + }) + + it('maps every other tool to a gating mutate risk', () => { + for (const tool of ['send_message', 'create_issue', 'delete_record']) { + expect(riskOf(tool)).toBe('mutate') + expect(shouldGate(riskOf(tool))).toBe(true) + } + }) + + it('agrees with isActionTool on which tools gate', () => { + for (const tool of ['list_channels', 'send_message', 'get_user', 'delete_record']) { + expect(shouldGate(riskOf(tool))).toBe(isActionTool(tool)) + } + }) +}) + describe('buildConnectorToolSchema', () => { it('namespaces the tool and retains its description and input schema', () => { expect( diff --git a/src/main/tools/__tests__/nativeActionToolExtension-engine.test.ts b/src/main/tools/__tests__/nativeActionToolExtension-engine.test.ts new file mode 100644 index 00000000..554c6ff2 --- /dev/null +++ b/src/main/tools/__tests__/nativeActionToolExtension-engine.test.ts @@ -0,0 +1,223 @@ +/** + * The chat tool's engine path (R1 box 13): a gated mutation becomes a + * durable Action through the injected actions port, reads stay inline, and + * a listening pro approval queue keeps the legacy path exactly as before. + */ +import { describe, expect, it, vi } from 'vitest' +import type { TickOutcome } from '@offgrid/use' +import { NativeActionToolExtension, type ActionsPort } from '../nativeActionToolExtension' +import { + actionTypeForTool, + NATIVE_TOOL_SPECS, + TOOL_ACTION_TYPES +} from '../nativeActionToolExtension-logic' + +function makePort( + overrides: Partial = {} +): ActionsPort & { proposed: unknown[] } { + const proposed: unknown[] = [] + return { + proposed, + approvalHookActive: () => false, + async propose(input) { + proposed.push(input) + return { accepted: true, id: 'act_1', deduped: false } + }, + async waitForOutcome() { + return { + id: 'act_1', + outcome: 'done', + record: { attemptLog: [] } + } as unknown as TickOutcome + }, + whenParked: () => new Promise(() => {}), + kick: () => {}, + ...overrides + } +} + +const run = vi.fn(async () => ({ ok: true as const, result: { id: 'r1' } })) +const proposeApproval = vi.fn(() => undefined) + +const makeExtension = (actions?: ActionsPort) => + new NativeActionToolExtension({ run, proposeApproval, actions }) + +describe('the tool-to-action-type map', () => { + it('covers exactly the mutating tools', () => { + expect(Object.keys(TOOL_ACTION_TYPES).sort()).toEqual([ + 'calendar_create_event', + 'mail_send', + 'messages_send', + 'reminders_create' + ]) + expect(actionTypeForTool('reminders_create')).toBe('reminder') + expect(actionTypeForTool('calendar_list_events')).toBeUndefined() + }) +}) + +describe('the spec table', () => { + it('every spec produces a title, mapped args, and a formatted result', () => { + const sample = { + title: 'x', start: 's', end: 'e', query: 'q', to: 't', text: 'm', url: 'u' + } + for (const spec of NATIVE_TOOL_SPECS) { + expect(typeof spec.title(sample)).toBe('string') + expect(spec.title(sample).length).toBeGreaterThan(0) + expect(typeof spec.buildArgs(sample)).toBe('object') + expect(typeof spec.formatResult({ id: 'r1' })).toBe('string') + } + // Only the engine-routed (mutating) specs must format an undefined + // result - the engine reports outcomes, not helper payloads. + for (const name of Object.keys(TOOL_ACTION_TYPES)) { + const spec = NATIVE_TOOL_SPECS.find((s) => s.name === name) + expect(typeof spec?.formatResult(undefined)).toBe('string') + } + }) + + it('the extension exposes its schemas and system hint', () => { + const extension = makeExtension(makePort()) + expect(extension.schemas()).toHaveLength(NATIVE_TOOL_SPECS.length) + expect(extension.systemHint()).toMatch(/act on the user's Mac/) + expect(extension.canHandle('reminders_create')).toBe(true) + }) +}) + +describe('the engine path', () => { + it('a mutation becomes a durable Action with the mapped type, intent, and risk', async () => { + run.mockClear() + const port = makePort() + const extension = makeExtension(port) + const reply = await extension.execute('reminders_create', { title: 'Send the deck' }) + expect(port.proposed[0]).toMatchObject({ + type: 'reminder', + intent: 'Create the reminder "Send the deck"', + args: { title: 'Send the deck' }, + risk: 'mutate' + }) + expect(reply).toBe('Created the reminder.') + expect(run).not.toHaveBeenCalled() + expect(proposeApproval).not.toHaveBeenCalled() + }) + + it('a read runs inline and never touches the engine', async () => { + run.mockClear() + const port = makePort() + const extension = makeExtension(port) + await extension.execute('reminders_list', {}) + expect(port.proposed).toEqual([]) + expect(run).toHaveBeenCalledWith({ command: 'reminders.list', args: {} }) + }) + + it('navigation (open_url) also stays inline', async () => { + run.mockClear() + const port = makePort() + const extension = makeExtension(port) + await extension.execute('open_url', { url: 'https://x.test' }) + expect(port.proposed).toEqual([]) + expect(run).toHaveBeenCalled() + }) + + it('an action parked at the gate reports pending approval', async () => { + const port = makePort({ + waitForOutcome: () => new Promise(() => {}), + whenParked: async () => {} + }) + const extension = makeExtension(port) + const reply = await extension.execute('messages_send', { to: 'x@y.z', text: 'hi' }) + expect(reply).toMatch(/pending approval/) + }) + + it('a deduped proposal says it is already queued', async () => { + const port = makePort({ + propose: async () => ({ accepted: true, id: 'act_1', deduped: true }) + }) + const extension = makeExtension(port) + const reply = await extension.execute('reminders_create', { title: 'x' }) + expect(reply).toMatch(/already queued/) + }) + + it('a refused proposal surfaces the reason', async () => { + const port = makePort({ + propose: async () => ({ accepted: false, reason: 'no handler' }) + }) + const extension = makeExtension(port) + const reply = await extension.execute('reminders_create', { title: 'x' }) + expect(reply).toMatch(/refused: no handler/) + }) + + it('rejected and needs_help outcomes report honestly', async () => { + const rejected = makeExtension( + makePort({ + waitForOutcome: async () => + ({ id: 'act_1', outcome: 'rejected', record: { attemptLog: [] } }) as unknown as TickOutcome + }) + ) + expect(await rejected.execute('mail_send', { to: 'a@b.c' })).toMatch(/declined/) + + const needsHelp = makeExtension( + makePort({ + waitForOutcome: async () => + ({ + id: 'act_1', + outcome: 'needs_help', + record: { attemptLog: [{ rail: 'semantic', at: 1, outcome: 'timeout', detail: 'no answer' }] } + }) as unknown as TickOutcome + }) + ) + expect(await needsHelp.execute('mail_send', { to: 'a@b.c' })).toMatch(/no answer/) + }) + + it('edited and poisoned outcomes report honestly too', async () => { + const edited = makeExtension( + makePort({ + waitForOutcome: async () => + ({ id: 'act_1', outcome: 'edited', record: { attemptLog: [] } }) as unknown as TickOutcome + }) + ) + expect(await edited.execute('reminders_create', { title: 'x' })).toMatch(/editing/) + + const poisoned = makeExtension( + makePort({ + waitForOutcome: async () => + ({ id: 'act_1', outcome: 'poisoned', error: 'bad body' }) as unknown as TickOutcome + }) + ) + expect(await poisoned.execute('reminders_create', { title: 'x' })).toMatch(/bad body/) + + const helpNoDetail = makeExtension( + makePort({ + waitForOutcome: async () => + ({ + id: 'act_1', + outcome: 'needs_help', + record: { attemptLog: [{ rail: 'semantic', at: 1, outcome: 'error' }] } + }) as unknown as TickOutcome + }) + ) + expect(await helpNoDetail.execute('reminders_create', { title: 'x' })).toMatch(/needs their attention/) + }) + + it('a listening pro approval queue keeps the legacy path untouched', async () => { + run.mockClear() + const legacyPropose = vi.fn(() => true) + const port = makePort({ approvalHookActive: () => true }) + const extension = new NativeActionToolExtension({ + run, + proposeApproval: legacyPropose, + actions: port + }) + const reply = await extension.execute('reminders_create', { title: 'x' }) + expect(port.proposed).toEqual([]) + expect(legacyPropose).toHaveBeenCalled() + expect(reply).toMatch(/pending approval/) + }) + + it('no actions port at all means the legacy path (existing behaviour)', async () => { + run.mockClear() + const legacyPropose = vi.fn(() => undefined) + const extension = new NativeActionToolExtension({ run, proposeApproval: legacyPropose }) + await extension.execute('reminders_create', { title: 'x' }) + expect(legacyPropose).toHaveBeenCalled() + expect(run).toHaveBeenCalled() + }) +}) diff --git a/src/main/tools/__tests__/nativeActionToolExtension-logic.test.ts b/src/main/tools/__tests__/nativeActionToolExtension-logic.test.ts new file mode 100644 index 00000000..bac8016a --- /dev/null +++ b/src/main/tools/__tests__/nativeActionToolExtension-logic.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from 'vitest' +import { + NATIVE_TOOL_SPECS, + findNativeToolSpec, + buildNativeToolSchemas +} from '../nativeActionToolExtension-logic' +import { shouldGate } from '../../actions/approval' + +describe('native tool specs', () => { + it('exposes calendar and reminder tools with matching helper commands', () => { + expect(NATIVE_TOOL_SPECS.map((s) => s.name)).toEqual([ + 'calendar_create_event', + 'calendar_list_events', + 'reminders_create', + 'reminders_list', + 'contacts_search', + 'messages_send', + 'mail_send', + 'open_url' + ]) + expect(findNativeToolSpec('calendar_create_event')?.command).toBe('calendar.createEvent') + expect(findNativeToolSpec('calendar_list_events')?.command).toBe('calendar.listEvents') + expect(findNativeToolSpec('reminders_create')?.command).toBe('reminders.create') + expect(findNativeToolSpec('reminders_list')?.command).toBe('reminders.list') + expect(findNativeToolSpec('contacts_search')?.command).toBe('contacts.search') + expect(findNativeToolSpec('messages_send')?.command).toBe('messages.send') + expect(findNativeToolSpec('mail_send')?.command).toBe('mail.send') + expect(findNativeToolSpec('open_url')?.command).toBe('system.openURL') + }) + + it('gates the send actions and runs the read lookups without approval', () => { + for (const name of ['messages_send', 'mail_send']) { + expect(shouldGate(findNativeToolSpec(name)!.risk)).toBe(true) + } + expect(shouldGate(findNativeToolSpec('contacts_search')!.risk)).toBe(false) + }) + + it('treats open_url as a navigate that runs without approval', () => { + expect(findNativeToolSpec('open_url')!.risk).toBe('navigate') + expect(shouldGate(findNativeToolSpec('open_url')!.risk)).toBe(false) + }) + + it('confirms a sent message and email without echoing arguments', () => { + expect(findNativeToolSpec('messages_send')!.formatResult({ sent: true })).toBe( + 'Sent the message.' + ) + expect(findNativeToolSpec('mail_send')!.formatResult({ sent: true })).toBe('Sent the email.') + }) + + it('classifies every create tool as a gating mutate and every list tool as a read', () => { + for (const name of ['calendar_create_event', 'reminders_create']) { + expect(shouldGate(findNativeToolSpec(name)!.risk)).toBe(true) + } + for (const name of ['calendar_list_events', 'reminders_list']) { + expect(shouldGate(findNativeToolSpec(name)!.risk)).toBe(false) + } + }) + + it('formats a created reminder with the shared confirmation shape', () => { + expect(findNativeToolSpec('reminders_create')!.formatResult({ id: 'R1' })).toBe( + 'Created the reminder (id R1).' + ) + }) + + it('returns undefined for an unknown tool name', () => { + expect(findNativeToolSpec('calendar_delete_everything')).toBeUndefined() + }) + + it('gates the mutating create tool and runs the read-only list tool freely', () => { + expect(shouldGate(findNativeToolSpec('calendar_create_event')!.risk)).toBe(true) + expect(shouldGate(findNativeToolSpec('calendar_list_events')!.risk)).toBe(false) + }) + + it('builds an approval title from the event title', () => { + expect(findNativeToolSpec('calendar_create_event')!.title({ title: 'Sync with Ali' })).toBe( + 'Create the calendar event "Sync with Ali"' + ) + }) + + it('formats a create result into a confirmation, with and without an id', () => { + const spec = findNativeToolSpec('calendar_create_event')! + expect(spec.formatResult({ id: 'E1' })).toBe('Created the calendar event (id E1).') + expect(spec.formatResult({})).toBe('Created the calendar event.') + }) + + it('builds OpenAI function schemas for every spec', () => { + const schemas = buildNativeToolSchemas() + expect(schemas).toHaveLength(NATIVE_TOOL_SPECS.length) + expect(schemas[0]).toMatchObject({ + type: 'function', + function: { name: 'calendar_create_event', parameters: { required: ['title', 'start'] } } + }) + }) +}) diff --git a/src/main/tools/__tests__/nativeActionToolExtension.test.ts b/src/main/tools/__tests__/nativeActionToolExtension.test.ts new file mode 100644 index 00000000..5b108519 --- /dev/null +++ b/src/main/tools/__tests__/nativeActionToolExtension.test.ts @@ -0,0 +1,143 @@ +/** + * Execute-path tests for the native-action tool extension against a fake boundary + * (the same injection seam the MCP extension uses). Pins the gate-then-run contract: + * a mutating tool queues for approval and does NOT run when queued, runs directly when + * nothing gates it (free build), and a read tool never gates. Platform registration is + * asserted so the tools stay out of the grammar budget off macOS. + */ +import { describe, expect, it, beforeEach } from 'vitest' +import { + NativeActionToolExtension, + registerNativeActionTools, + type NativeActionToolBoundary +} from '../nativeActionToolExtension' +import type { ToolExtension } from '../../tools' +import type { ActionApprovalRequest } from '../../actions/approval' +import type { NativeActionCommand, NativeActionResponse } from '../../actions/native-helper-logic' + +class FakeBoundary implements NativeActionToolBoundary { + readonly commands: NativeActionCommand[] = [] + readonly approvals: ActionApprovalRequest[] = [] + queueApprovals = false + response: NativeActionResponse = { ok: true, result: { id: 'E1' } } + + async run(cmd: NativeActionCommand): Promise { + this.commands.push(cmd) + return this.response + } + + proposeApproval(request: ActionApprovalRequest): boolean { + this.approvals.push(request) + return this.queueApprovals + } +} + +let boundary: FakeBoundary +let ext: NativeActionToolExtension + +beforeEach(() => { + boundary = new FakeBoundary() + ext = new NativeActionToolExtension(boundary) +}) + +describe('NativeActionToolExtension', () => { + it('owns only its known tool names', () => { + expect(ext.canHandle('calendar_create_event')).toBe(true) + expect(ext.canHandle('calendar_list_events')).toBe(true) + expect(ext.canHandle('mcp__1__send')).toBe(false) + }) + + it('queues a create for approval and does not run the helper when queued', async () => { + boundary.queueApprovals = true + const out = await ext.execute('calendar_create_event', { + title: 'Sync', + start: '2026-08-13T15:00:00' + }) + + expect(out).toContain('Queued for the user') + expect(boundary.approvals).toEqual([ + expect.objectContaining({ + kind: 'native', + risk: 'mutate', + command: 'calendar.createEvent', + args: { title: 'Sync', start: '2026-08-13T15:00:00' } + }) + ]) + expect(boundary.commands).toEqual([]) + }) + + it('runs a create directly when nothing gates it (free build)', async () => { + boundary.queueApprovals = false + const out = await ext.execute('calendar_create_event', { + title: 'Sync', + start: '2026-08-13T15:00:00' + }) + + expect(out).toBe('Created the calendar event (id E1).') + expect(boundary.approvals).toHaveLength(1) // it was offered + expect(boundary.commands).toEqual([ + { command: 'calendar.createEvent', args: { title: 'Sync', start: '2026-08-13T15:00:00' } } + ]) + }) + + it('gates a message send and does not run the helper when queued', async () => { + boundary.queueApprovals = true + const out = await ext.execute('messages_send', { to: '+15551234567', text: 'on my way' }) + + expect(out).toContain('Queued for the user') + expect(boundary.approvals).toEqual([ + expect.objectContaining({ + kind: 'native', + risk: 'mutate', + command: 'messages.send', + args: { to: '+15551234567', text: 'on my way' } + }) + ]) + expect(boundary.commands).toEqual([]) + }) + + it('runs a read tool without ever offering it for approval', async () => { + boundary.response = { ok: true, result: { events: [] } } + const out = await ext.execute('calendar_list_events', { + start: '2026-08-13T00:00:00', + end: '2026-08-14T00:00:00' + }) + + expect(out).toBe('{"events":[]}') + expect(boundary.approvals).toEqual([]) + expect(boundary.commands).toEqual([ + { + command: 'calendar.listEvents', + args: { start: '2026-08-13T00:00:00', end: '2026-08-14T00:00:00' } + } + ]) + }) + + it('passes a helper failure back as an error string', async () => { + boundary.response = { ok: false, error: 'calendar access was not granted' } + expect(await ext.execute('calendar_list_events', { start: 'a', end: 'b' })).toBe( + 'Error: calendar access was not granted' + ) + }) + + it('rejects an unknown tool name', async () => { + expect(await ext.execute('calendar_delete_all', {})).toBe( + 'Error: unknown action calendar_delete_all' + ) + }) +}) + +describe('registerNativeActionTools', () => { + it('registers the extension on macOS', () => { + const registered: ToolExtension[] = [] + registerNativeActionTools((e) => registered.push(e), 'darwin') + expect(registered.map((e) => e.id)).toEqual(['native-actions']) + }) + + it('registers nothing off macOS', () => { + const registered: ToolExtension[] = [] + registerNativeActionTools((e) => registered.push(e), 'win32') + registerNativeActionTools((e) => registered.push(e), 'linux') + expect(registered).toEqual([]) + }) +}) diff --git a/src/main/tools/extension-select.ts b/src/main/tools/extension-select.ts new file mode 100644 index 00000000..18804e6d --- /dev/null +++ b/src/main/tools/extension-select.ts @@ -0,0 +1,23 @@ +/** + * Which registered tool extensions join an agentic turn. Pure, so the rule + * is testable and defined once: the assistant's own tools ride every + * agentic turn; connector extensions (external accounts) join only when the + * user turned Connectors on. An extension that declares no category is + * treated as a connector - fail closed for anything that might touch an + * external service. + * + * Structural on purpose: importing ToolExtension from ../tools would create + * the cycle tools -> extension-select -> tools (dependency-cruiser blocks + * it). The selector only needs the category field, so it asks for exactly + * that and stays generic over the caller's richer type. + */ +export interface CategorizedExtension { + category?: 'tool' | 'connector' +} + +export function selectToolExtensions( + extensions: T[], + opts: { connectors: boolean } +): T[] { + return extensions.filter((e) => e.category === 'tool' || opts.connectors) +} diff --git a/src/main/tools/mcpConnectorToolExtension-logic.ts b/src/main/tools/mcpConnectorToolExtension-logic.ts index 4ada1f39..dcbf5b53 100644 --- a/src/main/tools/mcpConnectorToolExtension-logic.ts +++ b/src/main/tools/mcpConnectorToolExtension-logic.ts @@ -1,3 +1,5 @@ +import type { ActionRisk } from '../actions/approval' + export const MCP_TOOL_PREFIX = 'mcp__' export interface ConnectorToolDefinition { @@ -19,6 +21,14 @@ export function isActionTool(tool: string): boolean { return !/^(list|get|search|read|fetch|whoami|describe)[_-]/i.test(tool) } +/** Classify a connector tool for the shared approval seam. MCP gives us only the + * tool name, so read-verb tools are reads and everything else is a mutate — we + * cannot tell an irreversible connector call from a recoverable one by name, so + * we gate conservatively as mutate rather than guessing 'irreversible'. */ +export function riskOf(tool: string): ActionRisk { + return isActionTool(tool) ? 'mutate' : 'read' +} + export function buildConnectorToolSchema( connector: { id: number; name: string }, tool: ConnectorToolDefinition diff --git a/src/main/tools/mcpConnectorToolExtension.ts b/src/main/tools/mcpConnectorToolExtension.ts index 12192dc9..0a77241b 100644 --- a/src/main/tools/mcpConnectorToolExtension.ts +++ b/src/main/tools/mcpConnectorToolExtension.ts @@ -2,18 +2,19 @@ // loop via registerToolExtension. Connector tools are exposed to the model // namespaced as `mcp____` and executed directly. // -// Open-core seam: write tools first offer themselves to the `mcp:proposeApproval` -// hook — Pro registers it to route writes through its approval queue. In the free -// build no hook is registered, so connector tools just run. +// Open-core seam: mutating tools first offer themselves to the shared +// `actions:proposeApproval` hook via proposeActionApproval — Pro registers it to +// route writes through its approval queue. In the free build no hook is registered, +// so connector tools just run. import type { ToolExtension } from '../tools' import { listConnectors, fetchTools, callConnectorTool, setConnectorStatus } from '../mcp' -import { callHook } from '../bootstrap/hookRegistry' +import { proposeActionApproval, shouldGate, type ActionApprovalRequest } from '../actions/approval' import { MCP_TOOL_PREFIX, buildConnectorToolSchema, formatConnectorToolResult, - isActionTool, + riskOf, type ConnectorToolDefinition } from './mcpConnectorToolExtension-logic' @@ -30,13 +31,13 @@ export interface McpConnectorToolBoundary { tool: string, args: Record ) => Promise - proposeApproval: (request: Record) => boolean | undefined + proposeApproval: (request: ActionApprovalRequest) => boolean | undefined } const productionBoundary: McpConnectorToolBoundary = { fetchTools, callTool: callConnectorTool, - proposeApproval: (request) => callHook('mcp:proposeApproval', request) + proposeApproval: proposeActionApproval } export class McpConnectorToolExtension implements ToolExtension { @@ -90,11 +91,14 @@ export class McpConnectorToolExtension implements ToolExtension { async execute(name: string, args: Record): Promise { const meta = this.byName.get(name) if (!meta) return `Error: unknown connector tool ${name}` - // Pro can intercept writes for approval; returns true if it queued the action. - if (isActionTool(meta.tool)) { + // Pro can intercept mutating tools for approval; returns true if it queued them. + const risk = riskOf(meta.tool) + if (shouldGate(risk)) { const queued = this.boundary.proposeApproval({ + kind: 'mcp', title: `${meta.tool} via ${meta.connector}`, detail: `Requested from chat. Arguments: ${JSON.stringify(args)}`, + risk, connectorId: meta.id, connector: meta.connector, tool: meta.tool, diff --git a/src/main/tools/nativeActionToolExtension-logic.ts b/src/main/tools/nativeActionToolExtension-logic.ts new file mode 100644 index 00000000..06aa10b3 --- /dev/null +++ b/src/main/tools/nativeActionToolExtension-logic.ts @@ -0,0 +1,227 @@ +// Pure logic for the native-action tool extension: the table of semantic tools the +// model can call (calendar today; reminders / contacts / photos add as rows), plus +// schema building, risk classification, argument mapping, and result formatting. No +// Electron or process I/O here, so it is unit testable; the extension shell wires it +// to runNativeAction + the approval seam. + +import type { ActionRisk } from '../actions/approval' + +export interface NativeToolSpec { + /** Model-facing tool name. */ + name: string + /** Model-facing description (kept plain, no marketing voice — this is a prompt). */ + description: string + /** JSON schema for the tool's arguments. */ + parameters: Record + /** The native helper command this tool invokes. */ + command: string + /** How consequential the action is — decides whether it gates for approval. */ + risk: ActionRisk + /** Map the model's tool arguments to the helper command's args. */ + buildArgs: (toolArgs: Record) => Record + /** One-line, user-facing approval-card title for a gated action. */ + title: (toolArgs: Record) => string + /** Turn a successful helper result into a string for the model. */ + formatResult: (result: unknown) => string +} + +function asString(value: unknown, fallback = ''): string { + return typeof value === 'string' ? value : fallback +} + +/** Shared "Created the