From 9e958f44aca885e4e00945f40265ef2a24fba1dd Mon Sep 17 00:00:00 2001 From: softmaxe <94467584+softmaxe@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:19:23 +0800 Subject: [PATCH] feat(verification): add native Whisper verification skill --- .agents/skills/verify-whisper/SKILL.md | 110 +++++++ .../skills/verify-whisper/features/README.md | 33 +++ .../verify-whisper/features/dictation.md | 38 +++ .../verify-whisper/features/dictionary.md | 45 +++ .../skills/verify-whisper/features/history.md | 37 +++ .../verify-whisper/features/settings.md | 37 +++ .../skills/verify-whisper/features/upload.md | 34 +++ .../skills/verify-whisper/scripts/control.py | 276 ++++++++++++++++++ 8 files changed, 610 insertions(+) create mode 100644 .agents/skills/verify-whisper/SKILL.md create mode 100644 .agents/skills/verify-whisper/features/README.md create mode 100644 .agents/skills/verify-whisper/features/dictation.md create mode 100644 .agents/skills/verify-whisper/features/dictionary.md create mode 100644 .agents/skills/verify-whisper/features/history.md create mode 100644 .agents/skills/verify-whisper/features/settings.md create mode 100644 .agents/skills/verify-whisper/features/upload.md create mode 100755 .agents/skills/verify-whisper/scripts/control.py diff --git a/.agents/skills/verify-whisper/SKILL.md b/.agents/skills/verify-whisper/SKILL.md new file mode 100644 index 0000000..361c453 --- /dev/null +++ b/.agents/skills/verify-whisper/SKILL.md @@ -0,0 +1,110 @@ +--- +name: verify-whisper +description: Verify Whisper's macOS GPUI desktop app through native Accessibility input, with isolated data, process ownership checks, and persistent screenshots and database evidence. Use after user-facing behavior changes or to reproduce Dictionary, History, Upload, Settings, and dictation journeys. +--- + +# Verify Whisper + +Whisper is an Apple Silicon macOS menu-bar dictation app with a GPUI control panel. Its primary surface is native UI; there is no browser, CDP, HTTP control API, or interactive CLI. Rust unit and service tests (`scripts/check.sh`) supplement real UI verification. + +Read [features/README.md](features/README.md), then the requested feature. Commands below run from the checkout root. Python helpers use `uv`; no extra Python packages are needed. + +## Launch + +Requirements: an Apple Silicon Mac with a logged-in GUI session, the pinned Rust toolchain, Xcode command line tools, `uv` with Python 3.11+, and the `cua_repl` native-app tool. Check repository contribution rules before editing. This workflow uses the debug executable and a disposable app wrapper, without running release packaging, signing, or replacing the installed app. Read `docs/macos-signing.md` before extending this to signing or packaging changes. + +```sh +cargo build --locked +VERIFY_RUN="$PWD/.e2e/verify-whisper/run-$(date +%Y%m%d-%H%M%S)" +uv run --no-project .agents/skills/verify-whisper/scripts/control.py launch --run-dir "$VERIFY_RUN" +``` + +Keep `VERIFY_RUN` for every subsequent command; use its absolute value in CUA file writes. If the shell does not retain variables, recover the chosen path from the launch invocation, not from whichever run is newest. Each launch requires a new run directory. + +The helper copies the unchanged `target/debug/whisper` into `scratch/Whisper.app/Contents/MacOS/whisper` and starts it in its own process group with an **empty** `scratch/data` directory. The wrapper reuses `packaging/Info.plist`, its original identifiers, and the Cargo version. It is verification scaffolding because CUA rejects bare executables with `Invalid app`. It does not provide release-signature or permission-retention proof. Whisper performs its own SQLite migrations (currently `user_version=4`). No seed rows, server configuration, or credentials are installed. Default interface language is English; History saving is on; ASR and cleanup URLs/models are empty. Dictionary and snippets are usable offline from this state. + +The helper sets `WHISPER_USER_DATA_DIR` and `WHISPER_HELPERS_DIR` explicitly and clears inherited `WHISPER_*` overrides. It sets `WHISPER_DEV_SCENE=home`: in `src/app.rs` this selects the ordinary History view and skips `start_hotkeys`. It injects no Dictionary or History records. Do not use `pill:*` scenes to claim real recording or processing behavior. + +Launch waits for the owned PID, source/copied binary hashes, lock PID, migrated DB, and `retention cleanup:` log message. `ready-for-ui-check` still requires a rendered UI check: bind the **run's exact app bundle path**, inspect its AX tree and screenshot, and require the Whisper window with `History`, `Upload`, and `Dictionary` navigation. An empty History should show `No transcripts yet`. A live process alone is insufficient. + +Use `cua_repl` for all UI input. Its first invocation must execute exactly the **`cua_bind` JavaScript command printed by Launch**, which calls `cua.getApp` with the run's exact `bundle` path. Read the returned CUA documentation before further calls. In a later invocation execute the printed `cua_evidence` assignment to set the evidence path. Never select by the display name `Whisper` or the shared bundle ID when the installed app is running. If exact-path binding cannot distinguish the owned process, stop and clean up; do not drive the installed session. After a compacted computer-use session, call `await cua.rewriteDocumentation()` first. + +On the validated host, the first AX read exposed only the window/menu. `await app.pressKey("Tab"); await app.getAXState({disableDiffing:true});` activated/populated the GPUI tree. AX clicks could leave the old tree visible until another native event; a subsequent Tab and fresh tree exposed the resulting state. Click the newly observed text field before typing if Tab moved focus. Coordinate clicks returned `Computer Use server error -10005: noWindowsAvailable`; AX-index clicks plus keyboard input worked. Do not keep retrying coordinates after that error. + +File isolation permits a panel-only run alongside the installed app, but global shortcuts, menu-bar entries, clipboard, Keychain, OS permissions, and login items are shared. Do not run two agents against the same verification instance. Do not edit API keys: Keychain uses fixed service/account names even under another data directory, and merely opening Settings constructs a Keychain reader. Avoid Settings during the default Dictionary proof. A Keychain authorization prompt is a blocker, not a reason to approve or reset permissions automatically. + +On launch failure the helper cleans its own process group and scratch state while retaining `run.json` and any `app.log` already created. Teardown after any later failure is the Cleanup command below. + +## Doctor + +Run this read-only command first whenever behavior looks wrong: + +```sh +uv run --no-project .agents/skills/verify-whisper/scripts/control.py doctor --run-dir "$VERIFY_RUN" +``` + +It verifies the recorded PID/start identity/process group, executable mapping and SHA-256, data lock, completed startup sweep, SQLite schema, and stored feature state. It never repairs state or supplies authentication. Pair its result with a fresh AX observation: neither a health check nor `Test Connection` proves a transcription journey. + +## Drive + +Use `app.getAXState()` to obtain current numeric indices. Click controls named by the feature map; GPUI IDs such as `dict-add-word` are source identifiers, **not** DOM selectors or guaranteed AX identifiers. Never carry an index across a UI transition without a new AX observation. If AX does not expose an actionable control, inspect a fresh screenshot, click the visible target, and record that fallback. + +For the default offline proof, follow [Dictionary](features/dictionary.md): navigate through `Dictionary`, choose `Add Your First Word`, enter `Verification Quokka`, press Return, and verify the visible row plus a read-only DB query. Navigate to `History` and back through `Dictionary` to confirm the row survives view reconstruction. This proves the empty-state add entry point, not Import, snippets, dictation influence, or every alternate add entry point. + +Typical native input calls (indices must come from the latest actual tree): + +```javascript +await app.click(index); +await app.getAXState(); +await app.typeText("Verification Quokka"); +await app.pressKey("Return"); +await app.getAXState(); +``` + +Record each action with its intended accessible name and observed outcome in `actions.jsonl` under the run directory. Use real typing/clicks, not setters on app internals, direct DB mutations, or test-only endpoints. + +## Evidence + +All proof artifacts live in `$VERIFY_RUN` (ignored by git through `/.e2e`), **outside** `scratch`. Save initial, action, result, and reopened AX/screenshot pairs, the action transcript, Doctor output, and stored state: + +```sh +uv run --no-project .agents/skills/verify-whisper/scripts/control.py doctor --run-dir "$VERIFY_RUN" > "$VERIFY_RUN/doctor.json" +uv run --no-project .agents/skills/verify-whisper/scripts/control.py state --run-dir "$VERIFY_RUN" > "$VERIFY_RUN/stored-state.json" +``` + +`state` uses a read-only SQLite connection and reports Dictionary/snippet contents plus History/analytics counts. Run it before cleanup. It is a second observation, not a way to seed or drive the feature. + +In CUA, standard Node filesystem writes can persist observations without synthesizing UI input: + +```javascript +var fs = await import("node:fs/promises"); +await fs.writeFile(evidence + "/before.ax.txt", await app.getAXState({emit:false, disableDiffing:true})); +await fs.writeFile(evidence + "/before.png", await app.getScreenshot({emit:false})); +``` + +Set `evidence` using the Launch output as described above. Repeat as `action`, `after`, and `reopened`. A full AX snapshot can renumber indices, even with `emit:false`; after saving screenshots or hidden AX captures, emit a fresh full tree before choosing an index. Capture the command/action and resulting state, not only a final screen. Mutation proofs require stored state alongside visible feedback; upload/dictation proofs additionally require resulting transcript/audio/files and the intended pasted text or clipboard. Use only disposable content. Do not log credentials or capture API-key inputs. Mocks are permitted only at existing production boundaries (for example an ASR HTTP server), must be disclosed, and cannot establish real-server or microphone compatibility. + +There is no dry-run flag. The panel workflow suppresses hotkeys but still initializes the native app, a lid watcher, menu bar, retention cleanup, and scratch files. Do not claim it is globally side-effect-free or network-isolated. Record the actual scene, empty initial data, processes, files, and configured endpoints; offline Dictionary success proves no ASR behavior. Unexpected network or authorization activity must be investigated, not dismissed because this is a verification run. + +For UI code changes, capture each changed view before and after, embed the pairs in the reply, and identify views that render identically. Motion/timing changes also need a short video using a recording facility available in that session; static snapshots cannot prove timing. Creating this skill does not change app UI. + +## Cleanup + +```sh +uv run --no-project .agents/skills/verify-whisper/scripts/control.py cleanup --run-dir "$VERIFY_RUN" +ls -l "$VERIFY_RUN"/before.png "$VERIFY_RUN"/after.png "$VERIFY_RUN"/stored-state.json "$VERIFY_RUN"/cleanup.json +``` + +The helper checks ownership, terminates only its recorded process group (including its native helpers), escalates after five seconds if needed, and removes `scratch`. It never kills by name or touches the installed app. Run it after **every** failed attempt too. `Cmd+W` only hides/closes the control panel and leaves the menu-bar process running, so it is not teardown. Retain evidence, logs, `run.json`, and `cleanup.json`; confirm the screenshot/state files still exist after cleanup. Finish the CUA portion with exactly one `completion_summary` call after observing its final outcome. + +## Helpers + +[`scripts/control.py`](scripts/control.py) is executable. The commands above show all four actions: `launch`, read-only `doctor`, read-only `state`, and `cleanup`; every action requires `--run-dir`. Do not reuse a cleaned run for another launch. Rebuild and launch a new run after changing the executable. + +Keep journeys and preconditions current in the feature map. Use `/maintain-verification-skill` as the app evolves. + +## Validated baseline + +The initial run on macOS built the actual checkout with `cargo build --locked`, launched from empty data, passed Doctor, and drove `words-add` through `Add Your First Word` with real AX clicks/typing/Return. `Verification Quokka` appeared on screen, survived History-to-Dictionary navigation, and existed as one `manual` SQLite row with zero uses; History and analytics counts remained zero. Cleanup terminated the owned app/helper processes and removed scratch while the installed app continued running. Screenshots, AX snapshots, actions, Doctor, stored state, and cleanup evidence survived under `.e2e/verify-whisper/proof-dictionary-bundle/`. Additional helper checks covered failure before spawning, metadata-write failure after spawning, and repeat cleanup; their results are in `helper-failure-checks.json` beside the UI evidence. + +This ran on the actual local Mac, not a container or simulated desktop. The temporary wrapper is scaffolding for CUA; this proof did not establish packaged-release behavior, process-restart persistence, real microphone/server transcription, or the other entry points in the feature map. diff --git a/.agents/skills/verify-whisper/features/README.md b/.agents/skills/verify-whisper/features/README.md new file mode 100644 index 0000000..3c1c3e6 --- /dev/null +++ b/.agents/skills/verify-whisper/features/README.md @@ -0,0 +1,33 @@ +# Whisper verification map + +This is the maintained source for user-facing verification. Read the parent [skill](../SKILL.md) for launch, ownership, CUA, evidence, and cleanup instructions. + +## Baseline preconditions + +- Use a new helper-owned debug instance with empty isolated data, English UI, and `WHISPER_DEV_SCENE=home`. +- Require Doctor success and the actual rendered Whisper control panel before driving it. +- There are no transcripts, words, snippets, analytics events, ASR/cleanup endpoints, or supplied credentials. The offline Dictionary journey creates its own data through the UI. +- Native Accessibility tooling must be available. Bind by the helper's exact app bundle path, never the installed app's name or bundle ID. +- The default panel-only scene disables global hotkeys; it is unsuitable for a keyboard dictation proof. + +## Driving conventions + +- All feature recipes use `cua_repl` and the `app` binding from Launch. Resolve `index` from a fresh `await app.getAXState()` by the stated accessible name; the recipes do not specify fixed indices. +- `await app.click(index)` activates that named element. `await app.typeText(text)` types into the focused input; `await app.pressKey(key)` sends real keyboard input. Inspect AX after each transition before choosing the next index. See the parent skill for initial Tab activation and Tab refresh after a stale AX-click result. Full/hidden AX snapshots can renumber indices; read a fresh emitted tree after saving evidence. +- If AX omits a target, save the screenshot and document a visual-coordinate fallback; do not invent an AX selector. +- Start from baseline unless the recipe states a stronger precondition. Create required rows through actual upload/dictation/UI actions; do not insert them in SQLite. + +## Proof and failure reporting + +- Save feature ID, entry point, action, resulting AX/screenshot, and read-only stored state under `$VERIFY_RUN`; preserve these through cleanup. +- A feature can have multiple entry points. Passing one never certifies another: report each as passed, failed with its unmet precondition, or not exercised. +- Missing audio, server, permission, or expected seed state is a failed journey precondition, not a successful empty-state proof. Record the reason in the matching feature's Gotchas when a new limitation is discovered. +- Initial generated-skill validation is intentionally one Dictionary journey. Other mapped paths are instructions for later coverage, not claims of execution. + +## Features + +- [Dictionary and snippets](dictionary.md): empty-state/header creation, import/export, filtering, edit/delete/undo, snippet replacement definitions. +- [History, search, and Insights](history.md): saved text, raw text, retry, deletion, search, dictation statistics. +- [File transcription](upload.md): chooser/drop, single/batch requests, cancellation, result and History. +- [Settings](settings.md): entry paths, appearance/language, server configuration, microphone/retention controls. +- [Live dictation](dictation.md): actual shortcut/menu start, finish/cancel, paste/fallback, cleanup. diff --git a/.agents/skills/verify-whisper/features/dictation.md b/.agents/skills/verify-whisper/features/dictation.md new file mode 100644 index 0000000..5590cf1 --- /dev/null +++ b/.agents/skills/verify-whisper/features/dictation.md @@ -0,0 +1,38 @@ +# Live dictation + +Users record speech with a global shortcut or menu-bar action, send it to ASR and optionally cleanup, then receive pasted text or a copy fallback in their target application. + +## Sub-features + +- `dictation-tap-hold`: double-tap and hold shortcut activation. +- `dictation-menu`: start/stop through the menu bar. +- `dictation-finish-cancel`: actual recording finish/cancel and pill feedback. +- `dictation-cleanup-delivery`: cleanup/skip/fallback and text delivery. +- `dictation-dictionary`: spoken terms/snippets and usage updates. + +## How to get to it (user POV) + +- Default empty configuration uses Globe: double-tap to begin hands-free, tap to finish. In hold mode press/release the configured shortcut. +- Menu-bar action `Start listening` / `Stop listening`. +- Recording pill buttons `Finish Dictation`, `Cancel Dictation`, and processing `Skip cleanup`; Escape cancels recording. + +## Driving it with cua_repl + +Preconditions: + +- Use a dedicated macOS test account/session with no other Whisper/global-shortcut driver, actual microphone and Accessibility authorization, configured ASR, and a disposable target editor. +- The standard helper's `home` scene intentionally disables global hotkeys. It cannot prove shortcut activation. A real dictation run needs an explicitly owned launch without `WHISPER_DEV_SCENE`, after excluding other instances; record its exact executable/PID/data directory and provide equivalent Doctor/cleanup ownership checks. Do not quit the user's installed session without authorization. +- Use synthetic/disposable spoken content and a known endpoint; cleanup needs its own configured model/server. Do not obtain or expose shared Keychain credentials for this proof. + +- **Target.** Bind the disposable editor with `cua.getApp`, focus its editable document, and capture the empty target. Resolve app/menu/pill AX targets separately; don't send dictation input into a real user's document. +- **Shortcut.** Through CUA send the configured key press/release sequence; for held keyboard keys use the documented `durationMs` press option. Require an actual live recording pill/timer and the active microphone. If CUA cannot synthesize Globe on this host, report that entry blocked; another shortcut/menu path does not certify Globe. +- **Menu/pill.** Independently choose `Start listening` from this run's menu-bar entry, speak the known sentence, then click `Finish Dictation` or choose `Stop listening`. Capture the recording action, ASR/cleanup progress, and delivered text. On a separate recording click `Cancel Dictation`/Escape; require no delivered text and verify stored side effects. +- **Delivery.** Inspect the target editor for the expected sentence and query History/audio state. When automatic paste fails, capture the actual fallback and paste the copied text into the disposable target. Exercise `Skip cleanup` during an actual pending cleanup request, not a dev-scene animation. +- **Dictionary.** Create a word/snippet through the mapped Dictionary UI, dictate its spoken form, and verify output/usage. Preserve the real audio fixture or spoken-content transcript and server response evidence where available, using disposable content only. + +## Gotchas + +- The baseline deliberately lacks server configuration, microphone authorization, a target editor, and registered hotkeys: live dictation fails its prerequisites there. This feature was not established by the generated skill's offline proof. +- Data-directory isolation alone permits duplicate recording/paste from global listeners. Do not launch a second live shortcut instance beside the installed app. +- `WHISPER_DEV_SCENE=pill:*` changes internal presentation state; its screenshots never prove capture, upload, cleanup, permission behavior, or paste. +- Real microphone/Accessibility/Keychain retention after a Homebrew upgrade belongs to `docs/testing.md`'s release smoke check and `docs/macos-signing.md`; a debug verification run cannot establish it. diff --git a/.agents/skills/verify-whisper/features/dictionary.md b/.agents/skills/verify-whisper/features/dictionary.md new file mode 100644 index 0000000..a486827 --- /dev/null +++ b/.agents/skills/verify-whisper/features/dictionary.md @@ -0,0 +1,45 @@ +# Dictionary and snippets + +Users teach Whisper names and terms, import word lists, and define spoken triggers that expand to saved text. Editing these definitions is available offline; proving their effect on transcription requires live dictation. + +## Sub-features + +- `words-add`: empty-state and header add paths save manual words. +- `words-import-export`: import comma/newline lists and export the full dictionary through its page menu, regardless of the current filter. +- `words-filter-edit`: filter words, edit a row, and reject duplicates. +- `words-remove-undo`: remove a row and undo within its deadline. +- `snippets-create-edit`: create, filter, edit, and remove trigger/replacement definitions. +- `dictionary-influence`: actual dictation uses words/snippets; Upload and History retry do not increment dictionary usage. + +## How to get to it (user POV) + +- Sidebar `Dictionary` > `Words`; choose `Add Your First Word` or `Add Word`. +- Empty state `Import a List`, or `Dictionary actions` > `Import List…`; populated page menu also offers `Export…`, `Dictionary Settings…`, and `Clear Dictionary…`. +- Search with `Filter words`, or use `Add “”` from the no-match result. +- Switch to `Snippets` > `New Snippet` (header or empty-state action). +- Focus a word/snippet row and press Return to edit or Delete to remove; row actions/context menus offer editing and removal too. + +## Driving it with cua_repl + +Preconditions: + +- Baseline instance is healthy. No word equals `Verification Quokka`. +- Read the shared index for resolving each `index`; never reuse indices from a previous observation. + +- **Reach empty Dictionary.** `await app.click(index)` on `Dictionary`, then `await app.pressKey("Tab"); await app.getAXState({disableDiffing:true});`. Require `Your dictionary is empty` in the screenshot and Words/Snippets/add controls in AX; capture `before` evidence, then emit a fresh full tree for new indices. +- **Empty-state add.** Click `Add Your First Word`, send Tab and inspect a fresh full AX tree, then click the `Add words, separated by commas` field and `await app.typeText("Verification Quokka")`. Capture `action` evidence with the entered value before submitting. +- **Save.** `await app.pressKey("Return"); await app.getAXState();` Require the row `Verification Quokka` and `Added 1 word` feedback (toast may expire). Alternatively activate the add panel's `Add` button; mark the actual entry point used. +- **Stored proof.** Run the skill's `state` command. Require exactly one matching dictionary row with source `manual` and use count `0`; no transcripts or analytics should have been created by this offline action. Capture `after` evidence. +- **Reopen.** Click `History`, send Tab and inspect AX; click `Dictionary`, send Tab and inspect AX. Require the same word row and capture `reopened` evidence. This verifies leaving/returning to the view plus persisted SQLite state, not a process-restart journey. +- **Header add and duplicate.** On the populated page, click `Add Word`, type `Verification Otter`, and Return. Re-enter `verification quokka`; require duplicate feedback and still one case-insensitive matching DB row. This is a separate path from the initial proof. +- **Filter/edit/remove.** Click the `Filter words` input and type `Quokka`; require only the matching row. Focus that row and Return, replace its word with `Verification Quokka Revised`, then Return. Read DB state. Focus the row and Delete; require removal and `Undo`, activate `Undo` and require restoration. To prove committed deletion, repeat without Undo and wait for the six-second deadline, then query state. +- **Import/export.** Click the stated empty-state or page-menu Import entry, paste `Verification Finch\nVerification Heron` in the import field using `await app.paste("Verification Finch\nVerification Heron")`, and activate `Import`. Require two DB rows. Use `Dictionary actions` > `Export…`, save through the native dialog to `$VERIFY_RUN/export.txt`, and read that file to verify the expected words. Cover each entry separately. +- **Snippet definition.** Click `Snippets`, `New Snippet`; type `verification signoff` in `e.g. cal link` and `Regards, Verification` in the replacement field. Choose `Create Snippet` or `await app.pressKey("super+Return")`. Require the visible row plus matching `snippets` state. Reopen it with Return, edit and save; verify both values again. Filtering uses `Filter snippets`. These actions do not prove spoken expansion. + +## Gotchas + +- Empty state exposes two add controls; label the entry actually tested. Header, no-match Add, import, context menu, and keyboard paths remain separate coverage. +- GPUI text inputs may appear as multiple AX nodes; click the current editable field, not its label. Adding words keeps the panel open and clears its input. +- An empty Dictionary has no filter field or page menu. Import/export use native clipboard/save UI and must stay within scratch/evidence destinations. +- Undo has a six-second deadline. Clear Dictionary is destructive and cannot be undone; it is not part of the initial proof. +- Empty baseline cannot prove learned-word filters or dictation influence. Those paths fail their precondition until a real dictation/learning journey produces the needed state. diff --git a/.agents/skills/verify-whisper/features/history.md b/.agents/skills/verify-whisper/features/history.md new file mode 100644 index 0000000..6255048 --- /dev/null +++ b/.agents/skills/verify-whisper/features/history.md @@ -0,0 +1,37 @@ +# History, search, and Insights + +Users inspect saved dictations/uploads, compare raw and cleaned text, copy or retry audio, search transcripts, and see dictation activity statistics. + +## Sub-features + +- `history-read-copy`: open saved text, copy, and expand differing raw text. +- `history-search`: toolbar and Cmd+K search, matching/empty results, Return to a result. +- `history-retry`: transcribe retained audio again. +- `history-delete`: delete/Undo, deadline commit, close/reopen persistence. +- `insights-period`: inspect dictation totals and period comparisons. + +## How to get to it (user POV) + +- Sidebar `History`; each transcript row exposes `Copy text`, optional `Show raw transcript`, and `More actions`. +- Toolbar search or `Cmd+K` opens `Search transcripts`. +- Row `More actions` > `Re-transcribe`, `Reveal in Finder`, or `Delete`; keyboard row Return copies, Space expands raw text, Cmd+Backspace/Delete removes. +- Sidebar `Insights` opens aggregate dictation statistics. + +## Driving it with cua_repl + +Preconditions: + +- Produce a known transcript through [Upload](upload.md) or [dictation](dictation.md), with History saving enabled. Retry also needs retained audio and reachable ASR. Insights activity needs real dictation; an upload alone does not establish dictation analytics. + +- **Inspect/copy.** Click `History`, inspect AX, select the known transcript. Click `Copy text` or focus its row and `await app.pressKey("Return")`. Paste into a disposable editor to observe the clipboard result; preserve/restore the user's clipboard if the tooling permits. Capture action and result, plus the read-only History count. +- **Raw text.** Click `Show raw transcript`, inspect AX and require the actual raw text, then `Hide raw transcript`. When raw and final text are identical the control is absent and this sub-feature lacks its required fixture. +- **Search entries.** `await app.pressKey("super+k")`, inspect AX, type a known substring into `Search transcripts`, and require its result. Return should close search and highlight that History item. Separately use the visible toolbar search button. Replace the query with `VerificationNoMatchQuokka` to require `No results`; `Clear search` empties it and Escape closes search. +- **Retry.** Choose `More actions` > `Re-transcribe`; capture request progress and the resulting text. Verify retained audio existed before the action and query stored state afterward. A connection-test success is insufficient. +- **Delete/Undo.** Choose `Delete` or the row keyboard shortcut; require `Transcript deleted` and `Undo`. Activate Undo before six seconds and require the row. Repeat without Undo, then close/reopen the panel after the deadline and require the stored count to decrease. Never run this on the installed database. +- **Insights.** Click `Insights`, choose the visible period control from the fresh AX tree, and compare its rendered totals to read-only `analytics_events` for the selected dates. Save both periods' screenshots/data; an empty panel is only empty-state evidence. + +## Gotchas + +- Fresh data has no transcripts, retained audio, or analytics: populated search, raw toggle, retry, deletion, and nonempty Insights journeys fail until real actions create their prerequisites. Do not seed SQLite to bypass this. +- History saving/retention settings can remove prerequisites. A hidden row during Undo is not yet a committed deletion. +- `Reveal in Finder` opens another app; close only the window the run opened. Do not mistake files from a different instance for proof. diff --git a/.agents/skills/verify-whisper/features/settings.md b/.agents/skills/verify-whisper/features/settings.md new file mode 100644 index 0000000..42b6c79 --- /dev/null +++ b/.agents/skills/verify-whisper/features/settings.md @@ -0,0 +1,37 @@ +# Settings + +Users configure appearance/language, dictation controls, speech and cleanup servers, microphone selection, and local data retention. + +## Sub-features + +- `settings-entry`: sidebar, app menu, Cmd+comma; close and restore focus. +- `settings-appearance`: theme/language change visibly and persist. +- `settings-servers`: actual connection tests, model selection, optional cleanup. +- `settings-dictation-privacy`: microphone/test, controls and retention behavior. + +## How to get to it (user POV) + +- Sidebar `Settings`, `Cmd+,`, or app menu `Whisper` > `Settings…`. +- Sections `General`, `Dictation`, `Speech to text`, `Text cleanup`, `Privacy & data`. +- Upload's `Speech to Text Settings…` and Dictionary's `Dictionary Settings…` are contextual entries. +- Close with `Close settings`, Escape, or the visible backdrop. + +## Driving it with cua_repl + +Preconditions: + +- A healthy isolated instance; understand the shared Keychain limitation in the parent skill before opening Settings. Do not authorize a Keychain prompt or edit a stored API key as part of routine verification. +- Server paths additionally need a disposable reachable endpoint/model; microphone paths need actual macOS permission and hardware in a suitable test session. + +- **Entry/close.** `await app.pressKey("super+comma")`, inspect AX and require the sheet/section names. Close with `await app.pressKey("Escape")` and check focus returns to the previous page. Repeat through sidebar, menu, and relevant contextual entries to claim their coverage. +- **Appearance.** Click `General`, then `Dark`; inspect AX and capture the rendered change. Read `scratch/data/settings.json` and require `theme` equal to `dark`. Repeat `Light`, `Tokyo Night`, and `Auto` only as required, using current indices. Change `Interface language` and require the actual translated navigation, not just a saved value. Record which language subsequent selectors use. +- **Server.** Click `Speech to text`, fill the visible `Server URL` and `Model` inputs with the disposable endpoint/model, choose `Test Connection`, and capture success or specific failure. Repeat cleanup under `Text cleanup` where needed; never record API key content. Then prove a real Upload/dictation journey using that configuration. +- **Controls/privacy.** Open `Dictation` to select the microphone and exercise its actual test recording/playback; open `Privacy & data` to change `Save history`/retention only within isolated data and prove the resulting History/audio behavior through a new real transcription. +- **Stored proof.** Read only the relevant nonsecret `settings.json` keys, close/reopen Settings and verify selection/rendering. Report restart persistence separately; the initial Dictionary proof does not restart the process. + +## Gotchas + +- `WHISPER_USER_DATA_DIR` does not isolate Keychain, Accessibility/microphone permission, login items, media controls, or system preferences. Avoid Start at login and permission resets in routine runs. +- Constructing Settings reads the fixed cleanup Keychain entry even before choosing Text cleanup. If this triggers authorization, stop that journey and retain the blocker evidence. +- No server or microphone authorization exists in the baseline. Never downgrade failed connection/recording prerequisites into a pass for opening the sheet. +- Menus are localized and dropdown queries reset when reopened. Resolve fresh accessible names after a language change. diff --git a/.agents/skills/verify-whisper/features/upload.md b/.agents/skills/verify-whisper/features/upload.md new file mode 100644 index 0000000..1bc53c9 --- /dev/null +++ b/.agents/skills/verify-whisper/features/upload.md @@ -0,0 +1,34 @@ +# File transcription + +Users choose or drop audio/video files, transcribe one or a batch, inspect the text, and open saved results in History. + +## Sub-features + +- `upload-choose-drop`: file chooser and drop-target entry paths. +- `upload-single-batch`: single and batch progress, final text, History and audio storage. +- `upload-cancel-retry`: cancellation, failed-file retry, reset for another upload. + +## How to get to it (user POV) + +- Sidebar `Upload` > `Choose Files`, click the drop target, or drag actual files onto it. +- `Speech to Text Settings…` opens server configuration. +- `Transcribe` / `Transcribe N Files` starts work; `Cancel Transcription` / `Cancel All` cancels. +- Success offers `Copy Full Text`, `View in History`, `Upload Another`; batches offer `Retry failed` and `Clear` as appropriate. + +## Driving it with cua_repl + +Preconditions: + +- A disposable known spoken-audio fixture exists and a compatible ASR server/model is configured through Settings. Record fixture path/hash and intended endpoint. History saving is on for the History assertion. Keep fixtures and outputs under the run directory, outside `scratch` if they are evidence. + +- **Choose file.** Click `Upload`, inspect AX, then `Choose Files`. Use the native file dialog (Cmd+Shift+G with `await app.pressKey("super+shift+g")` if needed) to select the fixture's absolute path. Require its filename in the queue. Exercise clicking the drop target and actual file drag/drop separately when claiming those paths. +- **Transcribe.** Click `Transcribe`, inspect AX and capture active progress. Require `Transcription complete` and the expected transcript; record actual requests/responses at a production ASR boundary when available, without recording credentials. +- **Side effects.** Click `View in History`, require matching text, query the isolated database and list its `audio` directory before cleanup. Copy the relevant disposable audio/file manifest into evidence. Upload success with History off requires a different expected stored state; do not silently change the assertion. +- **Batch/cancel.** Choose two fixtures and click `Transcribe 2 Files`; capture each queue outcome and stored results. On a separate sufficiently long request activate `Cancel All` (or single `Cancel Transcription`), require processing to stop, and check for partially saved results. Retry a genuine failure with `Retry failed` and inspect the new result; `Upload Another` returns to input state. + +## Gotchas + +- Baseline has no server/model or audio fixture: successful transcription fails its precondition until supplied. An unconfigured empty view is not a transcription proof. +- MP3, WAV, M4A, WebM, Ogg/Opus, FLAC and AAC can upload directly. Video/other audio may need ffmpeg from `resources/bin`, Homebrew, or PATH; record which executable was used. +- A fake ASR can test UI/storage at the existing HTTP boundary but cannot prove actual recognition quality or compatibility with the intended server. Connection tests alone do not exercise upload. +- Native file dialogs and Finder drag/drop require their own current AX bindings; do not manufacture an internal queued-file object. diff --git a/.agents/skills/verify-whisper/scripts/control.py b/.agents/skills/verify-whisper/scripts/control.py new file mode 100755 index 0000000..2edb548 --- /dev/null +++ b/.agents/skills/verify-whisper/scripts/control.py @@ -0,0 +1,276 @@ +#!/usr/bin/env python3 +"""Own an isolated Whisper process and retain verification evidence.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import platform +import plistlib +import shutil +import signal +import sqlite3 +import subprocess +import time +import tomllib +from pathlib import Path +from typing import Any + +ROOT = Path(__file__).resolve().parents[4] + + +def command(*args: str) -> str: + return subprocess.check_output(args, text=True).strip() + + +def digest(path: Path) -> str: + with path.open("rb") as stream: + return hashlib.file_digest(stream, "sha256").hexdigest() + + +def identity(pid: int) -> str: + result = subprocess.run( + ["/bin/ps", "-p", str(pid), "-o", "lstart=,pgid=,comm="], + capture_output=True, + text=True, + check=False, + ) + return result.stdout.strip() + + +def write_json(path: Path, value: Any) -> None: + path.write_text(json.dumps(value, indent=2) + "\n") + + +def load(run: Path) -> dict[str, Any]: + data: dict[str, Any] = json.loads((run / "run.json").read_text()) + if Path(data["run"]) != run or Path(data["root"]) != ROOT: + raise RuntimeError("Run belongs to a different directory or checkout") + return data + + +def database_state(run: Path) -> dict[str, Any]: + database = run / "scratch" / "data" / "transcriptions.db" + with sqlite3.connect(database.as_uri() + "?mode=ro", uri=True) as connection: + connection.execute("PRAGMA query_only = ON") + return { + "schema_version": connection.execute("PRAGMA user_version").fetchone()[0], + "dictionary": connection.execute( + "SELECT word, source, use_count FROM custom_dictionary ORDER BY id" + ).fetchall(), + "snippets": connection.execute( + "SELECT trigger, replacement FROM snippets ORDER BY id" + ).fetchall(), + "transcription_count": connection.execute( + "SELECT count(*) FROM transcriptions" + ).fetchone()[0], + "analytics_count": connection.execute( + "SELECT count(*) FROM analytics_events" + ).fetchone()[0], + } + + +def doctor(run: Path) -> dict[str, Any]: + data = load(run) + pid = data["pid"] + if not data["identity"] or identity(pid) != data["identity"]: + raise RuntimeError("Owned Whisper process is absent or its identity changed") + if int(command("/bin/ps", "-p", str(pid), "-o", "pgid=")) != pid: + raise RuntimeError("Whisper no longer owns its process group") + binary = Path(data["binary"]) + if ( + digest(binary) != data["binary_sha256"] + or digest(Path(data["source_binary"])) != data["binary_sha256"] + ): + raise RuntimeError("Binary changed since launch; clean up and launch again") + files = command("/usr/sbin/lsof", "-a", "-p", str(pid), "-d", "txt", "-Fn") + if f"n{binary}\n" not in files + "\n": + raise RuntimeError("Process is not running the recorded checkout binary") + lock = run / "scratch" / "data" / "instance.lock" + if lock.read_text().strip() != str(pid): + raise RuntimeError("Data directory lock does not name the owned process") + if "retention cleanup:" not in (run / "app.log").read_text(): + raise RuntimeError("Startup retention cleanup has not completed") + state = database_state(run) + if state["schema_version"] != 4: + raise RuntimeError("Expected database schema version 4") + return { + "status": "ready-for-ui-check", + "pid": pid, + "binary": str(binary), + "bundle": data["bundle"], + "cua_bind": f"var app = await cua.getApp({json.dumps(data['bundle'])});", + "cua_evidence": f"var evidence = {json.dumps(str(run))};", + "revision": data["revision"], + "scene": "home", + **state, + } + + +def group_alive(pid: int) -> bool: + # macOS can report EPERM for killpg(..., 0) on an exiting group. Inspect + # live members instead; a child zombie is reaped by launch's Popen handle. + rows = command("/bin/ps", "-axo", "pgid=,stat=").splitlines() + return any( + int(fields[0]) == pid and not fields[1].startswith("Z") + for row in rows + if len(fields := row.split()) == 2 + ) + + +def terminate_group(pid: int) -> None: + if not group_alive(pid): + return + try: + os.killpg(pid, signal.SIGTERM) + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + if not group_alive(pid): + break + time.sleep(0.1) + else: + os.killpg(pid, signal.SIGKILL) + except ProcessLookupError: + pass + + +def cleanup(run: Path) -> dict[str, Any]: + data = load(run) + if data.get("cleaned"): + return {"status": "already-cleaned", "evidence": str(run)} + pid = data.get("pid") + if pid is not None: + current = identity(pid) + if current and current != data["identity"]: + raise RuntimeError( + "PID was reused; refusing to signal it or remove scratch state" + ) + # With a dead leader, still reclaim its recorded process group and helpers. + if current and int(command("/bin/ps", "-p", str(pid), "-o", "pgid=")) != pid: + raise RuntimeError("Process group changed; refusing cleanup") + terminate_group(pid) + if (run / "scratch").exists(): + shutil.rmtree(run / "scratch") + data["cleaned"] = True + write_json(run / "run.json", data) + report = { + "status": "cleaned", + "evidence": str(run), + "retained_files": sorted(p.name for p in run.iterdir() if p.is_file()), + } + write_json(run / "cleanup.json", report) + return report + + +def launch(run: Path) -> dict[str, Any]: + if platform.system() != "Darwin" or platform.machine() != "arm64": + raise RuntimeError( + "Whisper verification requires an Apple Silicon macOS GUI session" + ) + source_binary = ROOT / "target" / "debug" / "whisper" + if not source_binary.is_file(): + raise RuntimeError( + "Build first with cargo build --locked (default target directory)" + ) + run.mkdir(parents=True, exist_ok=False) + scratch = run / "scratch" + process: subprocess.Popen[Any] | None = None + metadata: dict[str, Any] = {"run": str(run), "root": str(ROOT), "pid": None} + try: + write_json(run / "run.json", metadata) + data_dir = scratch / "data" + data_dir.mkdir(parents=True) + # CUA cannot bind a bare Mach-O. Retain the repository's identifiers + # in a disposable wrapper with the unchanged debug executable. + bundle = scratch / "Whisper.app" + binary = bundle / "Contents" / "MacOS" / "whisper" + binary.parent.mkdir(parents=True) + shutil.copy2(source_binary, binary) + with (ROOT / "packaging" / "Info.plist").open("rb") as stream: + info = plistlib.load(stream) + with (ROOT / "Cargo.toml").open("rb") as stream: + version = tomllib.load(stream)["package"]["version"] + info["CFBundleShortVersionString"] = version + info["CFBundleVersion"] = version + with (bundle / "Contents" / "Info.plist").open("wb") as stream: + plistlib.dump(info, stream) + env = os.environ.copy() + # home selects the initial view and disables global hotkey listeners; + # it injects no rows or simulated Dictionary actions. + for key in list(env): + if key.startswith("WHISPER_"): + del env[key] + env.update( + WHISPER_USER_DATA_DIR=str(data_dir), + WHISPER_DEV_SCENE="home", + WHISPER_HELPERS_DIR=str(ROOT / "resources" / "bin"), + RUST_LOG="info", + ) + metadata.update( + binary=str(binary), + source_binary=str(source_binary), + bundle=str(bundle), + binary_sha256=digest(binary), + revision=command("git", "-C", str(ROOT), "rev-parse", "HEAD"), + initial_data_entries=[], + scene="home", + ) + with (run / "app.log").open("w") as log: + process = subprocess.Popen( + [str(binary)], + cwd=ROOT, + env=env, + stdout=log, + stderr=subprocess.STDOUT, + start_new_session=True, + ) + metadata.update(pid=process.pid, identity=identity(process.pid)) + write_json(run / "run.json", metadata) + deadline = time.monotonic() + 30 + while time.monotonic() < deadline: + if process.poll() is not None: + raise RuntimeError( + f"Whisper exited {process.returncode}; inspect app.log" + ) + try: + return doctor(run) + except (RuntimeError, sqlite3.Error, FileNotFoundError): + time.sleep(0.2) + raise RuntimeError( + "Startup did not become ready within 30 seconds; inspect app.log" + ) + except BaseException: + # Keep the live Popen handle authoritative if metadata creation failed. + # Terminate before any further filesystem writes, including on disk errors. + if process is not None: + terminate_group(process.pid) + process.wait(timeout=5) + if scratch.exists(): + shutil.rmtree(scratch) + metadata["cleaned"] = True + write_json(run / "run.json", metadata) + write_json(run / "cleanup.json", {"status": "cleaned-after-launch-failure"}) + raise + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=["launch", "doctor", "state", "cleanup"]) + parser.add_argument("--run-dir", type=Path, required=True) + args = parser.parse_args() + run = args.run_dir.resolve() + if not run.is_relative_to(ROOT / ".e2e" / "verify-whisper"): + parser.error("--run-dir must be inside this checkout's .e2e/verify-whisper") + actions = { + "launch": launch, + "doctor": doctor, + "state": database_state, + "cleanup": cleanup, + } + print(json.dumps(actions[args.action](run), indent=2)) + + +if __name__ == "__main__": + main()