From f722162377b3712c1df1d91abadecc6211c1ea1f Mon Sep 17 00:00:00 2001 From: softmaxe <94467584+softmaxe@users.noreply.github.com> Date: Mon, 16 Nov 2026 22:16:39 +0800 Subject: [PATCH] chore: add QuotaBar verification skill --- .agents/skills/verify-quota-bar/SKILL.md | 195 ++++++++++++++++++ .../verify-quota-bar/features/README.md | 32 +++ .../verify-quota-bar/features/local-usage.md | 39 ++++ .../features/quota-and-recovery.md | 40 ++++ .../features/saved-usage-and-sources.md | 41 ++++ .../features/settings-and-pricing.md | 41 ++++ .../verify-quota-bar/features/usage-report.md | 40 ++++ .../verify-quota-bar/scripts/control.py | 150 ++++++++++++++ 8 files changed, 578 insertions(+) create mode 100644 .agents/skills/verify-quota-bar/SKILL.md create mode 100644 .agents/skills/verify-quota-bar/features/README.md create mode 100644 .agents/skills/verify-quota-bar/features/local-usage.md create mode 100644 .agents/skills/verify-quota-bar/features/quota-and-recovery.md create mode 100644 .agents/skills/verify-quota-bar/features/saved-usage-and-sources.md create mode 100644 .agents/skills/verify-quota-bar/features/settings-and-pricing.md create mode 100644 .agents/skills/verify-quota-bar/features/usage-report.md create mode 100755 .agents/skills/verify-quota-bar/scripts/control.py diff --git a/.agents/skills/verify-quota-bar/SKILL.md b/.agents/skills/verify-quota-bar/SKILL.md new file mode 100644 index 0000000..b03076d --- /dev/null +++ b/.agents/skills/verify-quota-bar/SKILL.md @@ -0,0 +1,195 @@ +--- +name: verify-quota-bar +description: Verify QuotaBar's native macOS menu bar behavior using its existing AppKit interaction harness and isolated sample previews. Use after changes to quota cards, local usage, settings, pricing, or usage reports; record which user entry points were actually exercised. +--- + +# Verify QuotaBar + +Read `docs/design-implementation.md` and [the feature map](features/README.md) first. +The primary surface is a native AppKit/SwiftUI menu bar app, with a WKWebView usage +report. `QuotaBarProbe` is a secondary integration diagnostic, not the UI driver. +There is no HTTP server or port to start. + +## Launch + +Run from the repository root on a logged-in macOS desktop. Full Xcode with Swift 6, +`uv`, and the Rust toolchain in `rust-toolchain.toml` are required. The repository's +wrapper selects `/Applications/Xcode.app/Contents/Developer`; set `DEVELOPER_DIR` +only when using a different full Xcode installation. Do not use bare `swift build`: +it bypasses the Rust integration. + +```sh +make build +uv run --no-project python .agents/skills/verify-quota-bar/scripts/control.py doctor +``` + +The reproducible smoke journey is the existing short-lived native interaction +verifier. It creates its own status item and popover, supplies offline display +fixtures, routes native events, asserts their effects, clears its preference suite, +and exits. It needs no login, migrations, seed database, or long-running server. +Launch and drive it together: + +```sh +VERIFY_EVIDENCE="$(mktemp -d /tmp/quotabar-verification.XXXXXX)" +uv run --no-project python .agents/skills/verify-quota-bar/scripts/control.py menu "$VERIFY_EVIDENCE" +``` + +The helper runs `.build/debug/QuotaBar --verify-menu-interaction`. A successful +build or `status_item_init_ms` alone is not readiness/proof. Require exit 0 and +`Menu lifecycle, status-button toggling, dismissal, and foreground preservation checks passed`, +with `passed: true` in `result.json`. The verifier checks that the status button +exists, the popover is shown, and its native keyboard window is usable before +continuing. It covers the **fallback** status-button implementation, not macOS 27's +remote expanded-interface pointer handling. + +For other mapped journeys, start the repository's sample preview in a foreground +terminal session, retaining the session handle: + +```sh +make preview PREVIEW_STATE=loaded +``` + +Other exact states are `signed-out`, `stale`, and `motion`. Preview seeds quota, +chart, and pricing display data itself. Its history/database start empty; displayed +usage is **not** inserted into the database. Readiness requires the visible provider +tabs, sample values, and `Settings…`/`Quit` controls, plus the log +`Interface preview: loaded. Use the popover's Quit action to close.` +The log can be buffered when stdout is redirected, so use a PTY for interactive +preview. Quit through this preview's `Quit` control for normal teardown. + +Do not use `make run` or `make kill`: they kill every process named QuotaBar, +including the user's installed app. Preview can coexist with the installed app, +but two previews share the fixed `QuotaBarInterfacePreview` preference suite. +Refuse concurrent preview/verifier driving, including across worktrees. The helper +serializes its own runs through `/tmp/quotabar-menu-verification-.lock` and +refuses an existing preview/verifier. Do not delete a lock owned by another run. + +## Doctor + +Run this read-only check before driving or whenever behavior looks wrong: + +```sh +uv run --no-project python .agents/skills/verify-quota-bar/scripts/control.py doctor +``` + +It reports the resolved debug executable, SHA-256, Git revision and working-tree +status, macOS version, and desktop user; it rejects missing binaries and concurrent +preview/verifier processes. It is a **preflight** check, not proof that an arbitrary +GUI process is healthy. Build immediately beforehand; a hash does not establish +that source changes were compiled. + +For an already launched interactive preview, inspect the exact PID and arguments +from `ps -axo pid,comm,args`, matching this checkout's resolved executable and +`--preview-interface` state. Check that PID with `ps -p` and `lsof -nP -p`; then +require its own current accessibility tree to show the expected sample controls. +Never select an instance using only the name `QuotaBar`. If ownership or visible +readiness cannot be established, stop that journey and record the failure. + +## Drive + +Start with `quota-fallback-lifecycle` in [quota and recovery](features/quota-and-recovery.md). +The `menu` command above drives the existing `MenuLifecycleVerifier` without +modifying production code. Its event routing exercises status-button presses and +releases, accessibility activation, Cmd-1/Cmd-3 provider selection, Escape, +foreground preservation, and dismissal/reopening. Initial fixture setters arrange +state; they are not the claimed user actions. Read the verifier source when a +failure needs diagnosis; do not bypass its assertions to get a passing exit. + +For full interactive journeys use the native computer-use harness, when it can +bind the owned preview. Initialize it with `await cua.getState()`, identify the +exact running app target, then use `cua.getApp` only with an unambiguous target +supported by the tool. Read the returned API documentation before acting. Resolve +each control's **current** index from `getAXState()`, call `click(index)`, then read +the new tree. The feature files name real labels, not persistent numeric IDs. +Keyboard equivalents use `pressKey('super+1')`, `pressKey('super+3')`, +`pressKey('super+comma')`, `pressKey('super+u')`, and `pressKey('Escape')` with the +appropriate window focused. Do not use coordinates without a fresh screenshot. + +This environment's CUA rejected the bare SwiftPM executable with `Invalid app`; +binding a temporary uniquely identified debug app bundle returned +`Computer Use server error -10005: timeoutReached`. Finder remained accessible. +Consequently the initial proof covers the existing fallback harness only. If the +same blocker recurs, record the interactive journey as blocked, clean up its +preview, and do not drive the installed app instead. Offscreen screenshots and +passing fallback checks cannot stand in for native macOS 27 pointer interaction. + +## Evidence + +The helper's evidence directory is outside `.build` and survives both its cleanup +and `make clean`: + +- `doctor.json`: build identity and scope. +- `launch.json`: exact command, PID, and process start time. +- `stdout.txt`, `stderr.txt`, `result.json`: assertions, exit status, and failure reason. +- `open-files.txt`, `sockets.txt`: read-only observations of the owned process. +- `preferences-after.txt`: the verifier's preference domain after exit; absent or + empty is clean (`cfprefsd` can retain an empty domain). +- `cleanup.json`: child exit, lock removal, and retained proof. + +Read these files after the command, including on failure. Socket observations are +samples, not a trace proving that no transient network request occurred. The +verifier's source keeps observers stopped and quota cooldowns pinned. It does not +exercise OAuth, account recovery, models.dev downloads, or real log ingestion. + +Interactive proof additionally records the feature ID, entry point, user actions, +before/after AX trees and screenshots, and observed result under that same evidence +directory. Capture motion as video when evaluating timing; static images cannot +prove it. Use `make card-snapshot SNAPSHOT_DIR="$VERIFY_EVIDENCE/cards"` for additional +layout evidence only. Never claim it records the interactive journey. + +Exercise the actual user path and capture the action plus result. Verify exported +files and imported record counts as well as visible success messages. Do not call +internal setters or test endpoints as the action under test. Keep mocks at the +existing preview's data-provider boundaries. Missing preconditions are failures, +not permission to substitute a simpler entry point; list every untested entry. + +Preview is not globally side-effect-free: + +- Quota/scan fixtures bypass live credentials and logs on the ordinary card path. +- `Open at login` uses real `SMAppService`; leave it alone in a default smoke run. +- SSH `Add` starts real SSH sync and can scan real local logs. It requires an + explicitly selected disposable host and controlled session data. +- The usage window loads HTML from memory with a fixed `usage-report.html` base URL + under the real application-support directory and uses default persistent WebKit + storage. Use a separate macOS test account for an isolated report journey, or + explicitly account for shared report preferences before calling it isolated. +- Copying sign-in commands changes the system clipboard; it does not execute a CLI. + +The initial proof is on real macOS, with the existing debug verifier's fallback +controller and offline fixture state. It is not a release-build, native expanded +session, account-integration, or complete accessibility audit. Those paths remain +unverified until their mapped journeys are run. + +## Cleanup + +The `menu` helper always waits for its child, terminates only that child after a +30-second timeout, clears the verifier's isolated defaults when no competing run +exists, and removes its own lock. It retains every proof file. Interruptions handled +by SIGINT/SIGTERM follow the same cleanup path; SIGKILL cannot run cleanup. For an +abruptly killed helper, use `launch.json` to verify the PID, start time, and full +command before stopping its orphan. The lock's `owner.json` records the helper PID, +start time, and evidence path; confirm that ownership before removing a stale lock. + +Interactive preview: choose `Quit` or Cmd-Q on the identified preview. Normal Quit +clears `QuotaBarInterfacePreview` and removes its UUID scratch directory. Before +launch, record existing `QuotaBarInterfacePreview-*` directories under +`$(getconf DARWIN_USER_TEMP_DIR)`; after launch record the new one owned by this run. +Do not assume setting `TMPDIR` moves Foundation's scratch directory: on the verified +machine it did not. If UI teardown is blocked, stop only the recorded PID after +checking its identity, then remove only that run's new scratch directory. Clear +`QuotaBarInterfacePreview` only after confirming no other preview is active. Never +kill by process name, delete all matching scratch directories, or delete evidence. + +After every run, including a failed iteration, confirm the owned process is gone +and the evidence files still exist. Record failed attempts separately; do not +replace them with a later passing result. + +## Helpers + +[scripts/control.py](scripts/control.py) is executable and uses only Python's +standard library. Invoke it through `uv run --no-project python`, as shown in +Launch and Doctor. `--help` lists its two commands. It deliberately wraps the +repository's native verifier rather than recreating its assertions. + +Keep the feature map current with `/maintain-verification-skill` when app paths or +verification preconditions change. diff --git a/.agents/skills/verify-quota-bar/features/README.md b/.agents/skills/verify-quota-bar/features/README.md new file mode 100644 index 0000000..dcecffc --- /dev/null +++ b/.agents/skills/verify-quota-bar/features/README.md @@ -0,0 +1,32 @@ +# QuotaBar verification map + +Read [the verification skill](../SKILL.md) before choosing a journey. This map is the maintained source of user paths, required fixtures, and observable outcomes. + +## Baseline preconditions + +- Build through the repository's Swift/Rust wrapper and run the skill's doctor. +- Use one owned preview at a time. `QuotaBarInterfacePreview` is a shared, fixed preference suite; a second preview clears the first one's preferences. +- Preview startup is `make preview PREVIEW_STATE=loaded`; `signed-out`, `stale`, and `motion` select other fixtures. Follow the skill's process ownership and evidence instructions. +- A logged-in macOS desktop and working native accessibility automation are required for interactive journeys. Never drive the installed application by mistake. + +## Driving conventions + +- Bind native CUA to the exact owned app path. Read its current accessibility snapshot, resolve the named control to a fresh element index, and click it with the documented CUA API. +- Use `pressKey` with `super+1`, `super+2`, or `super+3` only after confirming which window owns focus: these select providers in the popover and panes in Settings. +- Reacquire handles after navigation. Dynamic quota readings, dates, and vendor counts are not fixed selectors. +- Use the existing lifecycle verifier for its mapped fallback behavior. It does not replace physical pointer interaction with the macOS 27 remote menu bar. + +## Proof and failure reporting + +- Record feature ID, entry point, command/action, before state, resulting state, and relevant side effects in the skill's evidence directory. +- Retain screenshots and accessibility snapshots for interactive proof; retain stdout, stderr, and exit code for the executable verifier. Keep evidence after cleanup. +- Every listed entry point remains a separate coverage obligation. Missing seed data or automation access is a failed/blocked journey, not permission to substitute an easier path. +- The creation run passed the fallback lifecycle verifier. Full native CUA journeys remain unverified: binding the bare executable returned `Invalid app`; binding a temporary app bundle returned `timeoutReached`. + +## Features + +- [Quota and recovery](quota-and-recovery.md): status item, provider selection, reset display, refresh, and sign-in recovery. +- [Local usage](local-usage.md): dates, Tokens/Cost, model details, and Pi upstream usage. +- [Settings and pricing](settings-and-pricing.md): settings navigation, preferences, and rate search. +- [Usage report](usage-report.md): report window, periods, units, language, and breakdown. +- [Saved usage and sources](saved-usage-and-sources.md): backup import/export and SSH sources. diff --git a/.agents/skills/verify-quota-bar/features/local-usage.md b/.agents/skills/verify-quota-bar/features/local-usage.md new file mode 100644 index 0000000..eac1829 --- /dev/null +++ b/.agents/skills/verify-quota-bar/features/local-usage.md @@ -0,0 +1,39 @@ +# Local usage + +The card shows daily tokens or estimated cost, selectable dates, model details, and Pi usage grouped by upstream provider. + +## Sub-features + +- `usage-unit` switches Tokens/Cost while preserving a valid selected day. +- `usage-date` selects a day by pointer or arrow keys. +- `usage-models` expands and collapses the selected day's model list. +- `usage-pi` shows Pi upstream usage without quota or sign-in controls. +- `usage-retry` recovers a failed local scan through its own retry action. + +## How to get to it (user POV) + +- Open the menu bar card and select Claude, Codex, or Pi by button or Cmd-1/2/3. +- In local usage, choose Tokens or Cost, click a date column, and expand the model list below it. +- A failed local scan offers `Retry local scan`; Pi's Refresh rescans Pi sessions. + +## Driving it with native CUA + +Preconditions: + +- Start `make preview PREVIEW_STATE=loaded` through the skill's owned-instance workflow and bind the native UI successfully. +- Resolve the actual dates and models from the current accessibility snapshot; sample dates are not fixed handles. + +- **Units.** Resolve `Usage unit`, then click `Tokens` and `Cost`. Capture the selected unit and changed labels/totals after each action. +- **Dates.** Click a day whose accessible label contains its date and year. Its hint is `Select day. Use Left and Right Arrow to change day.` Press Left/Right Arrow and verify the selected date and detail values move together. +- **Selection persistence.** Select a day, change units, and refresh when enabled. A still-valid date must remain selected; capture each action and state. +- **Models.** Resolve the disclosure with `Expand model list` hint and `Collapsed` value. Activate it, verify `Expanded` plus model rows, then collapse it again. Record visible full model labels and values. +- **Pi.** Click `Pi` and separately use `pressKey` with `super+3`. Verify upstream provider rows, local usage, and absence of quota/sign-in controls; the status gauge must retain the last quota provider. +- **Recovery.** With a genuine failed local scan fixture, choose `Retry local scan` and verify progress followed by usable data. Report the missing precondition if no failed scan is available. + +## Gotchas + +- Preview cost results are display fixtures, not records seeded into its SQLite database. They cannot prove log ingestion or backup contents. +- Existing preview states do not seed a local-scan failure. `usage-retry` is blocked without that precondition; ordinary Refresh is not equivalent coverage. +- `motion` delays the first local scan and later refreshes; use it only for a timing journey and capture video. +- Dynamic dates, model counts, and amounts should be read from the current snapshot rather than hard-coded. +- These interactive journeys remain unverified until native CUA can bind the owned app. diff --git a/.agents/skills/verify-quota-bar/features/quota-and-recovery.md b/.agents/skills/verify-quota-bar/features/quota-and-recovery.md new file mode 100644 index 0000000..33949fd --- /dev/null +++ b/.agents/skills/verify-quota-bar/features/quota-and-recovery.md @@ -0,0 +1,40 @@ +# Quota and recovery + +QuotaBar shows quota, reset times, freshness, and recovery actions from its menu bar card while preserving the last successful reading after a refresh failure. + +## Sub-features + +- `quota-fallback-lifecycle` checks native fallback status-button toggling, dismissal, foreground preservation, and provider key equivalents with the existing verifier. +- `quota-open` opens and closes the card through the actual status item. +- `quota-provider` selects Claude, Codex, or Pi by button and keyboard. +- `quota-reset` switches both reset labels between countdown and date. +- `quota-refresh` refreshes through the card or application command and observes cooldown/failure. +- `quota-sign-in` copies the CLI command and checks sign-in. + +## How to get to it (user POV) + +- Click the owned `QuotaBar, Codex` or `QuotaBar, Claude` status item; click again, outside the card, or press Escape to dismiss. +- Choose `Claude`, `Codex`, or `Pi`, or press Cmd-1, Cmd-2, or Cmd-3 in the card. +- Open either reset-time menu. Choose `Refresh` or Cmd-R; signed-out cards offer `Copy` and `Check sign-in`. + +## Driving it with native CUA and the menu interaction verifier + +Preconditions: + +- Use the skill's owned build and desktop session. Select the matching preview state; do not overlap preview instances. +- Interactive paths require a successful native CUA binding and a fresh accessibility snapshot. + +- **Fallback proof.** Run `.build/debug/QuotaBar --verify-menu-interaction`. Require exit 0 and `Menu lifecycle, status-button toggling, dismissal, and foreground preservation checks passed`. Save the complete transcript. This executable covers `quota-fallback-lifecycle`, including Cmd-1/Cmd-3 dispatch and Pi preserving the quota gauge. +- **Actual opening.** Start `make preview PREVIEW_STATE=loaded`. Resolve the owned status item's current label and click it. Capture open, repeated-click dismissal, and Escape dismissal as separate actions. +- **Provider selection.** Click each named provider button; also use `pressKey` with `super+1`, `super+2`, and `super+3`. Confirm the selected card; Pi must keep the last quota provider's status gauge. +- **Reset display.** Resolve a label beginning `Reset time display, resets `; choose `Show reset date`, then `Show countdown`. Both quota windows must adopt the selected mode. +- **Refresh.** Use `make preview PREVIEW_STATE=motion`, wait for an enabled `Refresh`, then exercise the button and Cmd-R separately. Capture working state and the completed reading. In `stale`, capture retained readings, warning, and retry countdown before retry becomes available. +- **Sign-in.** In `signed-out`, click `Copy claude command` and verify the clipboard contains `claude`. In `motion`, select Claude and choose `Check sign-in`; capture checking and the loaded result. + +## Gotchas + +- The fallback verifier synthesizes application events; passing it does not prove the macOS 27 remote menu bar's physical pointer path or a full accessibility audit. +- `signed-out` intentionally stays signed out after checking. Use `motion` for the fixture's successful sign-in path; neither verifies real OAuth credentials. +- Copy changes the system clipboard. Preserve and restore its prior contents when exercising that path; do not execute the copied login command. +- `stale` starts with a 90-second retry deadline; `motion` accelerates cooldown and introduces delayed fetches. Capture video for timing claims. +- Native CUA journeys were blocked during skill creation; only `quota-fallback-lifecycle` was executed successfully. diff --git a/.agents/skills/verify-quota-bar/features/saved-usage-and-sources.md b/.agents/skills/verify-quota-bar/features/saved-usage-and-sources.md new file mode 100644 index 0000000..decb9b1 --- /dev/null +++ b/.agents/skills/verify-quota-bar/features/saved-usage-and-sources.md @@ -0,0 +1,41 @@ +# Saved usage and sources + +The Usage settings pane summarizes recorded usage, exports and merges backups, and adds SSH machines whose session logs should be recorded. + +## Sub-features + +- `backup-export` writes a readable SQLite backup through the save panel. +- `backup-import` merges a backup and reports added records; repeat import does not duplicate them. +- `source-add` adds an SSH alias or typed host and starts synchronization. +- `source-sync` syncs all hosts, one host, or retries a failed host. +- `source-remove` stops syncing a host while retaining recorded usage. + +## How to get to it (user POV) + +- Open `Settings…` or Cmd-comma, then `Usage` or Cmd-3 in Settings. +- Under Saved usage, choose `Import…` or `Export…`. +- Under Sources, choose `Add host`, `Sync now`, or a host's `Actions for ` menu. A failed host offers `Retry`. + +## Driving it with native CUA + +Preconditions: + +- Bind the owned preview through the skill's native workflow and open Settings → Usage. +- Backup success requires a known synthetic `.sqlite` backup with expected record counts. The default preview does not seed one. +- SSH success requires a deliberately isolated, reachable host with synthetic session logs and noninteractive authentication. Do not substitute a personal host. + +- **Import.** Choose `Import…`; confirm the file panel is `Import Usage Backup`, select the synthetic backup, and choose `Import`. Capture the result and changed saved-record count. Repeat import and verify no duplicate records are added. +- **Export.** With saved records present, choose `Export…` and save inside the evidence directory. Its suggested name is `QuotaBar-Backup-YYYY-MM-DD.sqlite`. Verify the file exists, is readable SQLite, passes a read-only integrity check, and contains the expected records; retain the file after cleanup. +- **Add/cancel.** Choose `Add host`, resolve field `SSH host or alias` (placeholder `Search hosts or type devbox`), type the isolated host, then choose `Cancel`; no host row should be added. +- **Add/sync.** Repeat and choose `Add`. Capture syncing followed by the host's recorded/new counts and copied bytes. Independently confirm the expected synthetic records reached the scratch database and mirrored logs stay under the owned scratch directory. +- **Other sync entries.** Exercise `Sync now`, `Actions for ` → `Sync this host`, and `Retry` on a deliberately failed host separately. Each must complete or display an actionable failure. +- **Remove.** Choose `Actions for ` → `Stop syncing`; its source row disappears while saved records remain. + +## Gotchas + +- Preview card/report samples are not saved records. Initial scratch usage is empty and `Export…` becomes disabled when its record count is zero. Missing seed data blocks a successful export journey; do not claim success from opening a dialog. +- Import really mutates the preview database and export really writes the chosen destination. Use synthetic data and retain exported proof outside scratch cleanup. +- `Add host` immediately runs real SSH, reads local SSH configuration, and can copy complete remote logs. Preview does not mock this path. +- Remote synchronization also records through the real scanner using the process environment, so it can scan actual local Claude/Codex logs into the scratch database. A fully isolated host alone does not isolate this journey; local scan roots must also be isolated before running it. +- Default preview has no safe SSH fixture or seeded backup. Report these journeys blocked until those preconditions are supplied; never weaken them to mere button visibility. +- These interactive journeys remain unverified until native CUA can bind the owned app. diff --git a/.agents/skills/verify-quota-bar/features/settings-and-pricing.md b/.agents/skills/verify-quota-bar/features/settings-and-pricing.md new file mode 100644 index 0000000..285065e --- /dev/null +++ b/.agents/skills/verify-quota-bar/features/settings-and-pricing.md @@ -0,0 +1,41 @@ +# Settings and pricing + +Settings controls the menu bar provider and refresh cadence, and provides a searchable, read-only list of model rates. + +## Sub-features + +- `settings-open` opens and revisits Settings by button and shortcut. +- `settings-general` changes the displayed provider and refresh cadence. +- `pricing-search` filters rates by model and clears the filter. +- `pricing-vendors` expands and collapses vendor groups and model details. +- `pricing-source` changes the models.dev setting. +- `settings-login` changes the system login item when that integration is explicitly under test. + +## How to get to it (user POV) + +- Choose `Settings…` in the card or press Cmd-comma. +- Choose `General`, `Pricing`, or `Usage` in the sidebar, or Cmd-1/2/3 while Settings owns focus. +- General contains provider selection, `Refresh quota`, and `Open at login`. +- Pricing contains `Search models`, vendor groups, and `Price other models with models.dev`. + +## Driving it with native CUA + +Preconditions: + +- Start `make preview PREVIEW_STATE=loaded` through the skill's workflow; bind the owned native app and inspect its current accessibility tree. +- Confirm window title `QuotaBar Settings` before sending pane shortcuts. + +- **Open.** Exercise `Settings…` and Cmd-comma separately. Close and reopen the window; its position should be preserved within the available screen. +- **General.** Resolve provider rows by labels beginning `Claude, ` or `Codex, ` and choose one. Confirm the status item now names that provider. Select `Refresh quota` → `Manual`; reopen the pane to confirm the choice persists in this preview session. +- **Pricing pane.** Click `Pricing`; separately use `pressKey` with `super+2` while Settings owns focus. Confirm the rates list, not the popover's Codex tab. +- **Search.** Resolve the native search field with placeholder `Search models`. Read an existing model name from the list, enter that name, and verify matching model rows and rate columns. Clear the field and verify the unfiltered list returns. +- **Disclosure.** Resolve a vendor label ending `model` or `models`. Activate it and verify `Collapsed` changes to `Expanded` with its rows visible; activate again to collapse. Open a model with displayed tier details and inspect those details. +- **Pricing switch.** Toggle `Price other models with models.dev`, revisit Pricing, and verify the chosen setting remains. Restore the initial choice. This proves the preference interaction only in preview. + +## Gotchas + +- `Open at login` calls the real macOS login-item service even in preview. It is outside the default isolated journey; a scratch preview is not a safe stand-in for a packaged-app login-item test. +- Preview rates are fixtures and models.dev downloads are disabled at the service boundary. A switch interaction does not prove network fetching or repricing real logs. +- Search opens matching vendor groups and disables their disclosure control while filtering. Clear the search before testing manual collapse. +- Cmd-1/2/3 target different controls in Settings and in the card. Verify the active window before sending them. +- These interactive journeys remain unverified until native CUA can bind the owned app. diff --git a/.agents/skills/verify-quota-bar/features/usage-report.md b/.agents/skills/verify-quota-bar/features/usage-report.md new file mode 100644 index 0000000..ac20dcc --- /dev/null +++ b/.agents/skills/verify-quota-bar/features/usage-report.md @@ -0,0 +1,40 @@ +# Usage report + +The Usage window presents saved usage by period, metric, model or day, with English and Chinese labels. + +## Sub-features + +- `report-open` opens the Usage window through the card and shortcut. +- `report-period` changes 7-day, 30-day, and custom date ranges. +- `report-metric` switches tokens and cost. +- `report-language` changes English/Chinese labels. +- `report-breakdown` switches model/day detail. +- `report-live` preserves view choices as recorded usage changes. + +## How to get to it (user POV) + +- Choose `Usage…` from the card or press Cmd-U. +- Use the window's period, Tokens/Cost, EN/中, and Model/Day controls. +- Select Custom to reveal the start and end date fields. + +## Driving it with native CUA + +Preconditions: + +- Start `make preview PREVIEW_STATE=loaded` through the skill's workflow and obtain a working native CUA binding. +- Record initial report choices before changing them: its WebKit storage is persistent, unlike preview preferences. +- Preview report data covers 2026-09-01 through 2026-09-30; custom-range tests must use that range. + +- **Open.** Exercise `Usage…` and Cmd-U separately. Wait for populated summary, chart, and breakdown rather than only a visible window. +- **Language.** Resolve `EN` and `中` from the current tree. Activate each and confirm labels change while the selected period and metric remain intact. +- **Period.** In English, click `7 days`, then `30 days`; capture the range labels and corresponding totals. Choose `Custom`, set 2026-09-01 through 2026-09-07 using the visible date inputs, and confirm the requested period is displayed. +- **Metric.** Click `Tokens`, then `Cost`; verify summary, chart labels, and breakdown agree with the selected metric. +- **Breakdown.** Click `Model`, then `Day`; confirm rows represent the chosen grouping and totals remain consistent with the same range. +- **Restore.** Restore the initial report choices through the UI and close the owned window. Capture the restored state. + +## Gotchas + +- The preview report is a fixed sample snapshot, independent of card samples and its empty scratch database. It cannot prove live ingestion or import-driven report updates; `report-live` needs an actual changing, isolated database. +- The native report uses the default persistent WebKit data store and a fixed application-support page identity. Preview termination does not clear report choices. Never delete the user's WebKit store as cleanup. +- If separately exercising the existing HTML harness, real page handles are `[data-language="en"]`, `[data-language="zh"]`, `[data-period="7"]`, `[data-period="30"]`, `[data-period="custom"]`, `[data-range-from]`, `[data-range-to]`, `[data-metric="tokens"]`, `[data-metric="cost"]`, `[data-breakdown="model"]`, and `[data-breakdown="day"]`. HTML-only coverage does not prove native window entry points. +- These interactive journeys remain unverified until native CUA can bind the owned app. diff --git a/.agents/skills/verify-quota-bar/scripts/control.py b/.agents/skills/verify-quota-bar/scripts/control.py new file mode 100755 index 0000000..ddf6ec3 --- /dev/null +++ b/.agents/skills/verify-quota-bar/scripts/control.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Check a QuotaBar build and capture its existing native menu verifier's proof.""" + +import argparse +import hashlib +import json +import os +from pathlib import Path +import platform +import plistlib +import signal +import subprocess +import time +from typing import Any + +ROOT = Path(__file__).resolve().parents[4] +LOCK = Path(f"/tmp/quotabar-menu-verification-{os.getuid()}.lock") +SUITE = "QuotaBarMenuLifecycleVerifier" +PASS = "Menu lifecycle, status-button toggling, dismissal, and foreground preservation checks passed" + + +def query(*args: str) -> subprocess.CompletedProcess[str]: + return subprocess.run(args, capture_output=True, text=True, check=False) + + +def save(path: Path, value: dict[str, Any]) -> None: + path.write_text(json.dumps(value, indent=2) + "\n") + + +def competitors() -> list[str]: + rows = query("ps", "-axo", "pid=,comm=,args=").stdout.splitlines() + # Only app processes: shell command text can also contain these flags. + return [row.strip() for row in rows if any( + marker in row for marker in ("/QuotaBar --verify-menu-interaction", "/QuotaBar --preview-interface") + ) and " /bin/" not in row and "control.py" not in row] + + +def doctor() -> dict[str, Any]: + if platform.system() != "Darwin": + raise RuntimeError("Native verification requires a logged-in macOS desktop.") + binary = (ROOT / ".build/debug/QuotaBar").resolve(strict=True) + if not os.access(binary, os.X_OK): + raise RuntimeError("Build with make build first.") + if competitors(): + raise RuntimeError("Another preview/verifier is running; do not share desktop input or suites.") + console = query("stat", "-f", "%Su", "/dev/console").stdout.strip() + if console in ("", "root", "loginwindow"): + raise RuntimeError("No logged-in desktop user; native menu verification is unavailable.") + return { + "binary": str(binary), "sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), + "revision": query("git", "-C", str(ROOT), "rev-parse", "HEAD").stdout.strip(), + "working_tree": query("git", "-C", str(ROOT), "status", "--short").stdout.strip(), + "macos": platform.mac_ver()[0], "console_user": console, + "scope": "Fallback AppKit menu verifier; not native expanded-session pointer proof", + } + + +def interrupted(signum: int, frame: Any) -> None: + raise KeyboardInterrupt(f"Received signal {signum}") + + +def run_menu(evidence: Path) -> int: + evidence = evidence.resolve() + evidence.mkdir(parents=True, exist_ok=True) + if any(evidence.iterdir()): + raise RuntimeError("Use an empty evidence directory so earlier proof cannot be overwritten.") + process: subprocess.Popen[str] | None = None + lock_owned = False + result: dict[str, Any] = {"passed": False, "feature": "quota-fallback-lifecycle", "entry": "--verify-menu-interaction"} + try: + # Defer handled signals until ownership is recorded, including the mkdir boundary. + previous_mask = signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT, signal.SIGTERM}) + try: + LOCK.mkdir(mode=0o700) + lock_owned = True + save(LOCK / "owner.json", { + "evidence": str(evidence), "helper_pid": os.getpid(), + "started": query("ps", "-p", str(os.getpid()), "-o", "lstart=").stdout.strip(), + }) + finally: + signal.pthread_sigmask(signal.SIG_SETMASK, previous_mask) + build = doctor() + save(evidence / "doctor.json", build) + command = [build["binary"], "--verify-menu-interaction"] + with (evidence / "stdout.txt").open("w") as stdout, (evidence / "stderr.txt").open("w") as stderr: + process = subprocess.Popen(command, stdout=stdout, stderr=stderr, text=True) + save(evidence / "launch.json", { + "command": command, "pid": process.pid, + "started": query("ps", "-p", str(process.pid), "-o", "lstart=").stdout.strip(), + }) + # This is an observation sample, not proof that no transient socket ever existed. + (evidence / "open-files.txt").write_text(query("lsof", "-nP", "-p", str(process.pid)).stdout) + (evidence / "sockets.txt").write_text(query("lsof", "-nP", "-a", "-p", str(process.pid), "-i").stdout) + result["exit_code"] = process.wait(timeout=30) + result["assertion_summary_present"] = PASS in (evidence / "stdout.txt").read_text() + preferences = query("defaults", "export", SUITE, "-") + (evidence / "preferences-after.txt").write_text(preferences.stdout + preferences.stderr) + # removePersistentDomain may leave an empty domain registered with cfprefsd. + result["suite_empty_after_app"] = preferences.returncode != 0 or plistlib.loads( + preferences.stdout.encode() + ) == {} + result["binary_unchanged"] = doctor()["sha256"] == build["sha256"] + result["passed"] = result["exit_code"] == 0 and all(result[key] for key in ( + "assertion_summary_present", "suite_empty_after_app", "binary_unchanged" + )) + except (Exception, KeyboardInterrupt) as error: + result["error"] = f"{type(error).__name__}: {error}" + finally: + if process is not None and process.poll() is None: + process.terminate() + try: + process.wait(timeout=3) + except subprocess.TimeoutExpired: + process.kill() + process.wait(timeout=3) + # Only this run's child and ephemeral verifier suite are cleaned up. + if process is not None and not competitors(): + query("defaults", "delete", SUITE) + result["child_exited"] = process is None or process.poll() is not None + result["finished_at"] = time.strftime("%Y-%m-%dT%H:%M:%S%z") + save(evidence / "result.json", result) + if lock_owned: + (LOCK / "owner.json").unlink(missing_ok=True) + LOCK.rmdir() + save(evidence / "cleanup.json", { + "child_exited": result["child_exited"], "lock_removed": not LOCK.exists(), + "evidence_retained": (evidence / "result.json").is_file(), + }) + print(json.dumps(result, indent=2)) + print(f"Evidence retained at {evidence}") + return 0 if result["passed"] else 1 + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["doctor", "menu"]) + parser.add_argument("evidence", nargs="?", type=Path) + args = parser.parse_args() + if args.command == "doctor": + print(json.dumps(doctor(), indent=2)) + return 0 + if args.evidence is None: + parser.error("menu requires an evidence directory") + signal.signal(signal.SIGTERM, interrupted) + signal.signal(signal.SIGINT, interrupted) + return run_menu(args.evidence) + + +if __name__ == "__main__": + raise SystemExit(main())