From abe540b60c2204c2c98cb3666b368450ca6007d2 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 25 Jul 2026 04:26:13 -0500 Subject: [PATCH 1/3] feat: add provider extensibility and proactive starting-account selection Provider extensibility: user config now appends to catalog retry and passthrough lists by default, with *_replace siblings to override them outright. Adds a validation module behind `config validate` and `doctor`, an `examplecli` reference provider, and an authoring guide. Proactive starting-account selection: when quota data is available, clifwrap picks the account with the most headroom before the first upstream call instead of always starting on a possibly depleted default. Enabled by default for providers with `status_command` or `usage`; opt out per provider with `proactive_pick = false`. Also hardens the new surfaces following a pre-release audit: - doctor probes are opt-in via `--probe` so casual and CI runs no longer resolve credentials or make authenticated calls as a side effect - probe findings report exit status only, never subprocess output or interpolated argv, keeping tokens out of `doctor --json` - *_replace keys now apply to providers with no catalog entry, where they previously resolved to an empty list --- README.md | 3 + .../2026-07-25-agent-native-requirements.md | 159 +++++++ .../2026-07-25-extensibility-requirements.md | 142 +++++++ .../2026-07-25-reliability-requirements.md | 157 +++++++ docs/cli-reference.md | 13 +- docs/configuration.md | 48 ++- docs/operations.md | 14 +- ...25-001-feat-provider-extensibility-plan.md | 292 +++++++++++++ ...02-feat-proactive-starting-account-plan.md | 239 +++++++++++ docs/provider-authoring.md | 59 +++ docs/provider-catalog.md | 50 +++ src/clifwrap/__main__.py | 101 ++++- src/clifwrap/config.py | 87 +++- src/clifwrap/providers.toml | 41 ++ src/clifwrap/runtime.py | 199 +++++++-- src/clifwrap/scheduling.py | 132 ++++-- src/clifwrap/validation.py | 388 ++++++++++++++++++ tests/test_wrapper.py | 346 +++++++++++++++- 18 files changed, 2365 insertions(+), 105 deletions(-) create mode 100644 docs/brainstorms/2026-07-25-agent-native-requirements.md create mode 100644 docs/brainstorms/2026-07-25-extensibility-requirements.md create mode 100644 docs/brainstorms/2026-07-25-reliability-requirements.md create mode 100644 docs/plans/2026-07-25-001-feat-provider-extensibility-plan.md create mode 100644 docs/plans/2026-07-25-002-feat-proactive-starting-account-plan.md create mode 100644 docs/provider-authoring.md create mode 100644 src/clifwrap/validation.py diff --git a/README.md b/README.md index a0b1f63..b8293bf 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,8 @@ Command costs are estimates, not billing truth. SearchCLI and ScrapeCLI defaults When capacity control picks an account, retries for that request stay on that account. A retryable upstream error will not burn through your reserve accounts. +With `proactive_pick` enabled (default when usage or `status_command` is configured), clifwrap also picks the best starting account before the first upstream call so a depleted default does not burn a retry. See [docs/configuration.md](docs/configuration.md#proactive-starting-account). + ### Account management from the wrapped CLI Upstream auth still works as usual: @@ -313,6 +315,7 @@ scrapecli search "example" ## Docs - [Configuration](docs/configuration.md) +- [Provider authoring](docs/provider-authoring.md) - [CLI reference](docs/cli-reference.md) - [Built-in provider catalog](docs/provider-catalog.md) - [Migration from cli-fallback-wrapper](docs/migration.md) diff --git a/docs/brainstorms/2026-07-25-agent-native-requirements.md b/docs/brainstorms/2026-07-25-agent-native-requirements.md new file mode 100644 index 0000000..3628906 --- /dev/null +++ b/docs/brainstorms/2026-07-25-agent-native-requirements.md @@ -0,0 +1,159 @@ +--- +date: 2026-07-25 +topic: agent-native +--- + +## Summary + +Give clifwrap a shared agent-native contract so coding agents and CI can drive it without guessing flags or parsing ad-hoc JSON. Unify existing `--json` output under one envelope, document stable exit codes, and add `--dry-run` on mutating clifwrap commands. + +--- + +## Problem Frame + +clifwrap already exposes `--json` on several operational commands (`doctor`, `status`, `config validate`, `account list`, queue commands), but each command returns a different top-level shape. Exit codes are meaningful in practice (`--check` fails on issues) yet undocumented as a taxonomy agents can rely on. Mutating commands (`install`, `uninstall`, `account add`, queue replay) have no machine-readable output and no dry-run path, so agents and CI must either scrape human text or execute side effects to learn what would happen. + +The extensibility track improves provider validation and authoring; it deliberately deferred a full agent-native surface. This track closes that gap for the commands operators and automation already use daily. + +--- + +## Key Decisions + +- **Contract plus dry-run, not full spec compliance.** Ship a unified JSON envelope, documented exit codes, and dry-run on mutating clifwrap commands. Defer machine-readable schema discovery and ai-native-cli-spec coverage across every command. +- **Same contract for agents and CI.** One envelope and exit-code table serves interactive agent sessions and pipeline scripts; no separate "CI mode." +- **Wrap, don't redesign.** Existing per-command payloads move under a `data` field inside the envelope rather than redesigning each command's internal structure. +- **Additive migration for envelope.** When `--json` is used, nest the current payload under `data` and add envelope fields alongside. Human (non-JSON) output stays unchanged. Breaking top-level JSON keys ship only with a documented migration note in the release changelog. +- **Dry-run applies to clifwrap-owned mutations only.** Install, uninstall, account mutations, and queue mutations support dry-run. Wrapped upstream CLIs are out of scope. + +--- + +## Requirements + +**JSON envelope** + +- R1. Every command that supports `--json` emits a top-level envelope with at least: success indicator, command identity, and a `data` field holding the command's payload. +- R2. On failure, the envelope includes a structured error object with a stable machine-readable code and a human-readable message. Secret values never appear in JSON output. +- R3. Envelope fields are consistent across commands so a single parser can handle doctor, status, validate, account list, and queue output. + +**Exit codes** + +- R4. clifwrap documents a small stable exit-code taxonomy (success, usage/argument error, config error, operational check failure, internal error) mapped to real command behavior. +- R5. `--check` on doctor and status continues to exit nonzero on check failures; the taxonomy explains which code applies. +- R6. JSON error responses and process exit codes align: an agent reading only stdout can determine outcome; an agent reading only exit code can branch without parsing JSON. + +**Dry-run** + +- R7. `install`, `uninstall`, `account add`, `account remove`, `queue run`, and `queue drop` accept `--dry-run` and perform no persistent side effects when it is set. +- R8. Dry-run combined with `--json` returns the envelope with a `data` field describing planned changes (targets, accounts, queue items) without applying them. +- R9. Dry-run exits 0 when the operation would succeed; exits with the appropriate taxonomy code when it would fail (missing backup, invalid config, queue item not found). + +**Documentation and discoverability** + +- R10. `docs/operations.md` and generated CLI reference document the envelope shape, exit-code table, and which commands support `--json` and `--dry-run`. +- R11. `clifwrap doctor --json` includes envelope version or contract version in metadata so agents can detect contract generation. + +--- + +## Key Flows + +- F1. CI preflight before deploy + - **Trigger:** Pipeline runs health checks before releasing or rotating accounts. + - **Actors:** CI script, clifwrap CLI + - **Steps:** Run `config validate --json` → parse envelope → run `doctor --json --check` → branch on exit code and `data.issues`. + - **Outcome:** Pipeline fails fast with a parseable reason, no custom grep rules per subcommand. + - **Covered by:** R1, R4, R5, R6 + +- F2. Agent plans shim install without side effects + - **Trigger:** Coding agent needs to verify install targets before modifying PATH shims. + - **Actors:** Agent, clifwrap CLI + - **Steps:** Run `install --dry-run --json` → read planned shim and backup paths from `data` → confirm with user → run without dry-run. + - **Outcome:** Agent never replaces a binary based on guessed flags. + - **Covered by:** R7, R8, R9 + +- F3. Agent replays queue safely + - **Trigger:** Capacity recovered; automation needs to drain queued work. + - **Actors:** Agent or cron job + - **Steps:** `queue list --json` → `queue run --dry-run --json` to preview → `queue run --json` to execute. + - **Outcome:** No duplicate replay; agent sees item count and outcomes in envelope. + - **Covered by:** R1, R7, R8 + +--- + +## Acceptance Examples + +- AE1. Envelope on success + - **Covers:** R1, R3 + - **Given:** Valid config and `clifwrap config paths --json`. + - **When:** Command completes successfully. + - **Then:** stdout is JSON with success indicator true, command identity set, and prior path fields nested under `data`. + +- AE2. Envelope on validation failure + - **Covers:** R2, R6 + - **Given:** Config references a missing env ref. + - **When:** `clifwrap config validate --json` runs. + - **Then:** Envelope reports failure, `data` includes findings, exit code matches the config-error category in the documented taxonomy. + +- AE3. Dry-run install + - **Covers:** R7, R8, R9 + - **Given:** `searchcli` is on PATH and not yet shimmed. + - **When:** `clifwrap install searchcli --dry-run --json` runs. + - **Then:** Exit 0; `data` describes shim target and backup path; no shim file is written. + +- AE4. Check mode exit code + - **Covers:** R5, R6 + - **Given:** Doctor reports validation warnings and `--check` is set. + - **When:** `clifwrap doctor --json --check` runs. + - **Then:** Exit code is the documented operational-check-failure code; envelope success indicator is false or `data` reflects check failure per contract rules. + +--- + +## Success Criteria + +- A CI script can drive `validate` and `doctor --check` using only envelope parsing and documented exit codes, without per-command JSON shape knowledge. +- An agent can plan `install` and `queue run` via `--dry-run --json` without causing side effects. +- Operations docs list every `--json` and `--dry-run` command in one place. + +--- + +## Scope Boundaries + +**Deferred for later** + +- Machine-readable command/schema discovery command (`reference`, `schema`, or equivalent) +- Full ai-native-cli-spec compliance on every subcommand +- `--json` on wrapped upstream CLI passthrough invocations +- Reliability track (proactive quota pick, circuit breakers, failover event log) +- Security track (keychain refs, credential tiering) +- Context-routing track (project profiles, per-account config dirs) + +**Outside this product's identity** + +- HTTP API or MCP server wrapping clifwrap +- Agent-specific natural-language CLI (clifwrap remains a standard POSIX-style CLI) + +--- + +## Dependencies / Assumptions + +- Extensibility track (`config validate`, `doctor` validation section) lands first or in parallel; agent-native work extends those JSON outputs rather than replacing validation behavior. +- Existing `--json` consumers are few (early v0.1); additive nesting under `data` is acceptable with a changelog note. +- Human-readable command output remains the default; JSON is opt-in per command via `--json`. +- Generated `docs/cli-reference.md` continues to be the published command inventory. + +--- + +## Outstanding Questions + +**Deferred to planning** + +- Exact envelope field names and whether `ok`/`success` follows agent-native-design or an internal convention. +- Whether `account import` dry-run reuses the existing apply-without-apply pattern or gains an explicit `--dry-run` flag aligned with other mutators. +- Contract version bump policy: semver in envelope meta vs clifwrap package version. + +--- + +## Sources / Research + +- Prior brainstorm: `docs/brainstorms/2026-07-25-extensibility-requirements.md` (deferred agent-native track) +- Repo: `src/clifwrap/__main__.py` (existing `--json` flags), `docs/operations.md`, `docs/cli-reference.md` +- External: [agent-native-design](https://agents365-ai.github.io/agent-native-design/), [CLI Spec](https://clispec.dev/), codex-quota and gh CLI `--json` patterns from improvement research diff --git a/docs/brainstorms/2026-07-25-extensibility-requirements.md b/docs/brainstorms/2026-07-25-extensibility-requirements.md new file mode 100644 index 0000000..d6f081c --- /dev/null +++ b/docs/brainstorms/2026-07-25-extensibility-requirements.md @@ -0,0 +1,142 @@ +--- +date: 2026-07-25 +topic: extensibility +--- + +## Summary + +Make adding and customizing CLI providers in clifwrap safe, predictable, and self-documenting. Ship tiered provider validation, fix catalog override semantics for list fields, and publish a provider authoring guide with a minimal template catalog entry. + +--- + +## Problem Frame + +clifwrap already supports declarative providers through `src/clifwrap/providers.toml` and user `config.toml` overrides, but extensibility breaks down in practice. Operators discover misconfiguration only when a wrapped command fails at runtime. List fields such as retry patterns use all-or-nothing merge semantics, so a partial override silently replaces the entire catalog list. Only `searchcli` and `scrapecli` exist as built-in examples, and there is no dedicated guide for authoring a new provider. + +The user prioritized extensibility over reliability, security, agent automation, and public-adoption polish for this track. Research against tools like GitHub CLI, teamclaude, AWS credential chains, and the CLI Spec surfaced validation and merge semantics as the highest-leverage gaps without introducing a plugin framework yet. + +--- + +## Key Decisions + +- **Validation before runtime, not a plugin system.** Extend existing `config validate` and `doctor` surfaces rather than building pip-based provider discovery or Python hooks in v1 of this track. +- **Warn by default; fail on explicit check.** Offline and air-gapped setups should get warnings for unreachable usage probes. `--check` (and CI-style usage) exits nonzero on validation failures, matching existing `doctor --check` behavior. +- **Template provider, not a third production provider.** Add a minimal catalog entry whose purpose is documentation-by-example, not shipping another real upstream CLI. +- **Append-aware list merge for catalog overrides.** User config must be able to extend catalog list fields without replacing them wholesale unless the user explicitly opts into replace semantics. + +--- + +## Requirements + +**Provider validation** + +- R1. `clifwrap config validate` reports provider-level issues beyond TOML parsing: missing accounts, unresolvable env refs, invalid merge declarations, and malformed provider metadata. +- R2. `clifwrap doctor` includes a provider validation section that summarizes per-provider health and links each issue to a remediation command. +- R3. A provider probe mode validates live behavior without mutating config: usage URL or `status_command` returns expected JSON fields (`remaining`, `limit`, optionally `used`), and retry rule entries are syntactically valid. +- R4. Validation messages name the provider, the config location, and the next command to fix the issue. +- R5. Without `--check`, unreachable usage endpoints produce warnings. With `--check`, they produce errors and a nonzero exit code. + +**Config merge semantics** + +- R6. List fields inherited from the catalog (retry patterns, passthrough commands, and similar) support append-by-default when the user supplies entries in `config.toml`. +- R7. Users can opt into full replacement for a list field when they intend to drop catalog defaults entirely. +- R8. Merge rules for list fields are documented in `docs/configuration.md` with before/after examples. + +**Authoring and discovery** + +- R9. A provider authoring guide explains how to add a catalog provider vs a config-only generic provider, required fields, and common failure modes. +- R10. A minimal template catalog provider demonstrates every optional field with safe defaults and example placeholder URLs. +- R11. Generated `docs/provider-catalog.md` includes the template provider so operators can discover it from published docs. + +--- + +## Key Flows + +- F1. Author validates a new provider before first use + - **Trigger:** Operator adds accounts and overrides for a provider in `config.toml`. + - **Actors:** Operator, clifwrap CLI + - **Steps:** Run `config validate` → fix static issues → run `doctor --check` → run provider probe → install shim and test wrapped command. + - **Outcome:** First real upstream invocation succeeds or fails for upstream reasons, not local misconfiguration. + - **Covered by:** R1, R2, R3, R4, R5 + +- F2. Operator extends catalog retry rules without losing defaults + - **Trigger:** Operator adds one custom retry pattern for a built-in provider. + - **Actors:** Operator + - **Steps:** Add pattern in user config → validate → confirm merged provider includes catalog defaults plus the new pattern. + - **Outcome:** Failover behavior extends rather than resets. + - **Covered by:** R6, R7, R8 + +--- + +## Acceptance Examples + +- AE1. Missing env ref + - **Covers:** R1, R4 + - **Given:** An account references `env:MISSING_VAR` that is unset and has no `env_command`. + - **When:** The operator runs `clifwrap config validate`. + - **Then:** Output names the provider and account and suggests setting the variable or adding `env_command`. + +- AE2. Offline validation + - **Covers:** R5 + - **Given:** Usage URL is unreachable and the operator runs `doctor` without `--check`. + - **When:** Static validation passes but live probe fails. + - **Then:** Exit code is 0 with a warning; with `--check`, exit code is nonzero. + +- AE3. Partial retry pattern override + - **Covers:** R6, R7 + - **Given:** Catalog defines retry patterns A and B; user adds pattern C without a replace directive. + - **When:** Config is loaded and merged. + - **Then:** Effective patterns are A, B, and C. With explicit replace, effective patterns are only what the user specified. + +--- + +## Success Criteria + +- A new provider can be configured and validated locally before the first wrapped invocation. +- Operators can extend catalog list defaults without reading merge implementation source. +- `docs/provider-authoring.md` is sufficient for a contributor to propose a catalog entry in a PR without ad-hoc guidance. + +--- + +## Scope Boundaries + +**Deferred for later** + +- Python plugin or pip-discovered provider packages +- External provider fragments in `~/.config/clifwrap/providers.d/` (may follow if template + validation prove insufficient) +- Proactive quota gating, 429-vs-quota failover distinction, and switch audit JSONL (reliability track) +- keyring-first credential storage (security track) +- Full agent-native CLI surface (`schema` command, `--json` on every data command) (automation track) +- Windows shim install strategy (platform track) + +**Outside this product's identity** + +- Local HTTP proxy architecture (CliRelay-style) — clifwrap remains a process shim, not a proxy server +- Web management dashboard for accounts + +--- + +## Dependencies / Assumptions + +- Existing catalog in `src/clifwrap/providers.toml` and merge logic in `src/clifwrap/config.py` remain the extension mechanism. +- `doctor` and `config validate` already exist and are the right UX anchors; this track extends them rather than introducing parallel commands. +- Template provider uses anonymized placeholder hosts consistent with `scripts/check_anonymity.py` constraints. +- Operators may run validation offline; live probes must not be mandatory for basic config edits. + +--- + +## Outstanding Questions + +**Deferred to planning** + +- Exact syntax for list replace vs append (dedicated table key, operator prefix, or nested object). +- Whether provider probe runs one account or all enabled accounts by default. +- Whether template catalog provider ships enabled in the built-in catalog or is documented as a copy-paste starting point only. + +--- + +## Sources / Research + +- Repo: `src/clifwrap/config.py` (`merged_provider` list merge behavior), `src/clifwrap/__main__.py` (`config validate`, `doctor`), `docs/configuration.md` +- External: [CLI Spec](https://clispec.dev/) (schema, dry-run, scriptable exits), GitHub CLI `auth status` pattern, teamclaude quota-vs-rate-limit distinction, AWS credential precedence chain +- Prior exploration: improvement research covered reliability, security, and agent-native CLI as separate deferred tracks diff --git a/docs/brainstorms/2026-07-25-reliability-requirements.md b/docs/brainstorms/2026-07-25-reliability-requirements.md new file mode 100644 index 0000000..e40b7dd --- /dev/null +++ b/docs/brainstorms/2026-07-25-reliability-requirements.md @@ -0,0 +1,157 @@ +--- +date: 2026-07-25 +topic: reliability +--- + +## Summary + +Reduce wasted upstream attempts by picking a starting account with quota headroom before the first wrapped invocation. Extend existing capacity and status probing so failover begins on a viable account instead of always honoring a depleted default. + +--- + +## Problem Frame + +clifwrap fails over reactively: it runs on the default account (or `CLIFWRAP_CURRENT_ACCOUNT`), and only switches after a retryable exit code or output pattern match. When the default account is nearly exhausted, every invocation pays at least one doomed upstream call before failover — and may walk multiple accounts the same way on the next run. + +Capacity control can queue or shed before execution when snapshots show low headroom, but account selection for execution and failover does not consistently prefer the account with the most remaining quota. Operators with `status_command` configured still see wasted retries because headroom data is not used to choose where to start. + +--- + +## Key Decisions + +- **Headroom-first start, not full reliability suite.** This track targets proactive starting-account selection and tighter coupling between capacity snapshots and failover. Circuit breakers, streaming output scanning, 429-vs-quota taxonomy, and failover audit logs are deferred. +- **Lazy probe when default is healthy.** When the default account has headroom above reserve plus estimated cost, keep it — preserve operator intent and avoid probe latency on every invocation. +- **Probe and pick when default is low or unknown.** When default is below threshold or capacity is unknown everywhere, select the enabled account with the best headroom before the first upstream exec. +- **Fail open on missing status data.** If no account returns usable quota data, behavior matches today: start on default and fail over reactively. +- **Reuse scheduling, don't duplicate.** Extend the existing capacity admission path in `src/clifwrap/scheduling.py` rather than a parallel account-picker. + +--- + +## Requirements + +**Starting account selection** + +- R1. Before the first upstream invocation of a wrapped command, clifwrap evaluates enabled accounts that have usable capacity snapshots. +- R2. When the persisted default account has remaining quota at or above `reserve_threshold + estimated_cost`, that account is the starting account. +- R3. When the default is below threshold or capacity for the default is unknown, clifwrap selects the enabled account with the highest remaining quota (ties broken deterministically, e.g. config order). +- R4. Starting-account selection respects capacity-control policy: an account below reserve is not chosen unless no account meets reserve and policy is `execute` or `unknown_capacity_action` is `allow`. + +**Failover interaction** + +- R5. Once a starting account is chosen for a request, reactive failover (exit codes, output patterns) continues on that request's account chain without re-probing mid-request. +- R6. A retryable failure on the starting account still advances to the next account in the chain; proactive pick does not disable reactive failover. + +**Observability** + +- R7. When clifwrap overrides the default for headroom, stderr (human mode) notes which account was chosen and why (default low, better headroom elsewhere). +- R8. `status --json` and `doctor --json` expose per-account headroom and whether each account would be eligible as a starting account under current policy. + +**Configuration** + +- R9. Operators can disable proactive pick per provider (opt-out) while keeping capacity control; default is enabled when `status_command` or usage probing is configured. +- R10. `snapshot_ttl_seconds` behavior is documented: stale snapshots may approve a run that fails immediately — operators tune TTL vs probe cost. + +--- + +## Key Flows + +- F1. Default depleted, backup healthy + - **Trigger:** Operator runs a wrapped command; default account snapshot shows 2 remaining, reserve is 5, backup shows 40. + - **Actors:** Operator, clifwrap, upstream CLI + - **Steps:** Admission reads snapshots → default below reserve → pick backup → run upstream on backup env → succeed without trying default. + - **Outcome:** Zero wasted attempts on the depleted default. + - **Covered by:** R2, R3, R5 + +- F2. Default healthy + - **Trigger:** Default has ample headroom. + - **Actors:** Operator, clifwrap + - **Steps:** Snapshot check → default meets reserve → skip cross-account comparison → run on default. + - **Outcome:** No added latency from probing all accounts. + - **Covered by:** R2 + +- F3. All capacity unknown + - **Trigger:** `status_command` fails or returns unparseable data for every account. + - **Actors:** Operator, clifwrap + - **Steps:** Proactive pick skipped → start on default → reactive failover if retryable error occurs. + - **Outcome:** Behavior matches pre-track clifwrap; no regression when status unavailable. + - **Covered by:** R4, R6 + +--- + +## Acceptance Examples + +- AE1. Skip depleted default + - **Covers:** R1, R3, R5 + - **Given:** Default snapshot remaining=1, backup remaining=50, reserve=5, estimated cost=1. + - **When:** Wrapped command runs. + - **Then:** First upstream invocation uses backup account env; default is not called. + +- AE2. Keep healthy default + - **Covers:** R2 + - **Given:** Default remaining=100, backup remaining=200, reserve=5. + - **When:** Wrapped command runs. + - **Then:** First upstream invocation uses default; no account override message unless verbose/debug. + +- AE3. Unknown capacity fallback + - **Covers:** R4, R6 + - **Given:** No account returns valid snapshot; default is `acct-a`. + - **When:** Wrapped command runs and upstream returns retryable quota error on `acct-a`. + - **Then:** clifwrap fails over to `acct-b` reactively as today. + +- AE4. Status surfaces eligibility + - **Covers:** R8 + - **Given:** Mixed headroom across accounts. + - **When:** `clifwrap status searchcli --json`. + - **Then:** Each account entry includes headroom and a field indicating starting-account eligibility under current policy. + +--- + +## Success Criteria + +- A wrapped command with a depleted default and healthy backup completes without calling the depleted account. +- Healthy defaults see no perceptible latency regression (no full-account probe when default is clearly viable). +- Operators can see in `status` why an account would or would not be chosen to start. + +--- + +## Scope Boundaries + +**Deferred for later** + +- Per-account circuit breaker and cooldown after repeated failures +- Live stdout/stderr pattern scanning during streaming/TUI sessions +- Distinct handling for HTTP 429 rate-limit vs hard quota exhaustion +- Failover event log and switch audit trail (transparency track) +- Background quota watcher daemon for long-running interactive sessions + +**Outside this product's identity** + +- HTTP proxy that intercepts all traffic (teamclaude-style) +- Automatic purchase or provisioning of new upstream capacity + +--- + +## Dependencies / Assumptions + +- Extensibility track validation and merged provider config are available; proactive pick reads merged providers. +- `status_command` or usage URL probing already exists for built-in providers; providers without status data keep reactive-only behavior. +- Capacity control module (`src/clifwrap/scheduling.py`) and snapshot TTL in provider config remain the probe mechanism. +- Agent-native envelope work may land in parallel; this track adds fields to status payloads, compatible with a future envelope wrapper. + +--- + +## Outstanding Questions + +**Deferred to planning** + +- Exact opt-out config key name (`proactive_pick = false` vs nested under `capacity_control`). +- Whether queue replay uses the same starting-account logic as live wrapped commands. +- Verbose flag vs always-on stderr notice when default is overridden. + +--- + +## Sources / Research + +- Prior brainstorm: `docs/brainstorms/2026-07-25-extensibility-requirements.md` (deferred reliability items) +- Repo: `src/clifwrap/runtime.py` (account selection, failover), `src/clifwrap/scheduling.py` (admission, snapshots) +- External: claude-rotate and codex-quota proactive quota-pick patterns; improvement research ranking proactive pick as highest ROI for wasted retries diff --git a/docs/cli-reference.md b/docs/cli-reference.md index 0ce445b..aa58fb7 100644 --- a/docs/cli-reference.md +++ b/docs/cli-reference.md @@ -218,12 +218,17 @@ options: ## `clifwrap doctor` ```text -usage: clifwrap doctor [-h] [--json] [--check] +usage: clifwrap doctor [-h] [--json] [--check] [--provider PROVIDER] [--probe] [--all-accounts] options: - -h, --help show this help message and exit - --json Emit machine-readable JSON. - --check Exit nonzero when doctor finds config, shim, or queue issues. + -h, --help show this help message and exit + --json Emit machine-readable JSON. + --check Exit nonzero when doctor finds config, shim, or queue issues. + --provider PROVIDER Limit validation and probes to one provider. + --probe Run live credential probes (env_command, usage HTTP, status_command). Off + by default. + --all-accounts Probe every enabled account instead of the default account only. Implies + --probe. ``` ## `clifwrap init` diff --git a/docs/configuration.md b/docs/configuration.md index f2decde..4678f08 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -47,8 +47,32 @@ Provider tables can set: | `retry_on_any_error` | Retry any nonzero exit unless blocked by `never_retry_patterns` | | `interactive_mode` | Currently `line-repl` for line-by-line wrapped shells | | `status_command` | Command that returns JSON with `remaining`, `limit`, and optionally `used` | +| `proactive_pick` | Headroom-first starting account before the first upstream call (default `true` when `status_command` or `usage` is set) | | `passthrough_commands` | Upstream subcommands that skip wrapper auth handling | +### Catalog list merge + +Built-in catalog providers (`searchcli`, `scrapecli`, `examplecli`) ship default retry and passthrough lists in `src/clifwrap/providers.toml`. User overrides **append** by default: + +```toml +[providers.searchcli] +retry_patterns = ["my custom quota message"] +``` + +Effective patterns include catalog defaults plus `my custom quota message`. + +To replace catalog defaults entirely, use the `*_replace` sibling keys: + +```toml +[providers.searchcli] +retry_patterns_replace = ["only this pattern"] +never_retry_patterns_replace = ["parse error"] +retry_exit_codes_replace = [42] +passthrough_commands_replace = ["login"] +``` + +Append and replace keys apply to the same list fields documented above, including providers that have no catalog entry. See `examplecli` in [provider-catalog.md](provider-catalog.md) for a full catalog template. + ## Accounts Each account is a `[[providers..accounts]]` entry. @@ -150,7 +174,7 @@ remediation_commands = ["clifwrap account add somecli --env-ref SOMECLI_T | `command_costs` | Per-subcommand cost map | | `queue_retention_seconds` | How long queued items stay replayable | | `queue_max_items` | Max queued items per provider before new work is shed | -| `snapshot_ttl_seconds` | Usage cache TTL for admission decisions | +| `snapshot_ttl_seconds` | Usage cache TTL for admission and proactive starting-account decisions | | `remediation_message` | Message shown in status and queue output | | `remediation_commands` | Suggested commands for operators | @@ -167,6 +191,28 @@ Environment overrides: - `CLIFWRAP_PROVIDER__CAPACITY_REMEDIATION_MESSAGE` - `CLIFWRAP_PROVIDER__CAPACITY_REMEDIATION_COMMANDS` — comma-separated command strings +### Proactive starting account + +When `proactive_pick` is enabled (the default for providers with `status_command` or `usage` metadata), clifwrap probes account capacity before the first upstream call and starts on the account with the most headroom. The persisted default is kept when its remaining quota is at or above `reserve_threshold + estimated_cost` (or `1` when capacity control is absent). + +```toml +[providers.somecli] +status_command = ["somecli", "usage", "--json"] +proactive_pick = true # default when status_command or usage is set + +[providers.somecli.capacity_control] +reserve_threshold = 5 +default_cost = 1 +``` + +Set `proactive_pick = false` to keep legacy default-first ordering without headroom comparison. + +When the starting account differs from the default, clifwrap prints a one-line stderr notice naming the account and reason. Reactive failover after a retryable upstream error is unchanged. + +`status --json` and `doctor --json` include per-account `starting_eligible` and `starting_ineligible_reason` using the same reserve and cost rules. + +Higher `snapshot_ttl_seconds` reduces probe latency but can approve a run from stale quota data. Lower TTL increases probe cost and cold-start latency. + ## Usage lookups HTTP usage lookups feed `clifwrap status` and capacity control. diff --git a/docs/operations.md b/docs/operations.md index 9d0efbe..1cb2529 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -11,6 +11,7 @@ clifwrap config paths clifwrap config validate clifwrap doctor --check clifwrap doctor --json --check +clifwrap doctor --probe --check clifwrap status --check clifwrap status --json --check ``` @@ -18,8 +19,9 @@ clifwrap status --json --check | Command | What it checks | | --- | --- | | `config paths` | Resolved `CLIFWRAP_CONFIG`, `CLIFWRAP_STATE_DIR`, and `CLIFWRAP_BIN_DIR` | -| `config validate` | TOML parsing only — no shims or queue | -| `doctor --check` | Paths, config, shim records, backups, default account pointers, queue readability | +| `config validate` | Parsed config, provider static validation (accounts, env refs, metadata) | +| `doctor --check` | Above plus shim records, backups, default accounts, queue readability | +| `doctor --probe` | Adds live usage/status probes. Off by default because probes resolve credentials and make authenticated calls | | `status --check` | Low fallback pool, bad queue state, recovery-hook failures, capacity below policy | ## Account inventory @@ -49,6 +51,14 @@ clifwrap queue drop --expired `queue run` rechecks capacity before replay. If still below reserve, the item stays queued and replay metadata updates — nothing is duplicated. +## Proactive starting account + +When `proactive_pick` is enabled (default for providers with usage or `status_command` metadata), wrapped commands start on the account with the most known headroom instead of always trying the persisted default first. This reduces wasted retries when the default is nearly empty. + +- Tune `snapshot_ttl_seconds` against probe cost: stale snapshots may start on an account that fails immediately; lower TTL is fresher but slower on cold paths. +- `status --json` exposes `starting_eligible` per account for automation. +- Set `proactive_pick = false` per provider to restore default-first behavior. + ## Shim recovery Install is idempotent: diff --git a/docs/plans/2026-07-25-001-feat-provider-extensibility-plan.md b/docs/plans/2026-07-25-001-feat-provider-extensibility-plan.md new file mode 100644 index 0000000..d3ff0ad --- /dev/null +++ b/docs/plans/2026-07-25-001-feat-provider-extensibility-plan.md @@ -0,0 +1,292 @@ +--- +status: completed +date: 2026-07-25 +type: feat +origin: docs/brainstorms/2026-07-25-extensibility-requirements.md +title: Provider extensibility — validation, merge semantics, authoring +--- + +## Summary + +Extend clifwrap so operators can add and customize providers safely before runtime failure. Ship static and live provider validation through existing `config validate` and `doctor` commands, append-by-default catalog list merge with explicit replace opt-in, a template catalog provider, and a provider authoring guide. + +--- + +## Problem Frame + +Today `config validate` only checks that user TOML parses; `doctor` inspects shims, defaults, and queue state on raw user config. Neither uses `merged_provider()`, so catalog defaults and effective retry/passthrough rules are invisible until a wrapped command fails. List fields use Python `or` merge — a non-empty user list replaces the entire catalog list. Only `searchcli` and `scrapecli` exist as catalog examples with no authoring guide. + +(see origin: `docs/brainstorms/2026-07-25-extensibility-requirements.md`) + +--- + +## Requirements + +Traceability to origin R-IDs. Implementation must satisfy all. + +- R1. `config validate` reports provider-level issues beyond parse errors: missing accounts, unresolvable env refs, invalid merge declarations, malformed metadata. +- R2. `doctor` includes per-provider validation summary with remediation hints. +- R3. Live probe validates usage URL or `status_command` JSON shape and retry rule syntax without mutating config. +- R4. Validation messages name provider, config location, and next fix command. +- R5. Unreachable usage endpoints warn by default; `--check` fails with nonzero exit. +- R6. Catalog list fields append user entries by default. +- R7. Users can opt into full list replacement. +- R8. Merge rules documented in `docs/configuration.md` with examples. +- R9. `docs/provider-authoring.md` covers catalog vs config-only providers. +- R10. Template catalog provider demonstrates optional fields with safe placeholders. +- R11. Generated `docs/provider-catalog.md` includes the template provider. + +**Flows and acceptance examples (origin F1, F2, AE1–AE3)** are enforced via U2–U5 test scenarios. + +--- + +## Key Technical Decisions + +- **KTD1 — Dedicated validation module:** Add `src/clifwrap/validation.py` for static and probe logic. Keeps `__main__.py` thin and testable without provider name literals in generic modules (see `tests/test_wrapper.py` anonymity test). +- **KTD2 — Append + explicit replace keys for lists:** User `retry_patterns = ["c"]` appends to catalog defaults. User `retry_patterns_replace = ["c"]` replaces entirely. Same pattern for `never_retry_patterns`, `retry_exit_codes`, `passthrough_commands`, and nested lists exposed at provider level (`auth_aliases`, `capacity_remediation_commands` as optional replace siblings where catalog merge applies). Rationale: TOML-native, no magic prefixes, backward-compatible when user omits lists (catalog only). +- **KTD3 — Validate effective merged config:** Both `config validate` and `doctor` iterate providers present in user config **and** call `merged_provider(name, raw)` so catalog-backed providers reflect runtime behavior. Catalog-only providers with no user accounts are skipped unless `--provider` is set. +- **KTD4 — Probe scope:** Live probe runs against the default enabled account per provider, falling back to first enabled account. `--provider NAME` limits checks; `--all-accounts` (doctor only) probes every enabled account. +- **KTD5 — Severity model:** Findings carry `level` of `error` or `warning`. Static issues (missing env ref, no accounts, invalid replace key) are errors. Live probe failures (timeout, bad JSON, unreachable URL) are warnings unless `--check`. +- **KTD6 — Template provider name `examplecli`:** Ships in `providers.toml` with `*.example` hosts, minimal accounts none, documents every catalog subsection. Not intended for install/shim use; documented as reference-only in authoring guide. +- **KTD7 — No new top-level command:** Extend `config validate` and `doctor` per origin decision; optional `doctor --provider` filter only. + +--- + +## High-Level Technical Design + +```mermaid +flowchart TB + subgraph cli [CLI entrypoints] + CV[config validate] + DR[doctor] + end + subgraph val [validation.py] + ST[validate_static] + PR[probe_provider] + end + subgraph cfg [config.py] + LP[load_config] + MP[merged_provider] + LM[list merge append/replace] + end + CV --> LP + DR --> LP + CV --> ST + DR --> ST + DR --> PR + ST --> MP + PR --> MP + LP --> LM + LM --> MP +``` + +Validation pipeline: + +1. Load user config (fail fast on parse errors — existing behavior). +2. For each target provider, compute merged config via updated `merged_provider`. +3. Run static checks (accounts, env refs, duplicate names, replace-key consistency). +4. Optionally run live probe (HTTP GET for usage URL or subprocess for `status_command`). +5. Aggregate findings into validate/doctor payloads; apply `--check` exit policy. + +--- + +## Implementation Units + +### U1. Provider validation module + +**Goal:** Centralize static and live provider validation with structured findings. + +**Requirements:** R1, R3, R4, R5 + +**Dependencies:** None + +**Files:** +- Create `src/clifwrap/validation.py` +- Modify `tests/test_wrapper.py` + +**Approach:** +- Define `ValidationFinding` dataclass: `provider`, `field`, `level`, `message`, `remediation` (optional command string). +- `validate_provider_static(name, merged: ProviderConfig) -> list[ValidationFinding]`: + - No enabled accounts → error + - Per account: resolve `env` refs (`env:VAR`) — unset without `env_command` → error + - Empty `retry_patterns` entries, invalid usage URL scheme → error + - Duplicate account names → error +- `probe_provider(name, merged, account) -> list[ValidationFinding]`: + - If `usage.url` set: HTTP GET with timeout (reuse runtime status HTTP helpers or extract shared fetch to avoid duplication — prefer extracting minimal shared helper over importing runtime side effects) + - Elif `status_command`: run command, parse JSON for `remaining`/`limit` + - Timeout/unreachable/invalid JSON → warning-level finding +- Keep all logic provider-name agnostic; use merged config fields only. + +**Patterns to follow:** Dataclass style in `config.py`; no secrets in messages. + +**Test scenarios:** +- Covers AE1. Static validation flags missing `env:MISSING` with provider and account in message. +- Static validation passes for well-formed `somecli` fixture config. +- Probe returns warning finding when usage URL points at unreachable host (mock or localhost closed port). +- Probe accepts valid JSON from fake `status_command` script (subprocess fixture pattern from existing failover tests). +- Findings never contain secret values in `message` or `remediation`. + +**Verification:** Unit tests import `validation` directly; no provider literals in module source. + +--- + +### U2. Append-aware catalog list merge + +**Goal:** User list fields extend catalog defaults unless explicit replace keys are used. + +**Requirements:** R6, R7, R8 + +**Dependencies:** None (can land before U1; tests should cover merge in isolation) + +**Files:** +- Modify `src/clifwrap/config.py` +- Modify `tests/test_wrapper.py` +- Modify `docs/configuration.md` + +**Approach:** +- Extend `_provider_from_raw` to read optional `*_replace` keys alongside list fields. +- Update `merged_provider` merge block: + - If `retry_patterns_replace` set → use it; elif user `retry_patterns` non-empty → `catalog + user` deduped preserve order; else catalog. + - Apply same pattern to `never_retry_patterns`, `retry_exit_codes`, `passthrough_commands`. +- Document keys in `docs/configuration.md` with AE3 scenario (patterns A,B + user C → A,B,C). +- Update `test_configuration_docs_cover_runtime_override_surface` tokens if new env/table keys added. + +**Patterns to follow:** Existing `merged_provider` in `config.py`; env override layer remains full replace (unchanged). + +**Test scenarios:** +- Covers AE3. Catalog patterns `["a","b"]` + user `retry_patterns = ["c"]` → effective `["a","b","c"]`. +- User `retry_patterns_replace = ["c"]` → effective `["c"]` only. +- Empty user list inherits catalog (existing behavior preserved). +- `somecli` generic provider name only in tests. + +**Verification:** Direct `merged_provider()` assertions; doc contains "append" and "replace" examples. + +--- + +### U3. Extend `config validate` and `doctor` + +**Goal:** Wire validation into CLI commands with warn/check semantics and JSON payloads. + +**Requirements:** R1, R2, R4, R5 + +**Dependencies:** U1, U2 + +**Files:** +- Modify `src/clifwrap/__main__.py` +- Modify `tests/test_wrapper.py` +- Regenerate `docs/cli-reference.md` via `scripts/generate_cli_reference.py --write` + +**Approach:** +- Extend `_config_validate_payload`: + - After successful parse, run `validate_provider_static` on each merged provider. + - Add `findings: [{provider, level, message, remediation}]`, `warnings` count, `errors` count. + - `valid=False` when any error-level finding exists. +- Extend `_doctor_payload`: + - Add `validation` section per provider with static + probe findings. + - Add `--provider NAME` argparse filter. + - Add `--all-accounts` for probe fan-out. + - `--check`: exit 1 if any error **or** any warning (includes probe failures per R5). +- Human output: print findings grouped by provider; include remediation line when present. +- JSON output: stable keys, no secrets (extend existing tests). + +**Patterns to follow:** `_config_validate_payload` / `_doctor_payload` split; `--json` + `--check` conventions. + +**Test scenarios:** +- Covers AE1. `config validate --json` reports missing env ref with remediation text. +- Covers AE2. `doctor` without `--check` exits 0 with probe warning; with `--check` exits 1. +- `doctor --json` includes validation section when shims healthy. +- Existing doctor shim/queue tests still pass unchanged. + +**Verification:** `_run("config", "validate")` and `_run("doctor", "--check")` integration tests. + +--- + +### U4. Template catalog provider + +**Goal:** Add `examplecli` reference provider to built-in catalog and regenerate docs. + +**Requirements:** R10, R11 + +**Dependencies:** U2 (uses merge fields in template) + +**Files:** +- Modify `src/clifwrap/providers.toml` +- Regenerate `docs/provider-catalog.md` +- Modify `tests/test_wrapper.py` (if catalog section count asserted) + +**Approach:** +- Add `[providers.examplecli]` with one entry per optional subsection (`auth_management`, `fallback_monitor`, `usage`, `capacity_control`), placeholder `*.example` URLs, sample retry patterns, comment in TOML that provider is documentation-only. +- Run `python scripts/generate_provider_catalog.py --write`. +- Ensure `scripts/check_anonymity.py` passes (example hosts only). + +**Patterns to follow:** `searchcli` / `scrapecli` table layout in `providers.toml`. + +**Test scenarios:** +- `test_generated_provider_catalog_is_current` passes after regeneration. +- `merged_provider("examplecli", None)` loads without error. +- Catalog doc contains `examplecli` heading. + +**Verification:** `nox -s docs` check mode passes. + +--- + +### U5. Provider authoring guide and doc integration + +**Goal:** Document how to add providers and validate them before first use. + +**Requirements:** R8, R9, F1 + +**Dependencies:** U2, U3, U4 + +**Files:** +- Create `docs/provider-authoring.md` +- Modify `docs/operations.md` (validation workflow: validate → doctor → install) +- Modify `docs/configuration.md` (merge section from U2) +- Modify `scripts/verify_release.py` required public docs list if needed +- Modify `tests/test_wrapper.py` doc coverage test if manifest enforced + +**Approach:** +- Authoring guide sections: catalog vs config-only, field reference pointing to `providers.toml` + `examplecli`, validation workflow (F1 steps), common failure modes, PR checklist for new catalog providers. +- Link from `README.md` documentation table (one line — minimal scope). +- Add manifest entry in verify_release if other docs are enumerated. + +**Patterns to follow:** Tone and structure of `docs/configuration.md`, `docs/operations.md`. + +**Test scenarios:** +- Test expectation: none — documentation-only unit; verified by `verify_release.py` doc list and manual review. + +**Verification:** Doc paths exist; links use repo-relative references; release verifier passes. + +--- + +## Scope Boundaries + +### Deferred to Follow-Up Work + +- Python plugin / `providers.d` discovery (origin deferred) +- Splitting `runtime.py` (adjacent refactor, out of scope) +- `--json` schema command (automation track) + +### Carried from origin — out of scope + +- Reliability track (429 vs quota, audit JSONL) +- Security track (keyring) +- Windows shim strategy +- Proxy/dashboard architectures + +--- + +## Risks and Dependencies + +- **Merge behavior change:** Append semantics alter effective config for users who relied on full replace via non-empty lists. Mitigation: document in CHANGELOG; replace keys preserve old behavior explicitly. +- **Probe flakiness in CI:** Live HTTP probes must not run in default pytest (unit tests mock or use local fake servers only). Doctor probes are operator-initiated. +- **Runtime coupling:** Extract minimal HTTP/status parsing shared between validation and runtime to avoid duplicating logic or importing heavy runtime paths. + +--- + +## Sources and Research + +- Origin: `docs/brainstorms/2026-07-25-extensibility-requirements.md` +- Code: `src/clifwrap/config.py`, `src/clifwrap/__main__.py`, `src/clifwrap/providers.toml` +- Tests: `tests/test_wrapper.py` (`test_provider_specific_literals_stay_out_of_generic_wrapper_modules`, validate/doctor tests) +- External patterns: GitHub CLI `auth status` (--check, JSON), CLI Spec validation-before-mutation diff --git a/docs/plans/2026-07-25-002-feat-proactive-starting-account-plan.md b/docs/plans/2026-07-25-002-feat-proactive-starting-account-plan.md new file mode 100644 index 0000000..1bf4f15 --- /dev/null +++ b/docs/plans/2026-07-25-002-feat-proactive-starting-account-plan.md @@ -0,0 +1,239 @@ +--- +title: "feat: Proactive starting-account selection" +date: 2026-07-25 +type: feat +status: completed +origin: docs/brainstorms/2026-07-25-reliability-requirements.md +--- + +## Summary + +Pick the best starting account before the first upstream call when quota data is available, so depleted defaults are skipped instead of burning a retry. Extend the existing scheduling snapshots and admission path rather than building a parallel picker. + +--- + +## Problem Frame + +clifwrap fails over reactively: `_ordered_accounts` rotates the persisted default to the front, and `_run_attempts` walks the list until a retryable error triggers the next account. Capacity admission can approve execution and even select a different account, but selection today prefers the active/default account when it barely clears the floor, and tie-breaking follows config order rather than highest remaining headroom. + +Operators with `status_command` or usage URLs still pay at least one doomed upstream call when the default is nearly empty. The extensibility brainstorm deferred this work as the reliability track; this plan implements the headroom-first slice only. + +--- + +## Requirements + +- R1. Before the first upstream invocation, evaluate enabled accounts with usable capacity snapshots. (origin R1) +- R2. Keep the default account when its remaining quota is at or above `reserve_threshold + estimated_cost`. (origin R2) +- R3. When the default is below that threshold or capacity is unknown for the default, select the enabled account with the highest known remaining quota; ties break by config order. (origin R3) +- R4. Respect capacity-control policy: do not choose accounts below reserve unless policy allows execution with unknown or sub-reserve capacity. (origin R4) +- R5. Do not re-probe mid-request; starting account is fixed for the invocation. (origin R5) +- R6. Reactive failover after a retryable upstream error still walks the remaining account chain. (origin R6) +- R7. When the starting account differs from the default, emit a stderr notice naming the account and reason. (origin R7) +- R8. `status --json` and `doctor --json` expose per-account headroom and starting-account eligibility. (origin R8) +- R9. Per-provider opt-out via `proactive_pick = false`; default enabled when `status_command` or usage metadata exists. (origin R9) +- R10. Document `snapshot_ttl_seconds` trade-off in operations/configuration docs. (origin R10) + +--- + +## Key Technical Decisions + +- **KTD1: Extend `scheduling.py`, do not fork a second picker.** Add a `select_starting_account()` helper that consumes `CapacitySnapshot` lists and returns `(account_name, reason)`. Both `admission_decision` and a lightweight pre-run path call it. (see origin: reuse scheduling) + +- **KTD2: `proactive_pick` lives on `ProviderConfig`.** Default `true` when merged provider has `status_command` or `usage` metadata; operators set `proactive_pick = false` to keep legacy default-first behavior. Nested under `capacity_control` was rejected — proactive pick should work even when queue/shed policy is absent. + +- **KTD3: Starting pick rotates the attempt chain; it does not replace capacity lock semantics.** When capacity control approves a single account via `allowed_account_names`, existing reserve-protection behavior stays (see `test_capacity_admission_restricts_retry_failover_to_approved_account`). Proactive pick changes which account is tried first when multiple accounts are eligible — implemented by reordering `_ordered_accounts` to start at the picked account, not by narrowing the failover set unless capacity policy already does. + +- **KTD4: Lazy probe when default is healthy.** `select_starting_account` returns immediately when the default snapshot meets `preferred_remaining` without scanning other accounts for comparison. When default is below preferred or unknown, fetch/compare all enabled account snapshots (reuse `capacity_snapshots`). + +- **KTD5: Estimated cost source.** Use `estimate_command_cost` when `capacity_control` exists; otherwise use `default_cost` from catalog or `1` when only `status_command` is configured. Planning assumes catalog defaults remain conservative. + +--- + +## High-Level Technical Design + +```mermaid +flowchart TD + A[wrapped command] --> B{proactive_pick enabled?} + B -->|no| C[_ordered_accounts default-first] + B -->|yes| D[capacity_snapshots for enabled accounts] + D --> E{default meets preferred?} + E -->|yes| F[start on default] + E -->|no| G[pick max remaining eligible account] + F --> H[reorder account chain] + G --> H + H --> I{capacity_control?} + I -->|yes| J[admission_decision queue/shed/execute] + I -->|no| K[_run_attempts with reordered chain] + J --> L[_run_attempts] + C --> I +``` + +Admission and proactive pick share snapshots within one invocation to avoid duplicate status probes. + +--- + +## Scope Boundaries + +**In scope:** Headroom-first starting account, stderr override notice, status/doctor eligibility fields, config opt-out, tests, docs. + +**Deferred for later** (from origin, not this plan) + +- Per-account circuit breaker and cooldown +- Streaming stdout/stderr pattern scanning +- 429 vs hard-quota taxonomy +- Failover event log (transparency track) + +**Deferred to Follow-Up Work** + +- Agent-native JSON envelope wrapping for new status fields (parallel track; fields added in plain JSON first) + +**Outside this product's identity** + +- HTTP proxy architecture +- Automatic capacity provisioning + +--- + +## Open Questions + +**Deferred to implementation** + +- Whether queue replay should call the same `select_starting_account` path (default: yes — replay uses live snapshots at replay time) +- Exact stderr wording and whether a `CLIFWRAP_VERBOSE` gate is needed beyond always-on one-line notice + +--- + +## Implementation Units + +### U1. Provider config for proactive pick + +**Goal:** Add `proactive_pick` to merged provider config with sensible defaults. + +**Requirements:** R9 + +**Dependencies:** None + +**Files:** `src/clifwrap/config.py`, `src/clifwrap/providers.toml` (document default for examplecli), `docs/configuration.md` + +**Approach:** Add optional `proactive_pick: bool | None` on `ProviderConfig`. In `merged_provider`, resolve to `true` when `status_command` or `usage` is set unless user sets `false`. Catalog providers inherit enabled-by-default when they have status probes. + +**Patterns to follow:** Existing `capacity_control` merge in `config.py`. + +**Test scenarios:** + +- Merged provider with `status_command` and no user override → `proactive_pick` is true. +- User sets `proactive_pick = false` → remains false despite status_command. +- Provider with neither status nor usage → `proactive_pick` false. + +**Verification:** Config load tests pass; no behavior change until runtime wired. + +--- + +### U2. Starting-account selection helper + +**Goal:** Centralize headroom-first pick logic in scheduling. + +**Requirements:** R1, R2, R3, R4, R5 + +**Dependencies:** U1 + +**Files:** `src/clifwrap/scheduling.py`, `tests/test_wrapper.py` + +**Approach:** Add `select_starting_account(provider, snapshots, *, active_account_name, estimated_cost) -> tuple[str | None, str]` implementing lazy default-fast-path and max-remaining selection with config-order tie-break. Refactor `admission_decision` to call it instead of inline active-first loops so queue/shed paths share the same pick semantics. + +**Patterns to follow:** Existing `preferred_remaining` math in `admission_decision` (`src/clifwrap/scheduling.py`). + +**Test scenarios:** + +- Covers AE1. Default remaining=1, backup=50, reserve=5, cost=1 → returns backup. +- Covers AE2. Default remaining=100, backup=200 → returns default without considering backup. +- All snapshots unknown → returns active/default with reason `capacity unknown`. +- Two accounts tie on remaining → earlier config order wins. + +**Verification:** Unit-level tests on `select_starting_account` with synthetic snapshots. + +--- + +### U3. Wire proactive pick into runtime execution + +**Goal:** Reorder account attempts before first upstream exec when proactive pick is enabled. + +**Requirements:** R5, R6, R7 + +**Dependencies:** U2 + +**Files:** `src/clifwrap/runtime.py`, `tests/test_wrapper.py` + +**Approach:** + +- Add `_reorder_accounts_from(accounts, start_name)` helper. +- In `run_app`, after stdin capture when needed, if `proactive_pick` and enabled accounts exist: build snapshots once, call `select_starting_account`, reorder chain for `_run_attempts`. +- When capacity control runs, pass the same snapshots into `_admission` to avoid double probe; thread reordered start into admission's `active_account_name`. +- Emit stderr when start differs from default: `[clifwrap] starting on {name}: {reason}`. +- Preserve `allowed_account_names` capacity lock behavior unchanged. + +**Patterns to follow:** `_admission` + `run_app` flow (`src/clifwrap/runtime.py` ~697–745). + +**Test scenarios:** + +- Covers AE1. Integration: depleted default, healthy backup, wrapped command succeeds on first attempt without failover message. +- Covers AE3. No valid snapshots → starts on default, reactive failover still works. +- Covers R6. `default_action = execute`, primary picked but fails retryably → backup still runs (extends `test_capacity_default_execute_preserves_normal_retry_failover`). +- Override notice appears on stderr when backup chosen over depleted default. + +**Verification:** Targeted tests green; existing capacity admission tests still pass. + +--- + +### U4. Status and doctor eligibility fields + +**Goal:** Expose headroom and starting-account eligibility in JSON status surfaces. + +**Requirements:** R8 + +**Dependencies:** U2 + +**Files:** `src/clifwrap/runtime.py` (`_status_snapshot`), `src/clifwrap/__main__.py` (doctor payload if needed), `tests/test_wrapper.py`, `docs/operations.md` + +**Approach:** For each account row in `_status_snapshot`, add `starting_eligible: bool` and `starting_ineligible_reason: str | null` computed via the same `preferred_remaining` rules and `select_starting_account` inputs. Use a nominal `estimated_cost` from capacity control or catalog default. + +**Test scenarios:** + +- Covers AE4. `status --json` shows eligibility flags matching snapshot remainings and reserve policy. + +**Verification:** JSON shape stable aside from new keys; doctor inherits validation section unchanged. + +--- + +### U5. Documentation and catalog notes + +**Goal:** Operators understand proactive pick, opt-out, and TTL trade-offs. + +**Requirements:** R9, R10 + +**Dependencies:** U1, U3 + +**Files:** `docs/configuration.md`, `docs/operations.md`, `README.md` (short mention), regenerate `docs/cli-reference.md` if flags added + +**Approach:** Document `proactive_pick`, lazy default behavior, stderr notice, and `snapshot_ttl_seconds` stale-read risk. Add example TOML snippet. + +**Test scenarios:** Test expectation: none — documentation only. + +**Verification:** `python scripts/generate_cli_reference.py --check` if CLI surface changes. + +--- + +## Risks and Dependencies + +- **Depends on extensibility branch landing or merging first** if validation/doctor changes conflict; proactive pick is independent but shares `runtime.py` and `config.py`. +- **Stale snapshot false positives:** Mitigated by R10 documentation; no code change beyond existing TTL. +- **Latency on cold probe:** Mitigated by KTD4 lazy path when default is healthy. + +--- + +## Sources and Research + +- Origin: `docs/brainstorms/2026-07-25-reliability-requirements.md` +- Prior art: `docs/plans/2026-06-29-001-feat-quota-aware-gating-plan.md` (admission module) +- Code: `src/clifwrap/scheduling.py`, `src/clifwrap/runtime.py` (`run_app`, `_admission`, `_status_snapshot`) +- Tests: `test_capacity_admission_restricts_retry_failover_to_approved_account`, `test_capacity_default_execute_preserves_normal_retry_failover` diff --git a/docs/provider-authoring.md b/docs/provider-authoring.md new file mode 100644 index 0000000..35ada09 --- /dev/null +++ b/docs/provider-authoring.md @@ -0,0 +1,59 @@ +# Provider authoring + +This guide explains how to extend clifwrap with new CLI providers without reading implementation source. + +## Two paths + +**Config-only provider** — Add accounts and optional overrides under `[providers.]` in `~/.config/clifwrap/config.toml`. You must define retry rules, usage lookup, and capacity metadata yourself if you need failover or quota gating. + +**Catalog provider** — Add a `[providers.]` table to `src/clifwrap/providers.toml` in the repository (or ship a private fork). Catalog entries document retry behavior, usage URLs, auth aliases, and capacity defaults. Users normally only add `[[providers..accounts]]` entries locally. + +Use `examplecli` in [provider-catalog.md](provider-catalog.md) as the reference template. It is documentation-only — do not install a shim for it. + +## Required pieces + +| Concern | Config-only | Catalog | +| --- | --- | --- | +| Accounts | Required in user config | Required in user config | +| Retry rules | Recommended in user config | Defaults in catalog; append or replace | +| Usage or `status_command` | Required for `status` / capacity | Recommended in catalog | +| Auth metadata | Optional | Recommended when upstream has auth subcommands | + +## Validation workflow + +Before installing a shim, validate configuration locally: + +```bash +clifwrap config validate +clifwrap config validate --json +clifwrap doctor +clifwrap doctor --check +clifwrap doctor --probe --provider somecli +clifwrap doctor --provider somecli --all-accounts +``` + +`config validate` catches static issues: missing accounts, unset `env:` references, duplicate account names, and malformed metadata. + +`doctor --probe` adds live probes when usage URLs or `status_command` are configured (`--all-accounts` implies it). Probes are opt-in because they resolve account credentials, run `env_command` lookups, and make authenticated requests. Probes warn by default; `--check` exits nonzero on warnings or errors (useful in CI). + +Probe findings never echo command output or argv — a failing `status_command` is reported by exit status only, so tokens printed on stderr stay out of `doctor --json`. + +## Common failure modes + +- **Runtime failure on first wrap** — Usually a missing `env:VAR` or wrong retry patterns. Run `clifwrap config validate` first. +- **Catalog override dropped defaults** — Setting a non-empty list without `*_replace` previously replaced the whole catalog list. Append is now the default; use `retry_patterns_replace` when you intend to drop catalog defaults. +- **Usage probe warnings offline** — Expected when the usage endpoint is unreachable. Warnings do not fail `doctor` unless `--check` is set. + +## Contributing a catalog provider + +1. Copy the `examplecli` block in `src/clifwrap/providers.toml`. +2. Use anonymized `*.example` hosts (see `scripts/check_anonymity.py`). +3. Regenerate docs: `python scripts/generate_provider_catalog.py --write`. +4. Add integration tests in `tests/test_wrapper.py` using a generic `somecli` fixture where possible. +5. Do not add provider name literals to generic modules (`config.py`, `runtime.py`, etc.). + +## Related docs + +- [configuration.md](configuration.md) — field reference and merge semantics +- [provider-catalog.md](provider-catalog.md) — generated catalog reference +- [operations.md](operations.md) — day-two operations diff --git a/docs/provider-catalog.md b/docs/provider-catalog.md index cfe96dc..633513d 100644 --- a/docs/provider-catalog.md +++ b/docs/provider-catalog.md @@ -3,6 +3,56 @@ This file is generated from `src/clifwrap/providers.toml`. Run `python scripts/generate_provider_catalog.py --write` after changing built-in provider metadata. +## `examplecli` + +| Field | Value | +| --- | --- | +| `interactive_mode` | `line-repl` | +| `passthrough_commands` | `login` | +| `retry_exit_codes` | `1` | +| `retry_patterns` | `quota exceeded`, `rate limit` | +| `never_retry_patterns` | `unknown option` | + +### Auth Management + +| Field | Value | +| --- | --- | +| `command` | `auth` | +| `aliases` | `accounts`, `credentials` | + +### Fallback Monitor + +| Field | Value | +| --- | --- | +| `threshold` | `2` | +| `action` | `warn` | +| `journald` | false | +| `syslog` | true | +| `stderr` | true | + +### Usage + +| Field | Value | +| --- | --- | +| `url` | `https://api.examplecli.example/usage` | +| `auth_env` | `EXAMPLECLI_API_KEY` | +| `timeout_seconds` | `10` | +| `remaining_path` | `remaining` | +| `limit_path` | `limit` | +| `label` | `quota` | + +### Capacity Control + +| Field | Value | +| --- | --- | +| `default_action` | `queue` | +| `unknown_capacity_action` | `allow` | +| `reserve_threshold` | `1` | +| `default_cost` | `1` | +| `command_costs` | `crawl=3`, `search=1` | +| `remediation_message` | `Add another account or raise quota before replaying queued work.` | +| `remediation_commands` | `clifwrap account list examplecli`, `clifwrap config validate` | + ## `scrapecli` | Field | Value | diff --git a/src/clifwrap/__main__.py b/src/clifwrap/__main__.py index e2e10bf..658e3e7 100644 --- a/src/clifwrap/__main__.py +++ b/src/clifwrap/__main__.py @@ -9,10 +9,11 @@ from . import __version__ from .accounts import append_account, ensure_config_exists, ensure_default_account_valid, import_account_spec, list_accounts, parse_assignments, parse_command_assignments, remove_account, rename_account, set_account_enabled -from .config import SHIM_ENV, config_path, load_config, shim_bin_dir, state_dir +from .config import SHIM_ENV, config_path, load_config, merged_provider, shim_bin_dir, state_dir from .install import install_shim, is_managed_shim, load_state, uninstall_shim from .runtime import replay_queue, run_app, status_all, status_for from .state import drop_queue_items, get_default_account, list_queue_items, set_default_account +from .validation import findings_to_dict, probe_accounts, validate_provider_static def _parser() -> argparse.ArgumentParser: @@ -34,6 +35,9 @@ def _parser() -> argparse.ArgumentParser: doctor_parser = subparsers.add_parser("doctor", help="Inspect local config, state, shims, and queue health.") doctor_parser.add_argument("--json", action="store_true", help="Emit machine-readable JSON.") doctor_parser.add_argument("--check", action="store_true", help="Exit nonzero when doctor finds config, shim, or queue issues.") + doctor_parser.add_argument("--provider", help="Limit validation and probes to one provider.") + doctor_parser.add_argument("--probe", action="store_true", help="Run live credential probes (env_command, usage HTTP, status_command). Off by default.") + doctor_parser.add_argument("--all-accounts", action="store_true", help="Probe every enabled account instead of the default account only. Implies --probe.") init_parser = subparsers.add_parser("init", help="Create an empty config file if it does not exist.") init_parser.add_argument("--force", action="store_true", help="Overwrite the existing config with an empty config.") @@ -171,7 +175,13 @@ def main(argv: list[str] | None = None) -> int: return status_all(json_output=args.json, check=args.check) return status_for(args.app, json_output=args.json, check=args.check) if args.command == "doctor": - return _doctor(json_output=args.json, check=args.check) + return _doctor( + json_output=args.json, + check=args.check, + provider_filter=getattr(args, "provider", None), + probe=getattr(args, "probe", False), + all_accounts=getattr(args, "all_accounts", False), + ) if args.command == "init": path = ensure_config_exists(force=args.force) print(f"config: {path}") @@ -363,6 +373,14 @@ def _config_paths(*, json_output: bool) -> int: return 0 +def _provider_validation_findings(config) -> list: + findings = [] + for name, raw in sorted(config.providers.items()): + merged = merged_provider(name, raw) + findings.extend(validate_provider_static(merged)) + return findings + + def _config_validate_payload() -> dict[str, object]: path = config_path() try: @@ -376,16 +394,25 @@ def _config_validate_payload() -> dict[str, object]: "providers": 0, "accounts": 0, "enabled_accounts": 0, + "findings": [], + "warnings": 0, + "errors": 0, } accounts = [account for provider in config.providers.values() for account in provider.accounts] + findings = _provider_validation_findings(config) + errors = sum(1 for finding in findings if finding.level == "error") + warnings = sum(1 for finding in findings if finding.level == "warning") return { "path": str(path), "exists": path.exists(), - "valid": True, - "error": None, + "valid": errors == 0, + "error": None if errors == 0 else f"{errors} provider validation error(s)", "providers": len(config.providers), "accounts": len(accounts), "enabled_accounts": sum(1 for account in accounts if account.enabled), + "findings": findings_to_dict(findings), + "warnings": warnings, + "errors": errors, } @@ -399,13 +426,29 @@ def _config_validate(*, json_output: bool) -> int: f"valid: providers={payload['providers']} accounts={payload['accounts']} " f"enabled_accounts={payload['enabled_accounts']}" ) + findings = payload.get("findings") or [] + if findings: + print("findings:") + for finding in findings: + assert isinstance(finding, dict) + print(f"- [{finding['level']}] {finding['message']}") + if finding.get("remediation"): + print(f" remediation: {finding['remediation']}") else: print(f"config: {payload['path']} ({'present' if payload['exists'] else 'missing'})") print(f"invalid: {payload['error']}") + findings = payload.get("findings") or [] + if findings: + print("findings:") + for finding in findings: + assert isinstance(finding, dict) + print(f"- [{finding['level']}] {finding['message']}") + if finding.get("remediation"): + print(f" remediation: {finding['remediation']}") return 0 if payload["valid"] else 1 -def _doctor_payload() -> dict[str, object]: +def _doctor_payload(*, provider_filter: str | None = None, probe: bool = False, probe_all_accounts: bool = False) -> dict[str, object]: config_file = config_path() state_directory = state_dir() bin_directory = shim_bin_dir() @@ -434,6 +477,26 @@ def _doctor_payload() -> dict[str, object]: except ValueError as exc: queue_error = str(exc) issues.append(queue_error) + validation_findings = [] + if config is not None: + provider_names = sorted(config.providers.keys()) + if provider_filter: + if provider_filter not in config.providers: + issues.append(f"provider {provider_filter!r} is not configured") + provider_names = [] + else: + provider_names = [provider_filter] + for provider_name in provider_names: + raw = config.providers[provider_name] + merged = merged_provider(provider_name, raw) + validation_findings.extend(validate_provider_static(merged)) + if probe: + validation_findings.extend( + probe_accounts(merged, all_accounts=probe_all_accounts) + ) + for finding in validation_findings: + if finding.level == "error": + issues.append(finding.message) if config is not None: for provider_name, provider in sorted(config.providers.items()): enabled_accounts = [account.name for account in provider.accounts if account.enabled] @@ -502,11 +565,18 @@ def _doctor_payload() -> dict[str, object]: "error": queue_error, }, "issues": issues, + "validation": findings_to_dict(validation_findings), + "validation_warnings": sum(1 for finding in validation_findings if finding.level == "warning"), + "validation_errors": sum(1 for finding in validation_findings if finding.level == "error"), } -def _doctor(*, json_output: bool, check: bool) -> int: - payload = _doctor_payload() +def _doctor(*, json_output: bool, check: bool, provider_filter: str | None = None, probe: bool = False, all_accounts: bool = False) -> int: + payload = _doctor_payload( + provider_filter=provider_filter, + probe=probe or all_accounts, + probe_all_accounts=all_accounts, + ) if json_output: sys.stdout.write(json.dumps(payload, indent=2, sort_keys=True) + "\n") else: @@ -529,6 +599,16 @@ def _doctor(*, json_output: bool, check: bool) -> int: f"default={provider['default_account'] or '-'} shim={shim_state}" ) print(f"queue: pending={queue_state['pending']} expired={queue_state['expired']}") + validation = payload.get("validation") or [] + if validation: + print("validation:") + for finding in validation: + assert isinstance(finding, dict) + print(f"- [{finding['level']}] {finding['message']}") + if finding.get("remediation"): + print(f" remediation: {finding['remediation']}") + else: + print("validation: none") issues = payload["issues"] if issues: print("issues:") @@ -536,7 +616,12 @@ def _doctor(*, json_output: bool, check: bool) -> int: print(f"- {issue}") else: print("issues: none") - return 1 if check and payload["issues"] else 0 + validation = payload.get("validation") or [] + validation_fail = any( + isinstance(finding, dict) and finding.get("level") in {"error", "warning"} + for finding in validation + ) + return 1 if (check and (payload["issues"] or validation_fail)) else 0 if __name__ == "__main__": diff --git a/src/clifwrap/config.py b/src/clifwrap/config.py index 1a2e328..dbf18bb 100644 --- a/src/clifwrap/config.py +++ b/src/clifwrap/config.py @@ -50,21 +50,41 @@ class AccountConfig: prepare_on: str = "always" +def _merge_list_field(catalog: list, user: list, replace: list | None) -> list: + if replace is not None: + return list(replace) + if user: + merged: list = [] + seen: set = set() + for item in [*catalog, *user]: + if item in seen: + continue + seen.add(item) + merged.append(item) + return merged + return list(catalog) + + @dataclass class ProviderConfig: name: str command: list[str] | None = None retry_exit_codes: list[int] = field(default_factory=list) + retry_exit_codes_replace: list[int] | None = None retry_patterns: list[str] = field(default_factory=list) + retry_patterns_replace: list[str] | None = None never_retry_patterns: list[str] = field(default_factory=list) + never_retry_patterns_replace: list[str] | None = None retry_on_any_error: bool = False interactive_mode: str | None = None status_command: list[str] | None = None passthrough_commands: list[str] = field(default_factory=list) + passthrough_commands_replace: list[str] | None = None auth_management: "AuthManagementConfig | None" = None fallback_monitor: "FallbackMonitorConfig | None" = None usage: "UsageConfig | None" = None capacity_control: "CapacityControlConfig | None" = None + proactive_pick: bool | None = None accounts: list[AccountConfig] = field(default_factory=list) @@ -395,16 +415,41 @@ def _provider_from_raw(name: str, raw_provider: dict[str, Any]) -> ProviderConfi name=name, command=_command_list(raw_provider.get("command"), field_name=f"providers.{name}.command") or None, retry_exit_codes=[int(item) for item in raw_provider.get("retry_exit_codes", [])], + retry_exit_codes_replace=( + [int(item) for item in raw_provider["retry_exit_codes_replace"]] + if "retry_exit_codes_replace" in raw_provider + else None + ), retry_patterns=[str(item).lower() for item in raw_provider.get("retry_patterns", [])], + retry_patterns_replace=( + [str(item).lower() for item in raw_provider["retry_patterns_replace"]] + if "retry_patterns_replace" in raw_provider + else None + ), never_retry_patterns=[str(item).lower() for item in raw_provider.get("never_retry_patterns", [])], + never_retry_patterns_replace=( + [str(item).lower() for item in raw_provider["never_retry_patterns_replace"]] + if "never_retry_patterns_replace" in raw_provider + else None + ), retry_on_any_error=bool(raw_provider.get("retry_on_any_error", False)), interactive_mode=raw_provider.get("interactive_mode"), status_command=_command_list(raw_provider.get("status_command"), field_name=f"providers.{name}.status_command") or None, passthrough_commands=_command_list(raw_provider.get("passthrough_commands"), field_name=f"providers.{name}.passthrough_commands"), + passthrough_commands_replace=( + _command_list(raw_provider.get("passthrough_commands_replace"), field_name=f"providers.{name}.passthrough_commands_replace") + if "passthrough_commands_replace" in raw_provider + else None + ), auth_management=_auth_management_from_raw(raw_provider.get("auth_management"), field_name=f"providers.{name}.auth_management"), fallback_monitor=_fallback_monitor_from_raw(raw_provider.get("fallback_monitor"), field_name=f"providers.{name}.fallback_monitor"), usage=_usage_from_raw(raw_provider.get("usage"), field_name=f"providers.{name}.usage"), capacity_control=_capacity_control_from_raw(raw_provider.get("capacity_control"), field_name=f"providers.{name}.capacity_control"), + proactive_pick=( + bool(raw_provider["proactive_pick"]) + if "proactive_pick" in raw_provider + else None + ), accounts=_load_accounts(raw_provider.get("accounts")), ) @@ -434,25 +479,35 @@ def merged_provider(name: str, raw: ProviderConfig | None) -> ProviderConfig: catalog = catalog_provider(name) if catalog is None and raw is None: return ProviderConfig(name=name) - if catalog is None: - provider = raw or ProviderConfig(name=name) - elif raw is None: + if raw is None: provider = catalog else: + # Providers with no catalog entry merge against an empty base so that + # *_replace keys still take effect instead of silently doing nothing. + base = catalog or ProviderConfig(name=name) provider = ProviderConfig( name=name, - command=raw.command or catalog.command, - retry_exit_codes=raw.retry_exit_codes or catalog.retry_exit_codes, - retry_patterns=raw.retry_patterns or catalog.retry_patterns, - never_retry_patterns=raw.never_retry_patterns or catalog.never_retry_patterns, - retry_on_any_error=raw.retry_on_any_error or catalog.retry_on_any_error, - interactive_mode=raw.interactive_mode or catalog.interactive_mode, - status_command=raw.status_command or catalog.status_command, - passthrough_commands=raw.passthrough_commands or catalog.passthrough_commands, - auth_management=raw.auth_management or catalog.auth_management, - fallback_monitor=raw.fallback_monitor or catalog.fallback_monitor, - usage=raw.usage or catalog.usage, - capacity_control=raw.capacity_control or catalog.capacity_control, + command=raw.command or base.command, + retry_exit_codes=_merge_list_field(base.retry_exit_codes, raw.retry_exit_codes, raw.retry_exit_codes_replace), + retry_patterns=_merge_list_field(base.retry_patterns, raw.retry_patterns, raw.retry_patterns_replace), + never_retry_patterns=_merge_list_field( + base.never_retry_patterns, + raw.never_retry_patterns, + raw.never_retry_patterns_replace, + ), + retry_on_any_error=raw.retry_on_any_error or base.retry_on_any_error, + interactive_mode=raw.interactive_mode or base.interactive_mode, + status_command=raw.status_command or base.status_command, + passthrough_commands=_merge_list_field( + base.passthrough_commands, + raw.passthrough_commands, + raw.passthrough_commands_replace, + ), + auth_management=raw.auth_management or base.auth_management, + fallback_monitor=raw.fallback_monitor or base.fallback_monitor, + usage=raw.usage or base.usage, + capacity_control=raw.capacity_control or base.capacity_control, + proactive_pick=raw.proactive_pick if raw.proactive_pick is not None else base.proactive_pick, accounts=raw.accounts, ) auth = provider.auth_management @@ -560,6 +615,8 @@ def merged_provider(name: str, raw: ProviderConfig | None) -> ProviderConfig: raise ValueError(f"{capacity_max_items_env} must be at least 1") if provider.capacity_control.snapshot_ttl_seconds < 1: raise ValueError(f"{capacity_snapshot_ttl_env} must be at least 1") + if provider.proactive_pick is None: + provider.proactive_pick = bool(provider.status_command or provider.usage) return provider diff --git a/src/clifwrap/providers.toml b/src/clifwrap/providers.toml index 8acf481..bd1c6bb 100644 --- a/src/clifwrap/providers.toml +++ b/src/clifwrap/providers.toml @@ -117,3 +117,44 @@ remediation_commands = [ "clifwrap account list scrapecli", "clifwrap account add scrapecli --env-ref SCRAPECLI_API_KEY=ENVVAR", ] + +# Reference-only catalog entry. Copy fields into user config or a forked providers.toml +# when authoring a new provider. Do not install a shim for examplecli. +# proactive_pick defaults to true when usage or status_command is configured. +[providers.examplecli] +retry_exit_codes = [1] +retry_patterns = ["quota exceeded", "rate limit"] +never_retry_patterns = ["unknown option"] +passthrough_commands = ["login"] +interactive_mode = "line-repl" + +[providers.examplecli.auth_management] +command = "auth" +aliases = ["accounts", "credentials"] + +[providers.examplecli.fallback_monitor] +threshold = 2 +action = "warn" +journald = false +syslog = true +stderr = true + +[providers.examplecli.usage] +url = "https://api.examplecli.example/usage" +auth_env = "EXAMPLECLI_API_KEY" +timeout_seconds = 10 +remaining_path = "remaining" +limit_path = "limit" +label = "quota" + +[providers.examplecli.capacity_control] +default_action = "queue" +unknown_capacity_action = "allow" +reserve_threshold = 1 +default_cost = 1 +command_costs = { search = 1, crawl = 3 } +remediation_message = "Add another account or raise quota before replaying queued work." +remediation_commands = [ + "clifwrap account list examplecli", + "clifwrap config validate", +] diff --git a/src/clifwrap/runtime.py b/src/clifwrap/runtime.py index 1f89c3d..e47a6f5 100644 --- a/src/clifwrap/runtime.py +++ b/src/clifwrap/runtime.py @@ -16,7 +16,14 @@ from .accounts import append_account, ensure_default_account_valid, list_accounts, parse_assignments, parse_command_assignments, remove_account, rename_account, set_account_enabled from .config import BYPASS_ENV, CURRENT_ACCOUNT_ENV, AccountConfig, ProviderConfig, WrapperConfig, load_config, merged_provider, state_dir from .install import original_command_for -from .scheduling import admission_decision, capacity_snapshots, has_capacity_control +from .scheduling import ( + admission_decision, + capacity_snapshots, + has_capacity_control, + nominal_command_cost, + select_starting_account, + starting_account_eligibility, +) from .state import QueueItem, drop_queue_items, get_default_account, list_queue_items, replace_queue_item, set_default_account @@ -388,6 +395,47 @@ def _ordered_accounts(provider: ProviderConfig, allowed_names: set[str] | None = return accounts +def _proactive_pick_enabled(provider: ProviderConfig) -> bool: + if provider.proactive_pick is not None: + return provider.proactive_pick + return bool(provider.status_command or provider.usage) + + +def _reorder_accounts_from(accounts: list[AccountConfig], start_name: str | None) -> list[AccountConfig]: + if not start_name: + return accounts + for index, account in enumerate(accounts): + if account.name == start_name: + return accounts[index:] + accounts[:index] + return accounts + + +def _proactive_execution_plan( + provider: ProviderConfig, + args: list[str], +) -> tuple[list[AccountConfig] | None, list | None, str | None]: + if not _proactive_pick_enabled(provider): + return None, None, None + accounts = _ordered_accounts(provider) + if not accounts: + return None, None, None + reserve_threshold = provider.capacity_control.reserve_threshold if provider.capacity_control else 0 + estimated_cost = nominal_command_cost(provider, args) + snapshots = capacity_snapshots(provider, accounts, lambda account: _account_capacity_status(provider, account)) + active = _active_account_name(provider, accounts) + selection = select_starting_account( + snapshots, + active_account_name=active, + estimated_cost=estimated_cost, + reserve_threshold=reserve_threshold, + ) + reordered = _reorder_accounts_from(accounts, selection.account_name) + notice = None + if selection.account_name and active and selection.account_name != active: + notice = f"[clifwrap] starting on {selection.account_name}: {selection.reason}\n" + return reordered, snapshots, notice + + def _account_capacity_status(provider: ProviderConfig, account: AccountConfig) -> tuple[str, int | None]: env = _account_env(account) usage_status = _usage_status(provider, account, env) @@ -396,10 +444,18 @@ def _account_capacity_status(provider: ProviderConfig, account: AccountConfig) - return _generic_status(provider, account, env) -def _admission(provider: ProviderConfig, args: list[str], stdin_data: bytes | None, *, existing_item: QueueItem | None = None): +def _admission( + provider: ProviderConfig, + args: list[str], + stdin_data: bytes | None, + *, + existing_item: QueueItem | None = None, + snapshots: list | None = None, +): accounts = _ordered_accounts(provider) active = _active_account_name(provider, accounts) - snapshots = capacity_snapshots(provider, accounts, lambda account: _account_capacity_status(provider, account)) + if snapshots is None: + snapshots = capacity_snapshots(provider, accounts, lambda account: _account_capacity_status(provider, account)) decision = admission_decision( provider, args, @@ -421,9 +477,15 @@ def _run_attempts( *, emit_output: bool = True, allowed_account_names: set[str] | None = None, + account_order: list[AccountConfig] | None = None, ) -> int: command = _original_command(app, provider) - accounts = _ordered_accounts(provider, allowed_account_names) + if account_order is not None: + accounts = account_order + if allowed_account_names is not None: + accounts = [account for account in accounts if account.name in allowed_account_names] + else: + accounts = _ordered_accounts(provider, allowed_account_names) if not accounts: env = dict(os.environ) env[BYPASS_ENV] = "1" @@ -718,31 +780,48 @@ def run_app(app: str, args: list[str]) -> int: return _run_line_repl(app, provider) stdin_data: bytes | None | object = _STDIN_UNSET allowed_account_names: set[str] | None = None - if args and has_capacity_control(provider) and _enabled_accounts(provider) and not _requests_upstream_help(args): - stdin_data = _capture_stdin() - try: - decision = _admission(provider, args, stdin_data) - except (OSError, ValueError) as exc: - sys.stderr.write(f"[clifwrap] queue state error for {app}: {exc}\n") - return 74 - if decision.action == "queue": - item = decision.queue_item - sys.stderr.write(f"[clifwrap] deferred {app} {' '.join(args)}: {decision.reason}\n") - if item: - sys.stderr.write(f"[clifwrap] queued as {item.id}\n") - if decision.remediation_message: - sys.stderr.write(f"[clifwrap] {decision.remediation_message}\n") - return 73 - if decision.action == "shed": - sys.stderr.write(f"[clifwrap] blocked {app} {' '.join(args)}: {decision.reason}\n") - if decision.remediation_message: - sys.stderr.write(f"[clifwrap] {decision.remediation_message}\n") - return 69 - if decision.unknown_capacity: - sys.stderr.write(f"[clifwrap] capacity unknown for {app}; continuing because policy allows it\n") - elif decision.capacity_approved and decision.account_name: - allowed_account_names = {decision.account_name} - return _run_attempts(app, provider, args, stdin_data, allowed_account_names=allowed_account_names) + account_order: list[AccountConfig] | None = None + snapshots = None + if args and _enabled_accounts(provider) and not _requests_upstream_help(args): + if _proactive_pick_enabled(provider): + if stdin_data is _STDIN_UNSET: + stdin_data = _capture_stdin() + account_order, snapshots, notice = _proactive_execution_plan(provider, args) + if notice: + sys.stderr.write(notice) + if has_capacity_control(provider): + if stdin_data is _STDIN_UNSET: + stdin_data = _capture_stdin() + try: + decision = _admission(provider, args, stdin_data, snapshots=snapshots) + except (OSError, ValueError) as exc: + sys.stderr.write(f"[clifwrap] queue state error for {app}: {exc}\n") + return 74 + if decision.action == "queue": + item = decision.queue_item + sys.stderr.write(f"[clifwrap] deferred {app} {' '.join(args)}: {decision.reason}\n") + if item: + sys.stderr.write(f"[clifwrap] queued as {item.id}\n") + if decision.remediation_message: + sys.stderr.write(f"[clifwrap] {decision.remediation_message}\n") + return 73 + if decision.action == "shed": + sys.stderr.write(f"[clifwrap] blocked {app} {' '.join(args)}: {decision.reason}\n") + if decision.remediation_message: + sys.stderr.write(f"[clifwrap] {decision.remediation_message}\n") + return 69 + if decision.unknown_capacity: + sys.stderr.write(f"[clifwrap] capacity unknown for {app}; continuing because policy allows it\n") + elif decision.capacity_approved and decision.account_name: + allowed_account_names = {decision.account_name} + return _run_attempts( + app, + provider, + args, + stdin_data, + allowed_account_names=allowed_account_names, + account_order=account_order, + ) def _queue_item_pending_update(item: QueueItem, reason: str) -> QueueItem: @@ -792,16 +871,23 @@ def replay_queue(provider_name: str | None = None, *, item_id: str | None = None provider = _provider_for(item.provider, config) stdin_data = base64.b64decode(item.stdin_b64) if item.stdin_b64 else None allowed_account_names: set[str] | None = None - if provider.capacity_control and _enabled_accounts(provider): - decision = _admission(provider, item.argv, stdin_data, existing_item=item) - if decision.action != "execute": - result["status"] = "blocked" - result["reason"] = decision.reason - results.append(result) - exit_code = max(exit_code, 1) - continue - if decision.capacity_approved and decision.account_name: - allowed_account_names = {decision.account_name} + account_order: list[AccountConfig] | None = None + snapshots = None + if _enabled_accounts(provider): + if _proactive_pick_enabled(provider): + account_order, snapshots, notice = _proactive_execution_plan(provider, item.argv) + if notice and not json_output: + sys.stderr.write(notice) + if provider.capacity_control: + decision = _admission(provider, item.argv, stdin_data, existing_item=item, snapshots=snapshots) + if decision.action != "execute": + result["status"] = "blocked" + result["reason"] = decision.reason + results.append(result) + exit_code = max(exit_code, 1) + continue + if decision.capacity_approved and decision.account_name: + allowed_account_names = {decision.account_name} updated = _queue_item_pending_update(item, "replay started") replace_queue_item(updated) rc = _run_attempts( @@ -811,6 +897,7 @@ def replay_queue(provider_name: str | None = None, *, item_id: str | None = None stdin_data, emit_output=not json_output, allowed_account_names=allowed_account_names, + account_order=account_order, ) result["status"] = "executed" if rc == 0 else "failed" result["exit_code"] = rc @@ -977,6 +1064,7 @@ def _status_snapshot(app: str, provider: ProviderConfig) -> dict: payload = { "provider": app, "configured_accounts": len(provider.accounts), + "proactive_pick": provider.proactive_pick, "low_fallback": None, "recovery_error": None, "capacity_control": None, @@ -1012,6 +1100,8 @@ def _status_snapshot(app: str, provider: ProviderConfig) -> dict: "remediation_message": provider.capacity_control.remediation_message, "remediation_commands": provider.capacity_control.remediation_commands, } + reserve_threshold = provider.capacity_control.reserve_threshold if provider.capacity_control else 0 + estimated_cost = nominal_command_cost(provider) for account in provider.accounts: row = { "name": account.name, @@ -1019,22 +1109,46 @@ def _status_snapshot(app: str, provider: ProviderConfig) -> dict: "detail": None, "remaining": None, "error": None, + "starting_eligible": False, + "starting_ineligible_reason": None, } accounts.append(row) if not account.enabled: row["detail"] = "disabled" + eligible, reason = starting_account_eligibility( + None, + estimated_cost=estimated_cost, + reserve_threshold=reserve_threshold, + enabled=False, + ) + row["starting_eligible"] = eligible + row["starting_ineligible_reason"] = reason continue try: env = _account_env(account) except Exception as exc: row["detail"] = "configuration error" row["error"] = str(exc) + eligible, reason = starting_account_eligibility( + None, + estimated_cost=estimated_cost, + reserve_threshold=reserve_threshold, + ) + row["starting_eligible"] = eligible + row["starting_ineligible_reason"] = reason continue try: usage_status = _usage_status(provider, account, env) except Exception as exc: row["detail"] = "usage lookup failed" row["error"] = str(exc) + eligible, reason = starting_account_eligibility( + None, + estimated_cost=estimated_cost, + reserve_threshold=reserve_threshold, + ) + row["starting_eligible"] = eligible + row["starting_ineligible_reason"] = reason continue if usage_status: detail, remaining = usage_status @@ -1042,6 +1156,13 @@ def _status_snapshot(app: str, provider: ProviderConfig) -> dict: detail, remaining = _generic_status(provider, account, env) row["detail"] = detail row["remaining"] = remaining + eligible, reason = starting_account_eligibility( + remaining, + estimated_cost=estimated_cost, + reserve_threshold=reserve_threshold, + ) + row["starting_eligible"] = eligible + row["starting_ineligible_reason"] = reason if remaining is not None: remaining_values.append(remaining) if remaining_values: diff --git a/src/clifwrap/scheduling.py b/src/clifwrap/scheduling.py index 3cb79de..ab48a8c 100644 --- a/src/clifwrap/scheduling.py +++ b/src/clifwrap/scheduling.py @@ -22,6 +22,14 @@ def known(self) -> bool: return self.remaining is not None and self.error is None +@dataclass +class StartingAccountSelection: + account_name: str | None + reason: str + used_lazy_default: bool = False + meets_cost_floor: bool = False + + @dataclass class AdmissionDecision: action: str @@ -53,6 +61,87 @@ def estimate_command_cost(provider: ProviderConfig, args: list[str]) -> int: return capacity.default_cost +def nominal_command_cost(provider: ProviderConfig, args: list[str] | None = None) -> int: + if provider.capacity_control: + return estimate_command_cost(provider, args or []) + return 1 + + +def starting_account_eligibility( + remaining: int | None, + *, + estimated_cost: int, + reserve_threshold: int, + enabled: bool = True, +) -> tuple[bool, str | None]: + if not enabled: + return False, "disabled" + preferred_remaining = estimated_cost + reserve_threshold + if remaining is None: + return False, "capacity unknown" + if remaining >= preferred_remaining: + return True, None + if remaining >= estimated_cost: + return False, f"remaining {remaining} below preferred {preferred_remaining}" + return False, f"remaining {remaining} below cost floor {estimated_cost}" + + +def select_starting_account( + snapshots: list[CapacitySnapshot], + *, + active_account_name: str | None, + estimated_cost: int, + reserve_threshold: int, +) -> StartingAccountSelection: + preferred_remaining = estimated_cost + reserve_threshold + named = {snapshot.account_name: snapshot for snapshot in snapshots} + + if active_account_name and active_account_name in named: + active_snapshot = named[active_account_name] + if ( + active_snapshot.known + and active_snapshot.remaining is not None + and active_snapshot.remaining >= preferred_remaining + ): + return StartingAccountSelection( + active_account_name, + f"{active_account_name} has remaining {active_snapshot.remaining} >= preferred {preferred_remaining}", + used_lazy_default=True, + meets_cost_floor=True, + ) + + best_preferred: CapacitySnapshot | None = None + for snapshot in snapshots: + if snapshot.known and snapshot.remaining is not None and snapshot.remaining >= preferred_remaining: + if best_preferred is None or snapshot.remaining > best_preferred.remaining: + best_preferred = snapshot + if best_preferred is not None: + return StartingAccountSelection( + best_preferred.account_name, + f"{best_preferred.account_name} selected with remaining {best_preferred.remaining} >= preferred {preferred_remaining}", + meets_cost_floor=True, + ) + + best_affordable: CapacitySnapshot | None = None + for snapshot in snapshots: + if snapshot.known and snapshot.remaining is not None and snapshot.remaining >= estimated_cost: + if best_affordable is None or snapshot.remaining > best_affordable.remaining: + best_affordable = snapshot + if best_affordable is not None: + return StartingAccountSelection( + best_affordable.account_name, + f"{best_affordable.account_name} selected with remaining {best_affordable.remaining} >= cost {estimated_cost} (below soft reserve {preferred_remaining})", + meets_cost_floor=True, + ) + + fallback = active_account_name or (snapshots[0].account_name if snapshots else None) + if snapshots and all(not snapshot.known for snapshot in snapshots): + reason = "capacity unknown; starting on default account" + else: + reason = f"no account met preferred {preferred_remaining}; starting on {fallback}" + return StartingAccountSelection(fallback, reason) + + def _policy_snapshot(capacity: CapacityControlConfig, estimated_cost: int) -> dict[str, object]: return { "default_action": capacity.default_action, @@ -193,10 +282,6 @@ def admission_decision( if not capacity: return AdmissionDecision(action="execute", reason="capacity control disabled", estimated_cost=0, reserve_threshold=0) estimated_cost = estimate_command_cost(provider, args) - # Hard floor: enough credits to pay for this command. - # Soft reserve: prefer accounts that still have headroom above the floor. - preferred_remaining = estimated_cost + capacity.reserve_threshold - named = {snapshot.account_name: snapshot for snapshot in snapshots} def _execute(account_name: str, reason: str) -> AdmissionDecision: return AdmissionDecision( @@ -208,37 +293,14 @@ def _execute(account_name: str, reason: str) -> AdmissionDecision: capacity_approved=True, ) - if active_account_name and active_account_name in named: - active_snapshot = named[active_account_name] - if active_snapshot.known and active_snapshot.remaining is not None: - if active_snapshot.remaining >= preferred_remaining: - return _execute( - active_account_name, - f"{active_account_name} has remaining {active_snapshot.remaining} >= preferred {preferred_remaining}", - ) - - for snapshot in snapshots: - if snapshot.known and snapshot.remaining is not None and snapshot.remaining >= preferred_remaining: - return _execute( - snapshot.account_name, - f"{snapshot.account_name} selected with remaining {snapshot.remaining} >= preferred {preferred_remaining}", - ) - - # Fall back to any account that can afford the command even if below soft reserve. - if active_account_name and active_account_name in named: - active_snapshot = named[active_account_name] - if active_snapshot.known and active_snapshot.remaining is not None and active_snapshot.remaining >= estimated_cost: - return _execute( - active_account_name, - f"{active_account_name} has remaining {active_snapshot.remaining} >= cost {estimated_cost} (below soft reserve {preferred_remaining})", - ) - - for snapshot in snapshots: - if snapshot.known and snapshot.remaining is not None and snapshot.remaining >= estimated_cost: - return _execute( - snapshot.account_name, - f"{snapshot.account_name} selected with remaining {snapshot.remaining} >= cost {estimated_cost} (below soft reserve {preferred_remaining})", - ) + selection = select_starting_account( + snapshots, + active_account_name=active_account_name, + estimated_cost=estimated_cost, + reserve_threshold=capacity.reserve_threshold, + ) + if selection.meets_cost_floor and selection.account_name: + return _execute(selection.account_name, selection.reason) if snapshots and all(not snapshot.known for snapshot in snapshots): if capacity.unknown_capacity_action == "allow": diff --git a/src/clifwrap/validation.py b/src/clifwrap/validation.py new file mode 100644 index 0000000..ef889c5 --- /dev/null +++ b/src/clifwrap/validation.py @@ -0,0 +1,388 @@ +from __future__ import annotations + +import json +import os +import shlex +import subprocess +import urllib.error +import urllib.request +from dataclasses import dataclass +from pathlib import Path + +from .config import AccountConfig, ProviderConfig +from .state import get_default_account + + +@dataclass(frozen=True) +class ValidationFinding: + provider: str + field: str + level: str + message: str + remediation: str | None = None + + +def _load_env_file(path: str) -> dict[str, str]: + resolved_path = Path(os.path.expandvars(os.path.expanduser(path))) + values: dict[str, str] = {} + for line_number, raw_line in enumerate(resolved_path.read_text().splitlines(), start=1): + line = raw_line.strip() + if not line or line.startswith("#"): + continue + if line.startswith("export "): + line = line[len("export ") :].strip() + if "=" not in line: + raise ValueError(f"{resolved_path}:{line_number}: expected KEY=VALUE") + key, value = line.split("=", 1) + key = key.strip() + if not key: + raise ValueError(f"{resolved_path}:{line_number}: empty key") + parts = shlex.split(value, comments=False, posix=True) + values[key] = parts[0] if parts else "" + return values + + +def _env_lookup(name: str, account: AccountConfig) -> str | None: + source = dict(os.environ) + for env_file in account.env_files: + try: + source.update(_load_env_file(env_file)) + except (OSError, ValueError): + return None + return source.get(name) + + +def _resolve_env_refs(account: AccountConfig) -> list[ValidationFinding]: + findings: list[ValidationFinding] = [] + for key, value in account.env.items(): + if not value.startswith("env:"): + continue + env_name = value[4:] + if not env_name: + findings.append( + ValidationFinding( + provider="", + field=f"accounts.{account.name}.env.{key}", + level="error", + message=f"account {account.name!r} env ref for {key!r} is empty", + remediation=f"set {env_name} or use clifwrap account add with --env-ref {key}=VAR", + ) + ) + continue + if key in account.env_command: + continue + if _env_lookup(env_name, account) is None: + findings.append( + ValidationFinding( + provider="", + field=f"accounts.{account.name}.env.{key}", + level="error", + message=f"account {account.name!r} references unset env:{env_name}", + remediation=f"export {env_name} or add env_command for {key!r}", + ) + ) + return findings + + +def validate_provider_static(provider: ProviderConfig) -> list[ValidationFinding]: + findings: list[ValidationFinding] = [] + enabled = [account for account in provider.accounts if account.enabled] + if not enabled: + findings.append( + ValidationFinding( + provider=provider.name, + field="accounts", + level="error", + message=f"{provider.name}: no enabled accounts configured", + remediation=f"clifwrap account add {provider.name} --env-ref TOKEN=ENVVAR", + ) + ) + names: set[str] = set() + for account in provider.accounts: + if account.name in names: + findings.append( + ValidationFinding( + provider=provider.name, + field=f"accounts.{account.name}", + level="error", + message=f"{provider.name}: duplicate account name {account.name!r}", + remediation=f"clifwrap account rename {provider.name} {account.name} ", + ) + ) + names.add(account.name) + for finding in _resolve_env_refs(account): + findings.append( + ValidationFinding( + provider=provider.name, + field=finding.field, + level=finding.level, + message=finding.message, + remediation=finding.remediation, + ) + ) + for index, pattern in enumerate(provider.retry_patterns): + if not pattern.strip(): + findings.append( + ValidationFinding( + provider=provider.name, + field=f"retry_patterns[{index}]", + level="error", + message=f"{provider.name}: retry_patterns contains an empty entry", + remediation="remove empty retry_patterns entries from config.toml", + ) + ) + usage = provider.usage + if usage and not usage.url.startswith(("http://", "https://", "{")): + findings.append( + ValidationFinding( + provider=provider.name, + field="usage.url", + level="error", + message=f"{provider.name}: usage.url must be http(s) or a template", + remediation="fix providers..usage.url in config.toml", + ) + ) + return findings + + +def _safe_error(exc: Exception) -> str: + """Describe a probe failure without echoing argv or captured output. + + Probe commands are interpolated with resolved account credentials, and + subprocess exception strings embed the full argv, so stringifying them + directly would publish secrets into doctor output. + """ + if isinstance(exc, subprocess.TimeoutExpired): + return f"timed out after {exc.timeout:g}s" + if isinstance(exc, subprocess.CalledProcessError): + return f"exited with status {exc.returncode}" + if isinstance(exc, urllib.error.HTTPError): + return f"HTTP {exc.code}" + if isinstance(exc, urllib.error.URLError): + return str(exc.reason) + if isinstance(exc, OSError): + return exc.strerror or type(exc).__name__ + if isinstance(exc, ValueError): + return str(exc) + return type(exc).__name__ + + +def _render_url(template: str, env: dict[str, str]) -> str: + rendered = template + for key, value in env.items(): + rendered = rendered.replace(f"${{{key}}}", value) + while "{" in rendered and "}" in rendered: + start = rendered.find("{") + end = rendered.find("}", start) + if end == -1: + break + expression = rendered[start + 1 : end] + if ":" in expression: + key, fallback = expression.split(":", 1) + value = env.get(key, fallback) + else: + value = env.get(expression, "") + rendered = rendered[:start] + value + rendered[end + 1 :] + return rendered + + +def _account_env_for_probe(account: AccountConfig) -> dict[str, str]: + source = dict(os.environ) + for env_file in account.env_files: + source.update(_load_env_file(env_file)) + resolved: dict[str, str] = {} + for key, value in account.env.items(): + if value.startswith("env:"): + env_name = value[4:] + resolved_value = source.get(env_name) + if resolved_value is None: + raise ValueError(f"missing env:{env_name}") + resolved[key] = resolved_value + else: + resolved[key] = os.path.expandvars(value) + for key, command in account.env_command.items(): + proc = subprocess.run(command, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True) + resolved[key] = proc.stdout.strip() + return resolved + + +def _usage_probe(provider: ProviderConfig, env: dict[str, str]) -> list[ValidationFinding]: + usage = provider.usage + if not usage: + return [] + findings: list[ValidationFinding] = [] + url = _render_url(usage.url, env) + if not url.startswith(("http://", "https://")): + return [ + ValidationFinding( + provider=provider.name, + field="usage.url", + level="warning", + message=f"{provider.name}: usage URL did not resolve to http(s)", + remediation="set usage auth env vars or fix usage.url template", + ) + ] + token = env.get(usage.auth_env) or os.environ.get(usage.auth_env) + headers = {usage.auth_header: f"{usage.auth_scheme} {token}".strip()} + if usage.content_type: + headers["Content-Type"] = usage.content_type + try: + request = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(request, timeout=min(usage.timeout_seconds, 5.0)) as response: + payload = json.loads(response.read().decode("utf-8")) + except (urllib.error.URLError, TimeoutError, json.JSONDecodeError, ValueError) as exc: + return [ + ValidationFinding( + provider=provider.name, + field="usage", + level="warning", + message=f"{provider.name}: usage probe failed ({_safe_error(exc)})", + remediation=f"clifwrap doctor --provider {provider.name} --check", + ) + ] + if not isinstance(payload, dict): + return [ + ValidationFinding( + provider=provider.name, + field="usage", + level="warning", + message=f"{provider.name}: usage probe returned non-object JSON", + remediation="verify usage URL and auth_env for this provider", + ) + ] + return findings + + +def _status_command_probe(provider: ProviderConfig, env: dict[str, str]) -> list[ValidationFinding]: + if not provider.status_command: + return [] + command = [] + for part in provider.status_command: + rendered = part + for key, value in env.items(): + rendered = rendered.replace(f"${{{key}}}", value) + command.append(rendered) + try: + proc = subprocess.run( + command, + env={**os.environ, **env, "CLIFWRAP_BYPASS": "1"}, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + timeout=10, + ) + except (OSError, subprocess.TimeoutExpired) as exc: + return [ + ValidationFinding( + provider=provider.name, + field="status_command", + level="warning", + message=f"{provider.name}: status_command probe failed ({_safe_error(exc)})", + remediation="fix status_command or account env for this provider", + ) + ] + if proc.returncode != 0: + return [ + ValidationFinding( + provider=provider.name, + field="status_command", + level="warning", + message=f"{provider.name}: status_command exited with status {proc.returncode}", + remediation="verify status_command and credentials", + ) + ] + output = proc.stdout.strip() + if not output: + return [ + ValidationFinding( + provider=provider.name, + field="status_command", + level="warning", + message=f"{provider.name}: status_command returned no output", + remediation="ensure status_command prints JSON with remaining/limit", + ) + ] + try: + payload = json.loads(output) + except json.JSONDecodeError: + return [ + ValidationFinding( + provider=provider.name, + field="status_command", + level="warning", + message=f"{provider.name}: status_command output is not JSON", + remediation="configure status_command to emit JSON usage payload", + ) + ] + if not isinstance(payload, dict): + return [ + ValidationFinding( + provider=provider.name, + field="status_command", + level="warning", + message=f"{provider.name}: status_command JSON is not an object", + remediation="configure status_command to emit a JSON object", + ) + ] + keys = set(payload.keys()) + if not keys.intersection({"remaining", "remainingCredits", "limit", "planCredits", "used", "usage"}): + return [ + ValidationFinding( + provider=provider.name, + field="status_command", + level="warning", + message=f"{provider.name}: status_command JSON lacks remaining/limit fields", + remediation="include remaining and limit fields in status_command output", + ) + ] + return [] + + +def probe_provider(provider: ProviderConfig, account: AccountConfig) -> list[ValidationFinding]: + try: + env = _account_env_for_probe(account) + except (OSError, ValueError, subprocess.CalledProcessError) as exc: + return [ + ValidationFinding( + provider=provider.name, + field=f"accounts.{account.name}", + level="warning", + message=f"{provider.name}: could not resolve account {account.name!r} env ({_safe_error(exc)})", + remediation="clifwrap config validate --json", + ) + ] + findings: list[ValidationFinding] = [] + if provider.usage: + findings.extend(_usage_probe(provider, env)) + if provider.status_command: + findings.extend(_status_command_probe(provider, env)) + return findings + + +def probe_accounts(provider: ProviderConfig, *, all_accounts: bool = False) -> list[ValidationFinding]: + enabled = [account for account in provider.accounts if account.enabled] + if not enabled: + return [] + if all_accounts: + targets = enabled + else: + default_name = get_default_account(provider.name) + default = next((account for account in enabled if account.name == default_name), None) + targets = [default or enabled[0]] + findings: list[ValidationFinding] = [] + for account in targets: + findings.extend(probe_provider(provider, account)) + return findings + + +def findings_to_dict(findings: list[ValidationFinding]) -> list[dict[str, str | None]]: + return [ + { + "provider": finding.provider, + "field": finding.field, + "level": finding.level, + "message": finding.message, + "remediation": finding.remediation, + } + for finding in findings + ] diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py index 5fbe8c7..bda20d1 100644 --- a/tests/test_wrapper.py +++ b/tests/test_wrapper.py @@ -568,7 +568,7 @@ def test_doctor_reports_paths_providers_shims_and_queue(self) -> None: """\ [[providers.somecli.accounts]] name = "primary" - env = { SOMECLI_TOKEN = "env:SOMECLI_TOKEN_PRIMARY" } + env = { SOMECLI_TOKEN = "primary-token" } """ ) ) @@ -3265,6 +3265,350 @@ def test_builtin_searchcli_search_cost_is_not_crawl_scale(self) -> None: 50, ) + def test_merged_provider_appends_retry_patterns_by_default(self) -> None: + from clifwrap.config import ProviderConfig, merged_provider + + raw = ProviderConfig(name="searchcli", retry_patterns=["custom pattern"]) + merged = merged_provider("searchcli", raw) + self.assertIn("usage limit", merged.retry_patterns) + self.assertIn("custom pattern", merged.retry_patterns) + + def test_merged_provider_replace_retry_patterns(self) -> None: + from clifwrap.config import ProviderConfig, merged_provider + + raw = ProviderConfig(name="searchcli", retry_patterns_replace=["only this"]) + merged = merged_provider("searchcli", raw) + self.assertEqual(merged.retry_patterns, ["only this"]) + self.assertNotIn("usage limit", merged.retry_patterns) + + def test_config_validate_reports_missing_env_ref(self) -> None: + (self.config_dir / "config.toml").write_text( + textwrap.dedent( + """\ + [[providers.somecli.accounts]] + name = "team" + env = { SOMECLI_TOKEN = "env:MISSING_ENV_REF" } + """ + ) + ) + proc = self._run("config", "validate", "--json") + self.assertEqual(proc.returncode, 1) + payload = json.loads(proc.stdout) + self.assertFalse(payload["valid"]) + self.assertGreaterEqual(payload["errors"], 1) + self.assertIn("MISSING_ENV_REF", proc.stdout) + self.assertNotIn("secret-value", proc.stdout) + + def test_doctor_validation_warns_without_check_on_unreachable_usage(self) -> None: + (self.config_dir / "config.toml").write_text( + textwrap.dedent( + """\ + [providers.somecli.usage] + url = "http://127.0.0.1:1/usage" + auth_env = "SOMECLI_TOKEN" + + [[providers.somecli.accounts]] + name = "team" + env = { SOMECLI_TOKEN = "token" } + """ + ) + ) + proc = self._run("doctor", "--probe", "--json") + self.assertEqual(proc.returncode, 0) + payload = json.loads(proc.stdout) + validation = payload.get("validation") or [] + self.assertTrue( + any( + item.get("level") == "warning" and item.get("field") == "usage" + for item in validation + ), + validation, + ) + + def test_doctor_check_fails_on_validation_warnings(self) -> None: + (self.config_dir / "config.toml").write_text( + textwrap.dedent( + """\ + [providers.somecli.usage] + url = "http://127.0.0.1:1/usage" + auth_env = "SOMECLI_TOKEN" + + [[providers.somecli.accounts]] + name = "team" + env = { SOMECLI_TOKEN = "token" } + """ + ) + ) + proc = self._run("doctor", "--probe", "--check", "--json") + self.assertEqual(proc.returncode, 1) + payload = json.loads(proc.stdout) + self.assertGreaterEqual(payload["validation_warnings"], 1) + + def test_proactive_pick_defaults_true_with_status_command(self) -> None: + from clifwrap.config import ProviderConfig, merged_provider + + raw = ProviderConfig( + name="somecli", + status_command=["somecli", "usage", "--json"], + ) + merged = merged_provider("somecli", raw) + self.assertTrue(merged.proactive_pick) + + def test_proactive_pick_user_opt_out(self) -> None: + from clifwrap.config import ProviderConfig, merged_provider + + raw = ProviderConfig( + name="somecli", + status_command=["somecli", "usage", "--json"], + proactive_pick=False, + ) + merged = merged_provider("somecli", raw) + self.assertFalse(merged.proactive_pick) + + def test_proactive_pick_false_without_status_or_usage(self) -> None: + from clifwrap.config import ProviderConfig, merged_provider + + merged = merged_provider("somecli", ProviderConfig(name="somecli")) + self.assertFalse(merged.proactive_pick) + + def test_select_starting_account_picks_backup_when_default_depleted(self) -> None: + from clifwrap.scheduling import CapacitySnapshot, select_starting_account + + snapshots = [ + CapacitySnapshot("default", 1, "ok", 0), + CapacitySnapshot("backup", 50, "ok", 0), + ] + selection = select_starting_account( + snapshots, + active_account_name="default", + estimated_cost=1, + reserve_threshold=5, + ) + self.assertEqual(selection.account_name, "backup") + self.assertTrue(selection.meets_cost_floor) + + def test_select_starting_account_keeps_healthy_default(self) -> None: + from clifwrap.scheduling import CapacitySnapshot, select_starting_account + + snapshots = [ + CapacitySnapshot("default", 100, "ok", 0), + CapacitySnapshot("backup", 200, "ok", 0), + ] + selection = select_starting_account( + snapshots, + active_account_name="default", + estimated_cost=1, + reserve_threshold=5, + ) + self.assertEqual(selection.account_name, "default") + self.assertTrue(selection.used_lazy_default) + + def test_select_starting_account_unknown_capacity(self) -> None: + from clifwrap.scheduling import CapacitySnapshot, select_starting_account + + snapshots = [ + CapacitySnapshot("default", None, "unknown", 0, error="fail"), + ] + selection = select_starting_account( + snapshots, + active_account_name="default", + estimated_cost=1, + reserve_threshold=5, + ) + self.assertEqual(selection.account_name, "default") + self.assertFalse(selection.meets_cost_floor) + self.assertIn("capacity unknown", selection.reason) + + def test_select_starting_account_config_order_tiebreak(self) -> None: + from clifwrap.scheduling import CapacitySnapshot, select_starting_account + + snapshots = [ + CapacitySnapshot("first", 50, "ok", 0), + CapacitySnapshot("second", 50, "ok", 0), + ] + selection = select_starting_account( + snapshots, + active_account_name="first", + estimated_cost=1, + reserve_threshold=5, + ) + self.assertEqual(selection.account_name, "first") + + def test_proactive_pick_skips_depleted_default_without_failover(self) -> None: + target = self.bin_dir / "somecli" + make_executable( + target, + "#!/usr/bin/env python3\nimport os\nprint(os.environ.get('CLIFWRAP_ACCOUNT'))\n", + ) + self._run("install", "somecli") + (self.config_dir / "config.toml").write_text( + textwrap.dedent( + f"""\ + [providers.somecli] + status_command = ["{sys.executable}", "-c", "import os,json; print(json.dumps({{'remaining': int(os.environ['LEFT'])}}))"] + + [[providers.somecli.accounts]] + name = "first" + env = {{ LEFT = "0" }} + + [[providers.somecli.accounts]] + name = "second" + env = {{ LEFT = "20" }} + """ + ) + ) + self._run("account", "use", "somecli", "first") + proc = subprocess.run( + ["somecli", "search"], + cwd=ROOT, + env=self.env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(proc.stdout.strip(), "second") + self.assertNotIn("failover", proc.stderr.lower()) + self.assertIn("starting on second", proc.stderr) + + def test_status_json_includes_starting_eligibility(self) -> None: + (self.config_dir / "config.toml").write_text( + textwrap.dedent( + f"""\ + [providers.somecli] + status_command = ["{sys.executable}", "-c", "import os,json; print(json.dumps({{'remaining': int(os.environ['LEFT'])}}))"] + + [providers.somecli.capacity_control] + reserve_threshold = 5 + default_cost = 1 + + [[providers.somecli.accounts]] + name = "low" + env = {{ LEFT = "3" }} + + [[providers.somecli.accounts]] + name = "high" + env = {{ LEFT = "20" }} + """ + ) + ) + proc = self._run("status", "somecli", "--json") + self.assertEqual(proc.returncode, 0, proc.stderr) + payload = json.loads(proc.stdout) + self.assertTrue(payload["proactive_pick"]) + by_name = {row["name"]: row for row in payload["accounts"]} + self.assertFalse(by_name["low"]["starting_eligible"]) + self.assertIn("below preferred", by_name["low"]["starting_ineligible_reason"]) + self.assertTrue(by_name["high"]["starting_eligible"]) + self.assertIsNone(by_name["high"]["starting_ineligible_reason"]) + + def test_proactive_pick_opt_out_keeps_depleted_default_at_runtime(self) -> None: + target = self.bin_dir / "somecli" + make_executable( + target, + "#!/usr/bin/env python3\nimport os\nprint(os.environ.get('CLIFWRAP_ACCOUNT'))\n", + ) + self._run("install", "somecli") + (self.config_dir / "config.toml").write_text( + textwrap.dedent( + f"""\ + [providers.somecli] + proactive_pick = false + status_command = ["{sys.executable}", "-c", "import os,json; print(json.dumps({{'remaining': int(os.environ['LEFT'])}}))"] + + [[providers.somecli.accounts]] + name = "first" + env = {{ LEFT = "0" }} + + [[providers.somecli.accounts]] + name = "second" + env = {{ LEFT = "20" }} + """ + ) + ) + self._run("account", "use", "somecli", "first") + proc = subprocess.run( + ["somecli", "search"], + cwd=ROOT, + env=self.env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(proc.stdout.strip(), "first") + self.assertNotIn("starting on", proc.stderr) + + def test_doctor_does_not_run_live_probes_without_probe_flag(self) -> None: + marker = Path(self.temp_dir.name) / "probe-ran.txt" + (self.config_dir / "config.toml").write_text( + textwrap.dedent( + f"""\ + [providers.somecli] + status_command = ["{sys.executable}", "-c", "from pathlib import Path; Path({str(marker)!r}).write_text('probed'); print('{{}}')"] + + [[providers.somecli.accounts]] + name = "only" + env = {{ SOMECLI_TOKEN = "token" }} + """ + ) + ) + proc = self._run("doctor", "--json") + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertFalse(marker.exists(), "doctor ran a live probe without --probe") + + proc = self._run("doctor", "--probe", "--json") + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertTrue(marker.exists(), "doctor --probe did not run the status_command probe") + + def test_doctor_probe_does_not_leak_status_command_secrets(self) -> None: + secret = "supersecrettoken123" + (self.config_dir / "config.toml").write_text( + textwrap.dedent( + f"""\ + [providers.somecli] + status_command = ["{sys.executable}", "-c", "import os,sys; print(os.environ['SOMECLI_TOKEN'], file=sys.stderr); raise SystemExit(1)"] + + [[providers.somecli.accounts]] + name = "only" + env = {{ SOMECLI_TOKEN = "{secret}" }} + """ + ) + ) + proc = self._run("doctor", "--probe", "--json") + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertNotIn(secret, proc.stdout) + self.assertNotIn(secret, proc.stderr) + payload = json.loads(proc.stdout) + validation = payload.get("validation") or [] + self.assertTrue( + any(item.get("field") == "status_command" for item in validation), + validation, + ) + + def test_replace_keys_apply_to_provider_without_catalog_entry(self) -> None: + (self.config_dir / "config.toml").write_text( + textwrap.dedent( + """\ + [providers.somecli] + retry_patterns_replace = ["only this"] + retry_exit_codes_replace = [42] + passthrough_commands_replace = ["login"] + + [[providers.somecli.accounts]] + name = "only" + """ + ) + ) + env = self.env | {"CLIFWRAP_CONFIG": str(self.config_dir / "config.toml")} + with mock.patch.dict(os.environ, env, clear=True): + from clifwrap.config import load_config, merged_provider + + merged = merged_provider("somecli", load_config().providers["somecli"]) + self.assertEqual(merged.retry_patterns, ["only this"]) + self.assertEqual(merged.retry_exit_codes, [42]) + self.assertEqual(merged.passthrough_commands, ["login"]) + if __name__ == "__main__": unittest.main() From 25d9e314935cb5a0f0f992cb0648898b44c62656 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 25 Jul 2026 04:33:18 -0500 Subject: [PATCH 2/3] feat: wrap Cursor Agent CLI with tty-exec and print-mode failover Add catalog providers for `agent` and legacy `cursor-agent` matching Cursor's install layout and CLI syntax (interactive TUI, `agent -p` print mode, CURSOR_API_KEY auth, login/status passthrough). Introduce interactive_mode=tty-exec so full-screen TUIs keep a live terminal instead of being buffered; headless `-p`/`--print` runs still get retry failover across API-key accounts. --- README.md | 32 +++++++++++ docs/configuration.md | 2 +- docs/provider-authoring.md | 9 +++ docs/provider-catalog.md | 38 +++++++++++++ src/clifwrap/providers.toml | 108 ++++++++++++++++++++++++++++++++++++ src/clifwrap/runtime.py | 91 +++++++++++++++++++++++++++++- tests/test_wrapper.py | 100 +++++++++++++++++++++++++++++++++ 7 files changed, 376 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index b8293bf..ac92cac 100644 --- a/README.md +++ b/README.md @@ -88,6 +88,38 @@ env = { SCRAPECLI_API_KEY = "env:SCRAPECLI_TEAM_B_KEY" } Retry rules, auth command names, and usage endpoints for SearchCLI and ScrapeCLI ship in `src/clifwrap/providers.toml`. Your config usually only needs account entries. +### Cursor Agent CLI + +[Cursor Agent](https://cursor.com/docs/cli/overview) installs as `agent` (and legacy `cursor-agent`). Auth for automation uses `CURSOR_API_KEY` ([docs](https://cursor.com/docs/cli/reference/authentication)). + +```bash +clifwrap init +clifwrap account add agent primary --env-ref CURSOR_API_KEY=CURSOR_API_KEY_PRIMARY +clifwrap account add agent backup --env-ref CURSOR_API_KEY=CURSOR_API_KEY_BACKUP +clifwrap install agent +# If another tool already owns `agent` on PATH, wrap the legacy name instead: +# clifwrap account add cursor-agent primary --env-ref CURSOR_API_KEY=CURSOR_API_KEY_PRIMARY +# clifwrap install cursor-agent +``` + +```toml +[[providers.agent.accounts]] +name = "primary" +env = { CURSOR_API_KEY = "env:CURSOR_API_KEY_PRIMARY" } + +[[providers.agent.accounts]] +name = "backup" +env = { CURSOR_API_KEY = "env:CURSOR_API_KEY_BACKUP" } +``` + +Behavior matches Cursor CLI syntax: + +- Interactive TUI (`agent`, `agent "prompt"`) uses `tty-exec` — picks an account, then execs the real binary with a live terminal (no output buffering). +- Headless print mode (`agent -p "..."`, `--print`, optional `--force` / `--output-format`) gets buffered failover on rate-limit / auth errors. +- `agent login`, `logout`, `status`, `about`, `models`, `mcp`, … pass through unchanged. + +`agent status --format json` reports auth identity, not remaining quota, so proactive pick stays off unless you add your own `status_command` / `usage` metadata. + See [docs/configuration.md](docs/configuration.md) for the full reference. ### Capacity control diff --git a/docs/configuration.md b/docs/configuration.md index 4678f08..9927382 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -45,7 +45,7 @@ Provider tables can set: | `retry_patterns` | Lowercased stderr/stdout snippets that trigger retry | | `never_retry_patterns` | Snippets that block retry even if another rule matches | | `retry_on_any_error` | Retry any nonzero exit unless blocked by `never_retry_patterns` | -| `interactive_mode` | Currently `line-repl` for line-by-line wrapped shells | +| `interactive_mode` | `line-repl` for line-by-line shells; `tty-exec` for full-screen TUIs (e.g. Cursor Agent) | | `status_command` | Command that returns JSON with `remaining`, `limit`, and optionally `used` | | `proactive_pick` | Headroom-first starting account before the first upstream call (default `true` when `status_command` or `usage` is set) | | `passthrough_commands` | Upstream subcommands that skip wrapper auth handling | diff --git a/docs/provider-authoring.md b/docs/provider-authoring.md index 35ada09..a7b15f2 100644 --- a/docs/provider-authoring.md +++ b/docs/provider-authoring.md @@ -52,6 +52,15 @@ Probe findings never echo command output or argv — a failing `status_command` 4. Add integration tests in `tests/test_wrapper.py` using a generic `somecli` fixture where possible. 5. Do not add provider name literals to generic modules (`config.py`, `runtime.py`, etc.). +## Cursor Agent notes + +Cursor Agent is a full-screen TUI. Do not use `interactive_mode = "line-repl"` for it — that mode is for line-oriented shells like SearchCLI. Catalog entry `agent` / `cursor-agent` uses `tty-exec`: + +- Interactive sessions: select account → `exec` the real binary (TTY preserved). +- `agent -p` / `--print`: buffered run with retry/failover across accounts. + +Passthrough covers Cursor management commands (`login`, `logout`, `status`, `mcp`, …) so browser login and MCP setup stay upstream. + ## Related docs - [configuration.md](configuration.md) — field reference and merge semantics diff --git a/docs/provider-catalog.md b/docs/provider-catalog.md index 633513d..01f5b54 100644 --- a/docs/provider-catalog.md +++ b/docs/provider-catalog.md @@ -3,6 +3,44 @@ This file is generated from `src/clifwrap/providers.toml`. Run `python scripts/generate_provider_catalog.py --write` after changing built-in provider metadata. +## `agent` + +| Field | Value | +| --- | --- | +| `interactive_mode` | `tty-exec` | +| `passthrough_commands` | `login`, `logout`, `status`, `whoami`, `about`, `models`, `update`, `mcp`, `plugin`, `worker`, `ls`, `resume`, `create-chat`, `generate-rule`, `rule`, `install-shell-integration`, `uninstall-shell-integration`, `help` | +| `retry_patterns` | `rate limit`, `too many requests`, `429`, `usage limit`, `out of usage`, `spend limit`, `quota`, `not authenticated`, `unauthorized`, `authentication required`, `invalid api key`, `resource_exhausted` | +| `never_retry_patterns` | `unknown option`, `unknown command`, `missing required argument`, `invalid argument` | + +### Fallback Monitor + +| Field | Value | +| --- | --- | +| `threshold` | `2` | +| `action` | `warn` | +| `journald` | false | +| `syslog` | true | +| `stderr` | true | + +## `cursor-agent` + +| Field | Value | +| --- | --- | +| `interactive_mode` | `tty-exec` | +| `passthrough_commands` | `login`, `logout`, `status`, `whoami`, `about`, `models`, `update`, `mcp`, `plugin`, `worker`, `ls`, `resume`, `create-chat`, `generate-rule`, `rule`, `install-shell-integration`, `uninstall-shell-integration`, `help` | +| `retry_patterns` | `rate limit`, `too many requests`, `429`, `usage limit`, `out of usage`, `spend limit`, `quota`, `not authenticated`, `unauthorized`, `authentication required`, `invalid api key`, `resource_exhausted` | +| `never_retry_patterns` | `unknown option`, `unknown command`, `missing required argument`, `invalid argument` | + +### Fallback Monitor + +| Field | Value | +| --- | --- | +| `threshold` | `2` | +| `action` | `warn` | +| `journald` | false | +| `syslog` | true | +| `stderr` | true | + ## `examplecli` | Field | Value | diff --git a/src/clifwrap/providers.toml b/src/clifwrap/providers.toml index bd1c6bb..1e2a2b6 100644 --- a/src/clifwrap/providers.toml +++ b/src/clifwrap/providers.toml @@ -158,3 +158,111 @@ remediation_commands = [ "clifwrap account list examplecli", "clifwrap config validate", ] + +# Cursor Agent CLI (https://cursor.com/docs/cli/overview). +# Official install creates both `agent` (primary) and `cursor-agent` (legacy) PATH names. +# Prefer wrapping the name that resolves on your PATH. Auth is CURSOR_API_KEY per account +# (https://cursor.com/docs/cli/reference/authentication). Interactive TUI uses tty-exec; +# buffered failover applies to `agent -p` / `--print` headless runs. +[providers.agent] +interactive_mode = "tty-exec" +passthrough_commands = [ + "login", + "logout", + "status", + "whoami", + "about", + "models", + "update", + "mcp", + "plugin", + "worker", + "ls", + "resume", + "create-chat", + "generate-rule", + "rule", + "install-shell-integration", + "uninstall-shell-integration", + "help", +] +retry_patterns = [ + "rate limit", + "too many requests", + "429", + "usage limit", + "out of usage", + "spend limit", + "quota", + "not authenticated", + "unauthorized", + "authentication required", + "invalid api key", + "resource_exhausted", +] +never_retry_patterns = [ + "unknown option", + "unknown command", + "missing required argument", + "invalid argument", +] +proactive_pick = false + +[providers.agent.fallback_monitor] +threshold = 2 +action = "warn" +journald = false +syslog = true +stderr = true + +# Legacy PATH name created by the Cursor Agent installer alongside `agent`. +[providers.cursor-agent] +interactive_mode = "tty-exec" +passthrough_commands = [ + "login", + "logout", + "status", + "whoami", + "about", + "models", + "update", + "mcp", + "plugin", + "worker", + "ls", + "resume", + "create-chat", + "generate-rule", + "rule", + "install-shell-integration", + "uninstall-shell-integration", + "help", +] +retry_patterns = [ + "rate limit", + "too many requests", + "429", + "usage limit", + "out of usage", + "spend limit", + "quota", + "not authenticated", + "unauthorized", + "authentication required", + "invalid api key", + "resource_exhausted", +] +never_retry_patterns = [ + "unknown option", + "unknown command", + "missing required argument", + "invalid argument", +] +proactive_pick = false + +[providers.cursor-agent.fallback_monitor] +threshold = 2 +action = "warn" +journald = false +syslog = true +stderr = true diff --git a/src/clifwrap/runtime.py b/src/clifwrap/runtime.py index e47a6f5..646a0ea 100644 --- a/src/clifwrap/runtime.py +++ b/src/clifwrap/runtime.py @@ -756,6 +756,61 @@ def _handle_managed_auth(app: str, provider: ProviderConfig, args: list[str]) -> return None +def _is_print_mode(args: list[str]) -> bool: + return "-p" in args or "--print" in args + + +def _wants_tty_exec(provider: ProviderConfig, args: list[str]) -> bool: + if provider.interactive_mode != "tty-exec": + return False + if not (sys.stdin.isatty() and sys.stdout.isatty()): + return False + # Headless/print mode keeps buffered failover; interactive TUIs must not be piped. + return not _is_print_mode(args) + + +def _exec_with_account(app: str, provider: ProviderConfig, args: list[str], account: AccountConfig) -> int: + command = _original_command(app, provider) + env = dict(os.environ) + try: + env.update(_account_env(account)) + except Exception as exc: + sys.stderr.write(f"[clifwrap] {account.name} configuration error: {exc}\n") + return 1 + env[BYPASS_ENV] = "1" + env[CURRENT_ACCOUNT_ENV] = account.name + try: + _prepare_account(provider, account, env) + except Exception as exc: + sys.stderr.write(f"[clifwrap] {account.name} preparation error: {exc}\n") + return 1 + set_default_account(provider.name, account.name) + _exec_original(app, command, args, env) + return 1 + + +def _run_tty_exec( + app: str, + provider: ProviderConfig, + args: list[str], + *, + account_order: list[AccountConfig] | None = None, + allowed_account_names: set[str] | None = None, +) -> int: + if account_order is not None: + accounts = account_order + if allowed_account_names is not None: + accounts = [account for account in accounts if account.name in allowed_account_names] + else: + accounts = _ordered_accounts(provider, allowed_account_names) + if not accounts: + command = _original_command(app, provider) + env = dict(os.environ) + env[BYPASS_ENV] = "1" + _exec_original(app, command, args, env) + return _exec_with_account(app, provider, args, accounts[0]) + + def run_app(app: str, args: list[str]) -> int: if os.environ.get(BYPASS_ENV) == "1": command = original_command_for(app) @@ -782,14 +837,17 @@ def run_app(app: str, args: list[str]) -> int: allowed_account_names: set[str] | None = None account_order: list[AccountConfig] | None = None snapshots = None - if args and _enabled_accounts(provider) and not _requests_upstream_help(args): + tty_exec = _wants_tty_exec(provider, args) + # Interactive tty-exec may run with empty args; print/non-TTY always have args for capacity. + should_plan = _enabled_accounts(provider) and not _requests_upstream_help(args) and (args or tty_exec) + if should_plan: if _proactive_pick_enabled(provider): - if stdin_data is _STDIN_UNSET: + if stdin_data is _STDIN_UNSET and not tty_exec: stdin_data = _capture_stdin() account_order, snapshots, notice = _proactive_execution_plan(provider, args) if notice: sys.stderr.write(notice) - if has_capacity_control(provider): + if has_capacity_control(provider) and args and not tty_exec: if stdin_data is _STDIN_UNSET: stdin_data = _capture_stdin() try: @@ -814,6 +872,33 @@ def run_app(app: str, args: list[str]) -> int: sys.stderr.write(f"[clifwrap] capacity unknown for {app}; continuing because policy allows it\n") elif decision.capacity_approved and decision.account_name: allowed_account_names = {decision.account_name} + elif has_capacity_control(provider) and tty_exec: + # Interactive sessions cannot be queued mid-TUI; still honor shed by probing. + try: + decision = _admission(provider, args, None, snapshots=snapshots) + except (OSError, ValueError) as exc: + sys.stderr.write(f"[clifwrap] queue state error for {app}: {exc}\n") + return 74 + if decision.action == "shed": + sys.stderr.write(f"[clifwrap] blocked {app}: {decision.reason}\n") + if decision.remediation_message: + sys.stderr.write(f"[clifwrap] {decision.remediation_message}\n") + return 69 + if decision.action == "queue": + sys.stderr.write( + f"[clifwrap] capacity low for interactive {app}; starting anyway " + f"(queue only applies to non-interactive print mode)\n" + ) + elif decision.capacity_approved and decision.account_name: + allowed_account_names = {decision.account_name} + if tty_exec: + return _run_tty_exec( + app, + provider, + args, + account_order=account_order, + allowed_account_names=allowed_account_names, + ) return _run_attempts( app, provider, diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py index bda20d1..78b5ceb 100644 --- a/tests/test_wrapper.py +++ b/tests/test_wrapper.py @@ -3609,6 +3609,106 @@ def test_replace_keys_apply_to_provider_without_catalog_entry(self) -> None: self.assertEqual(merged.retry_exit_codes, [42]) self.assertEqual(merged.passthrough_commands, ["login"]) + def test_agent_catalog_provider_is_tty_exec(self) -> None: + from clifwrap.config import merged_provider + + agent = merged_provider("agent", None) + cursor_agent = merged_provider("cursor-agent", None) + self.assertEqual(agent.interactive_mode, "tty-exec") + self.assertEqual(cursor_agent.interactive_mode, "tty-exec") + self.assertFalse(agent.proactive_pick) + self.assertIn("login", agent.passthrough_commands) + self.assertIn("rate limit", agent.retry_patterns) + + def test_wants_tty_exec_skips_print_mode(self) -> None: + from clifwrap.config import ProviderConfig + from clifwrap.runtime import _wants_tty_exec + + provider = ProviderConfig(name="agent", interactive_mode="tty-exec") + with mock.patch("clifwrap.runtime.sys.stdin") as stdin, mock.patch("clifwrap.runtime.sys.stdout") as stdout: + stdin.isatty.return_value = True + stdout.isatty.return_value = True + self.assertTrue(_wants_tty_exec(provider, [])) + self.assertTrue(_wants_tty_exec(provider, ["refactor auth"])) + self.assertFalse(_wants_tty_exec(provider, ["-p", "refactor auth"])) + self.assertFalse(_wants_tty_exec(provider, ["--print", "refactor auth"])) + + def test_agent_print_mode_failovers_on_rate_limit(self) -> None: + target = self.bin_dir / "agent" + make_executable( + target, + textwrap.dedent( + """\ + #!/usr/bin/env python3 + import os + import sys + key = os.environ.get("CURSOR_API_KEY", "") + if key == "primary-key": + print("Error: rate limit exceeded", file=sys.stderr) + raise SystemExit(1) + print(f"ok:{key}") + """ + ), + ) + self._run("install", "agent") + (self.config_dir / "config.toml").write_text( + textwrap.dedent( + """\ + [[providers.agent.accounts]] + name = "primary" + env = { CURSOR_API_KEY = "primary-key" } + + [[providers.agent.accounts]] + name = "backup" + env = { CURSOR_API_KEY = "backup-key" } + """ + ) + ) + proc = subprocess.run( + ["agent", "-p", "say hello"], + cwd=ROOT, + env=self.env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(proc.stdout.strip(), "ok:backup-key") + self.assertIn("retrying with backup", proc.stderr) + + def test_agent_login_passthrough_skips_failover(self) -> None: + target = self.bin_dir / "agent" + make_executable( + target, + textwrap.dedent( + """\ + #!/usr/bin/env python3 + import sys + print("login:" + " ".join(sys.argv[1:])) + """ + ), + ) + self._run("install", "agent") + (self.config_dir / "config.toml").write_text( + textwrap.dedent( + """\ + [[providers.agent.accounts]] + name = "primary" + env = { CURSOR_API_KEY = "primary-key" } + """ + ) + ) + proc = subprocess.run( + ["agent", "login"], + cwd=ROOT, + env=self.env, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + self.assertEqual(proc.stdout.strip(), "login:login") + if __name__ == "__main__": unittest.main() From 8ad6f3a47051c3942fc14f2dd3691b23cc96b759 Mon Sep 17 00:00:00 2001 From: Copilot Date: Sat, 25 Jul 2026 04:59:54 -0500 Subject: [PATCH 3/3] chore: anonymize agent CLI catalog and docs Rename the alternate provider to agentcli, scrub vendor tokens from docs/tests/catalog comments, and block the token in the anonymity guard so templates stay vendor-neutral. --- README.md | 24 ++++++++++++------------ docs/configuration.md | 2 +- docs/provider-authoring.md | 8 ++++---- docs/provider-catalog.md | 2 +- scripts/check_anonymity.py | 3 ++- src/clifwrap/providers.toml | 16 ++++++++-------- tests/test_wrapper.py | 20 ++++++++++---------- 7 files changed, 38 insertions(+), 37 deletions(-) diff --git a/README.md b/README.md index ac92cac..4244421 100644 --- a/README.md +++ b/README.md @@ -88,37 +88,37 @@ env = { SCRAPECLI_API_KEY = "env:SCRAPECLI_TEAM_B_KEY" } Retry rules, auth command names, and usage endpoints for SearchCLI and ScrapeCLI ship in `src/clifwrap/providers.toml`. Your config usually only needs account entries. -### Cursor Agent CLI +### Interactive agent CLIs -[Cursor Agent](https://cursor.com/docs/cli/overview) installs as `agent` (and legacy `cursor-agent`). Auth for automation uses `CURSOR_API_KEY` ([docs](https://cursor.com/docs/cli/reference/authentication)). +Some coding-agent CLIs ship a full-screen TUI plus a headless print mode and authenticate with a per-account API key env var. The catalog names `agent` and `agentcli` are templates — wrap whatever binary resolves on your PATH and set the key env var the tool actually reads. ```bash clifwrap init -clifwrap account add agent primary --env-ref CURSOR_API_KEY=CURSOR_API_KEY_PRIMARY -clifwrap account add agent backup --env-ref CURSOR_API_KEY=CURSOR_API_KEY_BACKUP +clifwrap account add agent primary --env-ref AGENTCLI_API_KEY=AGENTCLI_API_KEY_PRIMARY +clifwrap account add agent backup --env-ref AGENTCLI_API_KEY=AGENTCLI_API_KEY_BACKUP clifwrap install agent -# If another tool already owns `agent` on PATH, wrap the legacy name instead: -# clifwrap account add cursor-agent primary --env-ref CURSOR_API_KEY=CURSOR_API_KEY_PRIMARY -# clifwrap install cursor-agent +# If another tool already owns `agent` on PATH, wrap the alternate name instead: +# clifwrap account add agentcli primary --env-ref AGENTCLI_API_KEY=AGENTCLI_API_KEY_PRIMARY +# clifwrap install agentcli ``` ```toml [[providers.agent.accounts]] name = "primary" -env = { CURSOR_API_KEY = "env:CURSOR_API_KEY_PRIMARY" } +env = { AGENTCLI_API_KEY = "env:AGENTCLI_API_KEY_PRIMARY" } [[providers.agent.accounts]] name = "backup" -env = { CURSOR_API_KEY = "env:CURSOR_API_KEY_BACKUP" } +env = { AGENTCLI_API_KEY = "env:AGENTCLI_API_KEY_BACKUP" } ``` -Behavior matches Cursor CLI syntax: +Behavior: - Interactive TUI (`agent`, `agent "prompt"`) uses `tty-exec` — picks an account, then execs the real binary with a live terminal (no output buffering). - Headless print mode (`agent -p "..."`, `--print`, optional `--force` / `--output-format`) gets buffered failover on rate-limit / auth errors. -- `agent login`, `logout`, `status`, `about`, `models`, `mcp`, … pass through unchanged. +- Management commands (`login`, `logout`, `status`, `about`, `models`, `mcp`, …) pass through unchanged. -`agent status --format json` reports auth identity, not remaining quota, so proactive pick stays off unless you add your own `status_command` / `usage` metadata. +If the tool's status command reports auth identity but not remaining quota, proactive pick stays off unless you add your own `status_command` / `usage` metadata. See [docs/configuration.md](docs/configuration.md) for the full reference. diff --git a/docs/configuration.md b/docs/configuration.md index 9927382..634d186 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -45,7 +45,7 @@ Provider tables can set: | `retry_patterns` | Lowercased stderr/stdout snippets that trigger retry | | `never_retry_patterns` | Snippets that block retry even if another rule matches | | `retry_on_any_error` | Retry any nonzero exit unless blocked by `never_retry_patterns` | -| `interactive_mode` | `line-repl` for line-by-line shells; `tty-exec` for full-screen TUIs (e.g. Cursor Agent) | +| `interactive_mode` | `line-repl` for line-by-line shells; `tty-exec` for full-screen TUIs (e.g. interactive agent CLIs) | | `status_command` | Command that returns JSON with `remaining`, `limit`, and optionally `used` | | `proactive_pick` | Headroom-first starting account before the first upstream call (default `true` when `status_command` or `usage` is set) | | `passthrough_commands` | Upstream subcommands that skip wrapper auth handling | diff --git a/docs/provider-authoring.md b/docs/provider-authoring.md index a7b15f2..1d7a597 100644 --- a/docs/provider-authoring.md +++ b/docs/provider-authoring.md @@ -52,14 +52,14 @@ Probe findings never echo command output or argv — a failing `status_command` 4. Add integration tests in `tests/test_wrapper.py` using a generic `somecli` fixture where possible. 5. Do not add provider name literals to generic modules (`config.py`, `runtime.py`, etc.). -## Cursor Agent notes +## Interactive agent CLI notes -Cursor Agent is a full-screen TUI. Do not use `interactive_mode = "line-repl"` for it — that mode is for line-oriented shells like SearchCLI. Catalog entry `agent` / `cursor-agent` uses `tty-exec`: +Some coding-agent CLIs are full-screen TUIs. Do not use `interactive_mode = "line-repl"` for those — that mode is for line-oriented shells like SearchCLI. Catalog entries `agent` / `agentcli` use `tty-exec`: - Interactive sessions: select account → `exec` the real binary (TTY preserved). -- `agent -p` / `--print`: buffered run with retry/failover across accounts. +- `-p` / `--print` headless runs: buffered run with retry/failover across accounts. -Passthrough covers Cursor management commands (`login`, `logout`, `status`, `mcp`, …) so browser login and MCP setup stay upstream. +Passthrough covers management commands (`login`, `logout`, `status`, `mcp`, …) so browser login and MCP setup stay upstream. ## Related docs diff --git a/docs/provider-catalog.md b/docs/provider-catalog.md index 01f5b54..6dba781 100644 --- a/docs/provider-catalog.md +++ b/docs/provider-catalog.md @@ -22,7 +22,7 @@ Run `python scripts/generate_provider_catalog.py --write` after changing built-i | `syslog` | true | | `stderr` | true | -## `cursor-agent` +## `agentcli` | Field | Value | | --- | --- | diff --git a/scripts/check_anonymity.py b/scripts/check_anonymity.py index e7b54fe..edcdbe7 100644 --- a/scripts/check_anonymity.py +++ b/scripts/check_anonymity.py @@ -9,7 +9,7 @@ from pathlib import Path ROOT = Path(__file__).resolve().parents[1] -SKIP_DIRS = {".git", "agentdecompile_projects", "__pycache__", ".pytest_cache", "dist", "build", ".nox"} +SKIP_DIRS = {".git", "agentdecompile_projects", "__pycache__", ".pytest_cache", "dist", "build", ".nox", ".playwright-mcp"} SKIP_NAMES = {"actionlint", "actionlint.exe"} _ENCODED_REGEX = ( @@ -31,6 +31,7 @@ "YXBpXC50YXZpbHlcLmNvbQ==", "ZmlyZWNyYXdsLWNsaQ==", "ZG9jc1wudGF2aWx5XC5jb20=", + "XGJjdXJzb3JcYg==", ) PATTERNS = [ diff --git a/src/clifwrap/providers.toml b/src/clifwrap/providers.toml index 1e2a2b6..7c2d5eb 100644 --- a/src/clifwrap/providers.toml +++ b/src/clifwrap/providers.toml @@ -159,11 +159,11 @@ remediation_commands = [ "clifwrap config validate", ] -# Cursor Agent CLI (https://cursor.com/docs/cli/overview). -# Official install creates both `agent` (primary) and `cursor-agent` (legacy) PATH names. -# Prefer wrapping the name that resolves on your PATH. Auth is CURSOR_API_KEY per account -# (https://cursor.com/docs/cli/reference/authentication). Interactive TUI uses tty-exec; -# buffered failover applies to `agent -p` / `--print` headless runs. +# Interactive coding-agent CLIs that ship a full-screen TUI plus a headless +# print mode. Auth is a per-account API key env var. Interactive sessions use +# tty-exec; buffered failover applies to headless `-p` / `--print` runs. +# `agent` and `agentcli` are template names — wrap whatever binary resolves on +# your PATH and set the key env var the tool actually reads. [providers.agent] interactive_mode = "tty-exec" passthrough_commands = [ @@ -215,8 +215,8 @@ journald = false syslog = true stderr = true -# Legacy PATH name created by the Cursor Agent installer alongside `agent`. -[providers.cursor-agent] +# Alternate binary name for the same class of agent CLI. +[providers.agentcli] interactive_mode = "tty-exec" passthrough_commands = [ "login", @@ -260,7 +260,7 @@ never_retry_patterns = [ ] proactive_pick = false -[providers.cursor-agent.fallback_monitor] +[providers.agentcli.fallback_monitor] threshold = 2 action = "warn" journald = false diff --git a/tests/test_wrapper.py b/tests/test_wrapper.py index 78b5ceb..94c1379 100644 --- a/tests/test_wrapper.py +++ b/tests/test_wrapper.py @@ -3562,7 +3562,7 @@ def test_doctor_does_not_run_live_probes_without_probe_flag(self) -> None: self.assertTrue(marker.exists(), "doctor --probe did not run the status_command probe") def test_doctor_probe_does_not_leak_status_command_secrets(self) -> None: - secret = "supersecrettoken123" + probe_value = "probe-token-xyz-do-not-echo" (self.config_dir / "config.toml").write_text( textwrap.dedent( f"""\ @@ -3571,14 +3571,14 @@ def test_doctor_probe_does_not_leak_status_command_secrets(self) -> None: [[providers.somecli.accounts]] name = "only" - env = {{ SOMECLI_TOKEN = "{secret}" }} + env = {{ SOMECLI_TOKEN = "{probe_value}" }} """ ) ) proc = self._run("doctor", "--probe", "--json") self.assertEqual(proc.returncode, 0, proc.stderr) - self.assertNotIn(secret, proc.stdout) - self.assertNotIn(secret, proc.stderr) + self.assertNotIn(probe_value, proc.stdout) + self.assertNotIn(probe_value, proc.stderr) payload = json.loads(proc.stdout) validation = payload.get("validation") or [] self.assertTrue( @@ -3613,9 +3613,9 @@ def test_agent_catalog_provider_is_tty_exec(self) -> None: from clifwrap.config import merged_provider agent = merged_provider("agent", None) - cursor_agent = merged_provider("cursor-agent", None) + agentcli = merged_provider("agentcli", None) self.assertEqual(agent.interactive_mode, "tty-exec") - self.assertEqual(cursor_agent.interactive_mode, "tty-exec") + self.assertEqual(agentcli.interactive_mode, "tty-exec") self.assertFalse(agent.proactive_pick) self.assertIn("login", agent.passthrough_commands) self.assertIn("rate limit", agent.retry_patterns) @@ -3642,7 +3642,7 @@ def test_agent_print_mode_failovers_on_rate_limit(self) -> None: #!/usr/bin/env python3 import os import sys - key = os.environ.get("CURSOR_API_KEY", "") + key = os.environ.get("AGENTCLI_API_KEY", "") if key == "primary-key": print("Error: rate limit exceeded", file=sys.stderr) raise SystemExit(1) @@ -3656,11 +3656,11 @@ def test_agent_print_mode_failovers_on_rate_limit(self) -> None: """\ [[providers.agent.accounts]] name = "primary" - env = { CURSOR_API_KEY = "primary-key" } + env = { AGENTCLI_API_KEY = "primary-key" } [[providers.agent.accounts]] name = "backup" - env = { CURSOR_API_KEY = "backup-key" } + env = { AGENTCLI_API_KEY = "backup-key" } """ ) ) @@ -3694,7 +3694,7 @@ def test_agent_login_passthrough_skips_failover(self) -> None: """\ [[providers.agent.accounts]] name = "primary" - env = { CURSOR_API_KEY = "primary-key" } + env = { AGENTCLI_API_KEY = "primary-key" } """ ) )