From c2e426fafdcc262ada9f1d693aaf8f3fe7eab891 Mon Sep 17 00:00:00 2001 From: ajianaz Date: Fri, 16 Oct 2026 11:41:53 +0700 Subject: [PATCH] feat(cli): tole doctor coverage ledger (#378, 3/3) --- CHANGELOG.md | 1 + CONTRIBUTING.md | 3 +- README.md | 6 + crates/tole-cli/src/doctor.rs | 250 +++++++++++++++++++++++++++++++++- crates/tole-cli/src/main.rs | 27 +++- 5 files changed, 275 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 719980b..e0679a0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] ### Added +- `tole doctor` coverage ledger (#378): every run is recorded per provider + model (level reached, first blocker, fix command, timestamp) in an owner-only `doctor-ledger.json` (atomic write, a corrupt file is an error, never replaced); `--summary` prints `READY n-of-m` + the first blocker and its fix, `--no-ledger` skips recording. - `tole doctor` (#378): tiered provider diagnostics — `offline` (wiring), `catalog` (live `/models`, model must be served) and `full` (chat, streaming and a real tool-call loop) — each blocker carries its fix command, and keys / URL credentials are never printed. Exits non-zero when the requested tier is not reached; `--tier full` is the release-validation gate. - Per-user-turn token stats (#379): `tole status` lists input / output / cache-read / cache-write / reasoning tokens per turn and the status JSON (`GET /sessions/{id}/status`, `tole_session_status`) gains an additive `usage_report.turns` array. Derived from the durable usage ledger, so old sessions read the same way and nothing is stored twice; a value no step reported is `n/a` / `null`, never 0, and no elapsed time is invented. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3ee7eb7..cd141c5 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -133,7 +133,8 @@ full tier with the repo checkout binary: cargo build --release -p tole-cli && ./target/release/tole doctor --tier full ``` -It must exit 0 (chat, streaming and a real tool-call loop all green). +It must exit 0 (chat, streaming and a real tool-call loop all green); the +result is recorded in the coverage ledger (`tole doctor --summary`). ## Design rules diff --git a/README.md b/README.md index dcf79ff..8d245c3 100644 --- a/README.md +++ b/README.md @@ -133,6 +133,12 @@ the command that fixes it. The exit code is non-zero when the requested tier is not reached, so `tole doctor --tier full` is the gate for "ready to release" claims. API keys and URL credentials never appear in the output. +Every run is recorded in a coverage ledger (`$CODECORA_HOME/tole/doctor-ledger.json`, +owner-readable only: level reached, first blocker, fix, timestamp per provider ++ model; `--no-ledger` skips it). `tole doctor --summary` prints `READY n-of-m` +plus the first blocker and its fix command without running anything; the latest +run wins per provider + model, so re-run `--tier full` before a release claim. + ## Configuration (project file) `/.tole/config.toml` (no parent-directory walk) holds per-project defaults. diff --git a/crates/tole-cli/src/doctor.rs b/crates/tole-cli/src/doctor.rs index cb830c5..fd41f33 100644 --- a/crates/tole-cli/src/doctor.rs +++ b/crates/tole-cli/src/doctor.rs @@ -1,4 +1,5 @@ -//! `tole doctor` — tiered provider diagnostics (#378). +//! `tole doctor` — tiered provider diagnostics and a persisted coverage +//! ledger (#378). //! //! One command answers "is provider/model X actually usable?": //! @@ -11,11 +12,14 @@ //! a probe tool and the loop must settle). This is the release gate: a //! `full` green is the precondition for "ready to release" claims. //! -//! Tiers are cumulative and stop at the first blocker, which carries the -//! command that fixes it. +//! Tiers are cumulative and stop at the first blocker. Every run is +//! recorded in a ledger on disk (per provider + model: level reached, first +//! blocker, fix command, timestamp); [`Ledger::summary`] renders the +//! `READY n-of-m` line. +use serde::{Deserialize, Serialize}; use serde_json::{json, Value}; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use std::time::Duration; use tole_core::approval::{Approver, ToolRequest, Verdict}; use tole_core::openai::{fetch_model_ids, OpenAiConfig, OpenAiProvider, ENV_PREFIXES}; @@ -23,6 +27,9 @@ use tole_core::storage::JsonlStorage; use tole_core::tool::{Risk, Tool, ToolRegistry}; use tole_core::turn::{run_turn, TurnOutcome}; +/// Ledger file name under the tole data dir. +pub const LEDGER_FILE: &str = "doctor-ledger.json"; + /// Per-call timeout of the `full` tier. const FULL_CALL_TIMEOUT: Duration = Duration::from_secs(60); @@ -477,23 +484,168 @@ fn diagnose_raw( out } -/// CLI entry: diagnose and print. Returns `true` when the requested tier -/// was reached. +// ------------------------------------------------------------------ ledger + +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct LedgerEntry { + pub provider: String, + pub model: String, + /// `none` | `offline` | `catalog` | `full`. + pub level: String, + pub blocker: Option, + pub fix: Option, + pub at_ms: u64, +} + +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +pub struct Ledger { + pub entries: Vec, +} + +/// `/doctor-ledger.json`. +pub fn ledger_path() -> Result { + tole_core::paths::tole_data_dir() + .map(|d| d.join(LEDGER_FILE)) + .ok_or_else(|| "cannot locate the ledger: set CODECORA_HOME or HOME".to_string()) +} + +impl Ledger { + /// A missing file is an empty ledger; a corrupt one is an error (it is + /// never silently replaced). + pub fn load(path: &Path) -> Result { + match std::fs::read_to_string(path) { + Ok(s) => serde_json::from_str(&s) + .map_err(|e| format!("ledger {} is corrupt: {e}", path.display())), + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Ok(Ledger::default()), + Err(e) => Err(format!("cannot read ledger {}: {e}", path.display())), + } + } + + /// Atomic write (temp file + rename). + pub fn save(&self, path: &Path) -> Result<(), String> { + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent) + .map_err(|e| format!("cannot create {parent:?}: {e}"))?; + } + let tmp = path.with_extension("json.tmp"); + let body = serde_json::to_string_pretty(self).map_err(|e| e.to_string())?; + write_private(&tmp, body.as_bytes()).map_err(|e| format!("cannot write {tmp:?}: {e}"))?; + std::fs::rename(&tmp, path).map_err(|e| format!("cannot replace {path:?}: {e}")) + } + + /// Record the latest run for `provider`+`model` (replaces the previous + /// entry for the pair; the latest evidence wins). + pub fn record(&mut self, provider: &str, model: &str, outcome: &Outcome, at_ms: u64) { + let entry = LedgerEntry { + provider: provider.to_string(), + model: model.to_string(), + level: outcome.reached.map_or("none", Tier::as_str).to_string(), + blocker: outcome + .blocker + .as_ref() + .map(|b| format!("{}: {}", b.check, b.message)), + fix: outcome.blocker.as_ref().map(|b| b.fix.clone()), + at_ms, + }; + match self + .entries + .iter_mut() + .find(|e| e.provider == provider && e.model == model) + { + Some(slot) => *slot = entry, + None => self.entries.push(entry), + } + } + + /// `READY n-of-m` (n = entries proven at `full`), then the first + /// blocker and its fix command when there is one. + pub fn summary(&self) -> String { + let m = self.entries.len(); + let n = self.entries.iter().filter(|e| e.level == "full").count(); + let mut out = format!("READY {n}-of-{m}"); + if let Some(e) = self.entries.iter().find(|e| e.blocker.is_some()) { + out.push_str(&format!( + "\nfirst blocker: {}/{} ({}): {}", + e.provider, + e.model, + e.level, + e.blocker.as_deref().unwrap_or_default() + )); + if let Some(fix) = &e.fix { + out.push_str(&format!("\nfix: {fix}")); + } + } + out + } +} + +/// Write `bytes` readable by the owner only (0600 on Unix). A stale file at +/// `path` is removed first and the new one is created exclusively +/// (`O_EXCL`), so a pre-planted file or symlink is never written through +/// and the mode always applies. +fn write_private(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + use std::io::Write; + match std::fs::remove_file(path) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => {} + Err(e) => return Err(e), + } + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + opts.open(path)?.write_all(bytes) +} + +fn now_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as u64) + .unwrap_or(0) +} + +/// CLI entry: diagnose, persist to the ledger (unless `no_ledger`), print. +/// Returns `true` when the requested tier was reached. pub fn run_cli( cfg: Option<&OpenAiConfig>, env: &dyn Fn(&str) -> Option, requested: Tier, + no_ledger: bool, ) -> bool { let outcome = diagnose(cfg, env, requested); println!("tole doctor — tier {}", requested.as_str()); for l in &outcome.lines { println!("{l}"); } + let ok = outcome.reached >= Some(requested) && outcome.blocker.is_none(); if let Some(b) = &outcome.blocker { println!("blocked at {}: {}", b.tier.as_str(), b.message); println!("fix: {}", b.fix); } - outcome.reached >= Some(requested) && outcome.blocker.is_none() + // Only a configured provider has a ledger identity. + if let (Some(cfg), false) = (cfg, no_ledger) { + let saved = ledger_path().and_then(|p| { + let mut l = Ledger::load(&p)?; + l.record(&redact_url(&cfg.base_url), &cfg.model, &outcome, now_ms()); + l.save(&p)?; + Ok(l) + }); + match saved { + Ok(l) => println!("{}", l.summary()), + Err(e) => eprintln!("tole doctor: ledger not updated: {e}"), + } + } + ok +} + +/// `tole doctor --summary`: print the ledger summary without running. +pub fn print_summary() -> Result<(), String> { + let l = Ledger::load(&ledger_path()?)?; + println!("{}", l.summary()); + Ok(()) } #[cfg(test)] @@ -728,6 +880,58 @@ mod tests { assert_eq!(o.reached, Some(Tier::Catalog)); } + fn outcome(reached: Option, blocker: Option) -> Outcome { + Outcome { + reached, + blocker, + lines: vec![], + } + } + + #[test] + fn ledger_records_summarises_and_roundtrips() { + let mut l = Ledger::default(); + l.record("https://a/v1", "m1", &outcome(Some(Tier::Full), None), 1); + let blocker = Blocker { + tier: Tier::Full, + check: "tools", + message: "no tool call".into(), + fix: "export TOLE_MODEL=".into(), + }; + l.record( + "https://b/v1", + "m2", + &outcome(Some(Tier::Catalog), Some(blocker)), + 2, + ); + assert_eq!( + l.summary(), + "READY 1-of-2\nfirst blocker: https://b/v1/m2 (catalog): tools: no tool call\nfix: export TOLE_MODEL=" + ); + // Latest evidence wins for the same pair. + l.record("https://b/v1", "m2", &outcome(Some(Tier::Full), None), 3); + assert_eq!(l.entries.len(), 2); + assert_eq!(l.summary(), "READY 2-of-2"); + + let dir = scratch_dir("ledger"); + let path = dir.join("nested").join(LEDGER_FILE); + l.save(&path).unwrap(); + assert_eq!(Ledger::load(&path).unwrap(), l); + assert!(!path.with_extension("json.tmp").exists()); + let _ = std::fs::remove_dir_all(&dir); + } + + #[test] + fn ledger_load_missing_is_empty_but_corrupt_is_an_error() { + let dir = scratch_dir("ledger-load"); + let path = dir.join(LEDGER_FILE); + assert_eq!(Ledger::load(&path).unwrap(), Ledger::default()); + assert_eq!(Ledger::default().summary(), "READY 0-of-0"); + std::fs::write(&path, "{not json").unwrap(); + assert!(Ledger::load(&path).unwrap_err().contains("corrupt")); + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn urls_are_reduced_to_scheme_host_path() { assert_eq!( @@ -750,6 +954,38 @@ mod tests { assert!(all.contains("ftp://x/v1"), "{all}"); } + #[cfg(unix)] + #[test] + fn stale_tmp_is_replaced_not_written_through() { + use std::os::unix::fs::PermissionsExt; + let dir = scratch_dir("ledger-stale"); + let path = dir.join(LEDGER_FILE); + let tmp = path.with_extension("json.tmp"); + // A pre-planted, world-readable tmp symlinked elsewhere. + let victim = dir.join("victim"); + std::fs::write(&victim, "keep").unwrap(); + std::os::unix::fs::symlink(&victim, &tmp).unwrap(); + Ledger::default().save(&path).unwrap(); + assert_eq!(std::fs::read_to_string(&victim).unwrap(), "keep"); + assert_eq!( + std::fs::metadata(&path).unwrap().permissions().mode() & 0o777, + 0o600 + ); + let _ = std::fs::remove_dir_all(&dir); + } + + #[cfg(unix)] + #[test] + fn ledger_file_is_owner_only() { + use std::os::unix::fs::PermissionsExt; + let dir = scratch_dir("ledger-mode"); + let path = dir.join(LEDGER_FILE); + Ledger::default().save(&path).unwrap(); + let mode = std::fs::metadata(&path).unwrap().permissions().mode() & 0o777; + assert_eq!(mode, 0o600); + let _ = std::fs::remove_dir_all(&dir); + } + #[test] fn tier_parse_roundtrip() { for t in [Tier::Offline, Tier::Catalog, Tier::Full] { diff --git a/crates/tole-cli/src/main.rs b/crates/tole-cli/src/main.rs index 9bfbacb..30d55e6 100644 --- a/crates/tole-cli/src/main.rs +++ b/crates/tole-cli/src/main.rs @@ -1,6 +1,6 @@ //! CLI host for tole-core: arg parsing and session wiring. -use anyhow::{bail, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use clap::{Parser, Subcommand}; use std::path::{Path, PathBuf}; @@ -318,12 +318,21 @@ enum Command { /// Is the configured provider/model actually usable? Tiered checks — /// offline (wiring), catalog (live /models), full (chat + streaming + - /// a real tool-call loop). Exits non-zero when the requested tier is not reached, so `--tier full` + /// a real tool-call loop), recorded in a coverage ledger. Exits + /// non-zero when the requested tier is not reached, so `--tier full` /// is the release-validation gate. Doctor { /// offline | catalog | full (full spends a few real tokens). #[arg(long, default_value = "catalog")] tier: String, + + /// Do not record this run in the ledger. + #[arg(long)] + no_ledger: bool, + + /// Only print the ledger summary (READY n-of-m); run nothing. + #[arg(long)] + summary: bool, }, /// Show durable state of a session. @@ -1039,12 +1048,22 @@ fn dispatch(cli: Cli) -> Result<()> { }) } Command::Sessions => sessions_command(&sessions_dir), - Command::Doctor { tier } => { + Command::Doctor { + tier, + no_ledger, + summary, + } => { + if summary { + tole_cli::doctor::print_summary().map_err(|e| anyhow!(e))?; + return Ok(()); + } let Some(tier) = tole_cli::doctor::Tier::parse(&tier) else { bail!("--tier must be offline, catalog or full (got {tier:?})"); }; let cfg = config_apply::provider_config(); - if !tole_cli::doctor::run_cli(cfg.as_ref(), &config_apply::real_env, tier) { + let ok = + tole_cli::doctor::run_cli(cfg.as_ref(), &config_apply::real_env, tier, no_ledger); + if !ok { std::process::exit(1); } Ok(())