From 6104757bbec276c5fad547aa75b2891660658400 Mon Sep 17 00:00:00 2001 From: Alex Oliveira <4482374+aroff@users.noreply.github.com> Date: Wed, 8 Jul 2026 20:41:11 +0000 Subject: [PATCH] feat(session-capture): implement passive adapters --- Cargo.toml | 27 + aikit-sdk/Cargo.toml | 11 + aikit-sdk/src/aikit_agent_adapter.rs | 55 +- aikit-sdk/src/cost/extract.rs | 172 +++ aikit-sdk/src/cost/mod.rs | 165 +++ aikit-sdk/src/lib.rs | 2 + aikit-sdk/src/runner/backend.rs | 76 + aikit-sdk/src/runner/backends/claude.rs | 17 + aikit-sdk/src/runner/backends/codex.rs | 13 + aikit-sdk/src/runner/backends/opencode.rs | 8 + aikit-sdk/src/runner/capabilities.rs | 11 + aikit-session-capture/Cargo.toml | 46 + aikit-session-capture/src/adapter.rs | 140 ++ aikit-session-capture/src/claudecode/mod.rs | 123 ++ .../src/claudecode/transcript.rs | 918 ++++++++++++ aikit-session-capture/src/codex/mod.rs | 104 ++ aikit-session-capture/src/codex/transcript.rs | 1228 +++++++++++++++++ aikit-session-capture/src/cursor_offset.rs | 166 +++ aikit-session-capture/src/event_store.rs | 393 ++++++ aikit-session-capture/src/homes.rs | 317 +++++ aikit-session-capture/src/lib.rs | 60 + aikit-session-capture/src/mcp.rs | 652 +++++++++ aikit-session-capture/src/models.rs | 227 +++ aikit-session-capture/src/opencode/db.rs | 65 + aikit-session-capture/src/opencode/mirror.rs | 202 +++ aikit-session-capture/src/opencode/mod.rs | 701 ++++++++++ aikit-session-capture/src/registry.rs | 163 +++ aikit-session-capture/src/scrub.rs | 162 +++ aikit-session-capture/src/watch.rs | 237 ++++ .../tests/fixtures/claudecode/README.md | 21 + .../tests/fixtures/claudecode/api-error.jsonl | 4 + .../claudecode/concatenated-records.jsonl | 3 + .../fixtures/claudecode/malformed-line.jsonl | 3 + .../claudecode/multi-block-dedup.jsonl | 7 + .../fixtures/claudecode/multi-tool-turn.jsonl | 2 + .../fixtures/claudecode/simple-session.jsonl | 5 + .../fixtures/claudecode/with-secrets.jsonl | 3 + .../tests/fixtures/codex/README.md | 12 + .../codex/rollout-response-item.jsonl | 27 + .../fixtures/codex/rollout-session.jsonl | 10 + .../tests/fixtures/codex/with-secrets.jsonl | 38 + aikit-session-capture/tests/opencode.rs | 156 +++ .../tests/opencode_fixture.rs | 151 ++ src/cli/commands/package.rs | 76 +- src/cli/mod.rs | 8 + src/cli/serve/capture.rs | 860 ++++++++++++ src/cli/serve/mod.rs | 80 ++ src/cli/serve/storage/cursor_store.rs | 145 ++ src/cli/serve/storage/event_store.rs | 602 ++++++++ src/cli/serve/storage/mod.rs | 20 + src/cli/serve/storage/schema.rs | 107 ++ tests/e2e_package_publish.rs | 56 +- tests/integration_package_upload.rs | 54 +- 53 files changed, 8826 insertions(+), 85 deletions(-) create mode 100644 aikit-sdk/src/cost/extract.rs create mode 100644 aikit-sdk/src/cost/mod.rs create mode 100644 aikit-session-capture/Cargo.toml create mode 100644 aikit-session-capture/src/adapter.rs create mode 100644 aikit-session-capture/src/claudecode/mod.rs create mode 100644 aikit-session-capture/src/claudecode/transcript.rs create mode 100644 aikit-session-capture/src/codex/mod.rs create mode 100644 aikit-session-capture/src/codex/transcript.rs create mode 100644 aikit-session-capture/src/cursor_offset.rs create mode 100644 aikit-session-capture/src/event_store.rs create mode 100644 aikit-session-capture/src/homes.rs create mode 100644 aikit-session-capture/src/lib.rs create mode 100644 aikit-session-capture/src/mcp.rs create mode 100644 aikit-session-capture/src/models.rs create mode 100644 aikit-session-capture/src/opencode/db.rs create mode 100644 aikit-session-capture/src/opencode/mirror.rs create mode 100644 aikit-session-capture/src/opencode/mod.rs create mode 100644 aikit-session-capture/src/registry.rs create mode 100644 aikit-session-capture/src/scrub.rs create mode 100644 aikit-session-capture/src/watch.rs create mode 100644 aikit-session-capture/tests/fixtures/claudecode/README.md create mode 100644 aikit-session-capture/tests/fixtures/claudecode/api-error.jsonl create mode 100644 aikit-session-capture/tests/fixtures/claudecode/concatenated-records.jsonl create mode 100644 aikit-session-capture/tests/fixtures/claudecode/malformed-line.jsonl create mode 100644 aikit-session-capture/tests/fixtures/claudecode/multi-block-dedup.jsonl create mode 100644 aikit-session-capture/tests/fixtures/claudecode/multi-tool-turn.jsonl create mode 100644 aikit-session-capture/tests/fixtures/claudecode/simple-session.jsonl create mode 100644 aikit-session-capture/tests/fixtures/claudecode/with-secrets.jsonl create mode 100644 aikit-session-capture/tests/fixtures/codex/README.md create mode 100644 aikit-session-capture/tests/fixtures/codex/rollout-response-item.jsonl create mode 100644 aikit-session-capture/tests/fixtures/codex/rollout-session.jsonl create mode 100644 aikit-session-capture/tests/fixtures/codex/with-secrets.jsonl create mode 100644 aikit-session-capture/tests/opencode.rs create mode 100644 aikit-session-capture/tests/opencode_fixture.rs create mode 100644 src/cli/serve/capture.rs create mode 100644 src/cli/serve/storage/cursor_store.rs create mode 100644 src/cli/serve/storage/event_store.rs create mode 100644 src/cli/serve/storage/mod.rs create mode 100644 src/cli/serve/storage/schema.rs diff --git a/Cargo.toml b/Cargo.toml index 793a98c..4ef4ca5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,6 +9,7 @@ members = [ "aikit-magictool", "aikit-textgrad", "aikit-skillopt", + "aikit-session-capture", ] [package] @@ -66,6 +67,14 @@ walkdir = "2" # Tool detection which = "8.0" +# Session capture (spec 010): on-disk adapter parsing for AI coding tools. +# Optional — pulled when `agent-adapters` feature is on. +aikit-session-capture = { path = "aikit-session-capture", version = "0.1.0", optional = true } +# SQLite for the production EventStore/CursorStore (capture serve surface). +rusqlite = { version = "0.39", features = ["bundled"], optional = true } +# async trait for the EventStore/CursorStore impls. +async-trait = { version = "0.1", optional = true } + # aikit SDK (agent catalog and deploy) # claude-control/codex-app-server enable bidirectional live-session bridges. aikit-sdk = { path = "aikit-sdk", version = "0.3.0", features = ["claude-control", "codex-app-server"] } @@ -147,6 +156,24 @@ path = "tests/serve/serve_disconnect_test.rs" [features] tools = ["dep:aikit-magictool"] +# Session capture (spec 010): on-disk adapter parsing for AI coding tools. +# Pulls the adapter crate + SQLite storage + flips aikit-sdk capabilities. +agent-adapters = [ + "dep:aikit-session-capture", + "dep:rusqlite", + "dep:async-trait", + "aikit-sdk/agent-adapters", +] +claudecode = ["agent-adapters", "aikit-sdk/claudecode", "aikit-session-capture/claudecode"] +codex = ["agent-adapters", "aikit-sdk/codex", "aikit-session-capture/codex"] +opencode = ["agent-adapters", "aikit-sdk/opencode", "aikit-session-capture/opencode"] +# Watcher (spec 010 §14.5): notify-based fs watcher for automated capture. +watcher = ["aikit-session-capture/watcher"] +# MCP tools (spec 010 §15): registers check_file_freshness etc. +mcp-tools = ["aikit-session-capture/mcp-tools"] +# Crossmount (spec 010 §8/§19.3): WSL2 ↔ Windows home enumeration. +crossmount = ["aikit-session-capture/crossmount"] +default = ["claudecode", "codex", "opencode", "watcher"] [profile.release] opt-level = 3 diff --git a/aikit-sdk/Cargo.toml b/aikit-sdk/Cargo.toml index d473965..4421d64 100644 --- a/aikit-sdk/Cargo.toml +++ b/aikit-sdk/Cargo.toml @@ -20,6 +20,15 @@ claude-control = ["claude-sdk", "dep:tokio"] # Bidirectional Codex sessions: drive `aikit-agent-codex`'s app-server client # (JSON-RPC over stdio) over a tokio bridge with a Control handle. codex-app-server = ["dep:aikit-agent-codex", "dep:tokio"] +# Agent session capture (spec 010): passive on-disk session parsing for AI +# coding tools. Pulls the aikit-session-capture crate and flips the matching +# Backend `passive_capture` capability. Per-adapter features below gate the +# individual adapter modules so consumers can opt in to just Claude Code +# without dragging rusqlite (opencode) etc. +agent-adapters = ["dep:aikit-session-capture"] +claudecode = ["agent-adapters", "aikit-session-capture/claudecode"] +codex = ["agent-adapters", "aikit-session-capture/codex"] +opencode = ["agent-adapters", "aikit-session-capture/opencode"] [dependencies] tracing = "0.1" @@ -38,9 +47,11 @@ claude-agent-sdk = { git = "https://github.com/aroff/claude-agent-sdk-rust", rev aikit-agent-codex = { path = "../aikit-agent-codex", version = "0.1.0", optional = true } tokio = { version = "1", features = ["rt", "sync", "macros"], optional = true } jsonschema = "0.46" +aikit-session-capture = { path = "../aikit-session-capture", version = "0.1.0", optional = true } [dev-dependencies] insta = { version = "1.39", features = ["json"] } assert_cmd = "2.0" mockito = "1.2" serde_json = "1.0" +tokio = { version = "1", features = ["rt", "macros"] } diff --git a/aikit-sdk/src/aikit_agent_adapter.rs b/aikit-sdk/src/aikit_agent_adapter.rs index 2070cc4..98b169b 100644 --- a/aikit-sdk/src/aikit_agent_adapter.rs +++ b/aikit-sdk/src/aikit_agent_adapter.rs @@ -79,8 +79,7 @@ where .current_dir .clone() .unwrap_or_else(|| std::env::current_dir().unwrap_or_else(|_| PathBuf::from("."))); - let mut config = AgentConfig::from_env(workdir, options.stream, options.model.clone()) - .map_err(|e| emit_error(prompt, options, &mut on_event, e.to_string()))?; + let mut config = config_for_injected_gateway(workdir, options); apply_session_options(options, &mut config); @@ -95,6 +94,58 @@ where ) } +fn config_for_injected_gateway(workdir: PathBuf, options: &RunOptions) -> AgentConfig { + AgentConfig::from_env(workdir.clone(), options.stream, options.model.clone()).unwrap_or_else( + |_| AgentConfig { + model: options + .model + .clone() + .or_else(|| std::env::var("AIKIT_MODEL").ok()) + .unwrap_or_else(|| "gpt-4o".to_string()), + base_url: std::env::var("AIKIT_LLM_URL") + .unwrap_or_else(|_| "https://api.openai.com/v1".to_string()), + api_key: "injected-gateway".to_string(), + stream: options.stream + || std::env::var("AIKIT_STREAM") + .map(|v| v.eq_ignore_ascii_case("true") || v == "1") + .unwrap_or(false), + max_iterations: std::env::var("AIKIT_MAX_ITERATIONS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(10u32), + max_subagent_depth: std::env::var("AIKIT_MAX_SUBAGENT_DEPTH") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(2u32), + context_budget_tokens: std::env::var("AIKIT_CONTEXT_BUDGET_TOKENS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(12000u64), + allowed_roots: vec![workdir.clone()], + skills_dirs: Vec::new(), + agents_md_path: agents_md_path(&workdir), + workdir, + timeout_secs: 60, + connect_timeout_secs: 10, + session_persona: None, + session_agents: std::collections::HashMap::new(), + host_tool_provider: None, + }, + ) +} + +fn agents_md_path(workdir: &std::path::Path) -> Option { + let agents_md = workdir.join("AGENTS.md"); + let claude_md = workdir.join("CLAUDE.md"); + if agents_md.exists() { + Some(agents_md) + } else if claude_md.exists() { + Some(claude_md) + } else { + None + } +} + fn apply_session_options(options: &RunOptions, config: &mut AgentConfig) { // Apply session persona: deserialize from JSON, then apply model override if no CLI --model. if let Some(ref persona_val) = options.session_persona { diff --git a/aikit-sdk/src/cost/extract.rs b/aikit-sdk/src/cost/extract.rs new file mode 100644 index 0000000..fc58ae9 --- /dev/null +++ b/aikit-sdk/src/cost/extract.rs @@ -0,0 +1,172 @@ +//! Cost extraction: the pull path from adapter-emitted `TokenEvent`s (spec +//! 010 §16). +//! +//! [`client_computed_cost`] is a **fallback only** — called when the +//! provider-side cost signal is absent AND `passive_capture` capability is +//! true. Produces a [`CostSnapshot`] with `ClientComputed` provenance and +//! `is_estimate == true`. + +use crate::cost::{CostProvenance, CostSnapshot, PricingTable, Spend, SpendScope}; +use crate::runner::backend::Backend; + +fn now_ms() -> i64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_millis() as i64) + .unwrap_or(0) +} + +/// Map a Backend to the ToolKind an adapter would carry. +/// Returns `None` for Backends with no adapter compiled in. +fn backend_to_tool_kind(b: Backend) -> Option { + match b { + #[cfg(feature = "claudecode")] + Backend::Claude => Some(aikit_session_capture::ToolKind::ClaudeCode), + #[cfg(feature = "codex")] + Backend::Codex => Some(aikit_session_capture::ToolKind::Codex), + #[cfg(feature = "opencode")] + Backend::OpenCode => Some(aikit_session_capture::ToolKind::OpenCode), + _ => None, + } +} + +/// Compute cost from adapter-emitted `TokenEvent`s. Fallback only — called +/// when the provider-side cost signal is absent (spec 010 §16). +/// +/// Produces a [`CostSnapshot`] with `ClientComputed` provenance and +/// `is_estimate == true` (spec 009 §5 invariant). +pub async fn client_computed_cost( + store: &dyn aikit_session_capture::EventStore, + backend: Backend, + session_id: &str, + pricing: &PricingTable, +) -> Option { + let tool_kind = backend_to_tool_kind(backend)?; + let tokens = store + .token_events_for_session(tool_kind, session_id) + .await + .ok()?; + if tokens.is_empty() { + return None; + } + let spend_usd = pricing.estimate(&tokens)?; + let per_model = pricing.per_model_breakdown(&tokens); + + Some(CostSnapshot { + backend: backend.key().to_string(), + captured_at_ms: now_ms(), + windows: vec![], + spend: Some(Spend { + amount_usd: spend_usd, + scope: SpendScope::Session, + is_estimate: true, + }), + per_model, + credits: None, + provenance: CostProvenance::ClientComputed, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cost::PricingTable; + + #[cfg(all(feature = "agent-adapters", feature = "claudecode"))] + #[tokio::test] + async fn client_computed_cost_produces_estimate() { + use aikit_session_capture::{EventBatch, EventStore, InMemoryEventStore, TokenEvent}; + + let store = InMemoryEventStore::new(); + let token_ev = TokenEvent { + source_event_id: "tk_1".into(), + session_id: "s1".into(), + tool: aikit_session_capture::ToolKind::ClaudeCode, + model: Some("claude-sonnet-4-20250514".into()), + request_id: None, + input_tokens: Some(1000), + cache_read_tokens: Some(500), + cache_creation_tokens: None, + cache_creation_1h_tokens: None, + output_tokens: Some(200), + reasoning_tokens: None, + captured_at_ms: 1000, + captured_via: aikit_session_capture::CaptureSource::Transcript, + }; + store + .upsert_events(EventBatch { + tool_events: vec![], + token_events: vec![token_ev], + cache_observations: vec![], + }) + .await + .unwrap(); + + let pricing = PricingTable::default(); + let snapshot = client_computed_cost(&store, Backend::Claude, "s1", &pricing).await; + + assert!(snapshot.is_some(), "should produce a CostSnapshot"); + let snapshot = snapshot.unwrap(); + assert_eq!( + snapshot.provenance, + CostProvenance::ClientComputed, + "provenance MUST be ClientComputed" + ); + assert!( + snapshot.spend.as_ref().unwrap().is_estimate, + "is_estimate MUST be true for ClientComputed (spec 009 §5)" + ); + assert!( + snapshot.spend.as_ref().unwrap().amount_usd > 0.0, + "cost should be non-zero" + ); + } + + #[test] + fn pricing_estimate_sums_all_events() { + use aikit_session_capture::{CaptureSource, TokenEvent}; + + let tokens = vec![ + TokenEvent { + source_event_id: "1".into(), + session_id: "s1".into(), + tool: aikit_session_capture::ToolKind::ClaudeCode, + model: Some("default".into()), + request_id: None, + input_tokens: Some(1_000_000), + cache_read_tokens: None, + cache_creation_tokens: None, + cache_creation_1h_tokens: None, + output_tokens: Some(1_000_000), + reasoning_tokens: None, + captured_at_ms: 0, + captured_via: CaptureSource::Transcript, + }, + TokenEvent { + source_event_id: "2".into(), + session_id: "s1".into(), + tool: aikit_session_capture::ToolKind::ClaudeCode, + model: Some("default".into()), + request_id: None, + input_tokens: Some(500_000), + cache_read_tokens: None, + cache_creation_tokens: None, + cache_creation_1h_tokens: None, + output_tokens: Some(500_000), + reasoning_tokens: None, + captured_at_ms: 0, + captured_via: CaptureSource::Transcript, + }, + ]; + let pricing = PricingTable::default(); + // 3M input * $3/Mtok + 3M output * $15/Mtok = $9 + $45 = $54 wait... + // Actually: (1M+0.5M) input = 1.5M * $3 = $4.50 + // (1M+0.5M) output = 1.5M * $15 = $22.50 + // Total = $27.00 + let cost = pricing.estimate(&tokens).unwrap(); + assert!( + (cost - 27.0).abs() < 0.01, + "expected $27.00, got ${cost:.2}" + ); + } +} diff --git a/aikit-sdk/src/cost/mod.rs b/aikit-sdk/src/cost/mod.rs new file mode 100644 index 0000000..604f40d --- /dev/null +++ b/aikit-sdk/src/cost/mod.rs @@ -0,0 +1,165 @@ +//! Cost monitoring types + adapter-backed cost extraction (spec 009/010 §16). +//! +//! When the run loop aggregates a `RunResult` and a Backend's `extract_cost` +//! returned `None` (no provider-side cost signal), the engine falls back to +//! adapter-emitted `TokenEvent`s via [`client_computed_cost`]. This is a +//! **fallback only**, never a replacement — provider-side cost always wins. + +pub mod extract; + +use serde::{Deserialize, Serialize}; + +/// Provenance of a [`CostSnapshot`]. Determines authority level. +#[non_exhaustive] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CostProvenance { + /// Sourced from the API reverse proxy (most authoritative). + Proxy, + /// Sourced from the provider's own cost API. + Provider, + /// Computed client-side from on-disk transcript token counts. + /// Always pairs with `Spend.is_estimate = true`. + ClientComputed, +} + +/// One cost snapshot for one session/run. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CostSnapshot { + pub backend: String, + pub captured_at_ms: i64, + pub windows: Vec, + pub spend: Option, + pub per_model: Vec, + pub credits: Option, + pub provenance: CostProvenance, +} + +/// A named time-window slice of cost data. +#[non_exhaustive] +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CostWindow { + pub label: String, + pub start_ms: i64, + pub end_ms: i64, + pub amount_usd: f64, +} + +/// Spend estimate for a session. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct Spend { + pub amount_usd: f64, + pub scope: SpendScope, + /// Always `true` when provenance is `ClientComputed` (spec 009 §5 invariant). + pub is_estimate: bool, +} + +#[non_exhaustive] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum SpendScope { + Session, + Window, + Daily, +} + +/// Per-model cost breakdown. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ModelSpend { + pub model: String, + pub input_tokens: u64, + pub output_tokens: u64, + pub cache_read_tokens: u64, + pub amount_usd: f64, +} + +/// Credit balance info (when available from the provider). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CreditBalance { + pub remaining_usd: f64, + pub currency: String, +} + +/// Pricing table for computing cost from token counts. Populated by the +/// host from their pricing config; the `client_computed_cost` function uses +/// it to estimate spend from adapter `TokenEvent`s. +#[derive(Debug, Clone, Default)] +pub struct PricingTable { + /// Map of model name → per-million-token prices. + pub models: std::collections::HashMap, +} + +/// Per-model pricing (USD per million tokens). +#[derive(Debug, Clone)] +pub struct ModelPricing { + pub input_per_mtok: f64, + pub output_per_mtok: f64, + pub cache_read_per_mtok: f64, +} + +impl Default for ModelPricing { + fn default() -> Self { + Self { + input_per_mtok: 3.0, + output_per_mtok: 15.0, + cache_read_per_mtok: 0.30, + } + } +} + +impl PricingTable { + /// Estimate total spend from a set of token events. + pub fn estimate(&self, tokens: &[aikit_session_capture::TokenEvent]) -> Option { + let total: f64 = tokens + .iter() + .map(|ev| { + let pricing = self + .models + .get(ev.model.as_deref().unwrap_or("default")) + .cloned() + .unwrap_or_default(); + let input = ev.input_tokens.unwrap_or(0) as f64 / 1_000_000.0; + let output = ev.output_tokens.unwrap_or(0) as f64 / 1_000_000.0; + let cache_read = ev.cache_read_tokens.unwrap_or(0) as f64 / 1_000_000.0; + input * pricing.input_per_mtok + + output * pricing.output_per_mtok + + cache_read * pricing.cache_read_per_mtok + }) + .sum(); + Some(total) + } + + /// Per-model breakdown for a set of token events. + pub fn per_model_breakdown( + &self, + tokens: &[aikit_session_capture::TokenEvent], + ) -> Vec { + let mut by_model: std::collections::HashMap = + std::collections::HashMap::new(); + for ev in tokens { + let model = ev.model.clone().unwrap_or_else(|| "unknown".to_string()); + let pricing = self.models.get(&model).cloned().unwrap_or_default(); + let input = ev.input_tokens.unwrap_or(0); + let output = ev.output_tokens.unwrap_or(0); + let cache_read = ev.cache_read_tokens.unwrap_or(0); + let cost = (input as f64 / 1_000_000.0) * pricing.input_per_mtok + + (output as f64 / 1_000_000.0) * pricing.output_per_mtok + + (cache_read as f64 / 1_000_000.0) * pricing.cache_read_per_mtok; + let entry = by_model.entry(model).or_insert((0, 0, 0, 0.0)); + entry.0 += input; + entry.1 += output; + entry.2 += cache_read; + entry.3 += cost; + } + by_model + .into_iter() + .map(|(model, (input, output, cache_read, amount))| ModelSpend { + model, + input_tokens: input, + output_tokens: output, + cache_read_tokens: cache_read, + amount_usd: amount, + }) + .collect() + } +} diff --git a/aikit-sdk/src/lib.rs b/aikit-sdk/src/lib.rs index 3c3589f..905889b 100644 --- a/aikit-sdk/src/lib.rs +++ b/aikit-sdk/src/lib.rs @@ -415,6 +415,8 @@ pub fn deploy_subagent( } pub mod aikit_agent_adapter; +#[cfg(feature = "agent-adapters")] +pub mod cost; pub mod runner; pub mod session_store; diff --git a/aikit-sdk/src/runner/backend.rs b/aikit-sdk/src/runner/backend.rs index 49c4b82..b90f082 100644 --- a/aikit-sdk/src/runner/backend.rs +++ b/aikit-sdk/src/runner/backend.rs @@ -307,4 +307,80 @@ mod tests { assert_eq!(a, c, "extract_usage not deterministic for {b:?}"); } } + + // ---- spec 010: passive_capture capability gate ---------------------- + + /// Spec 010 §18 acceptance criterion 1: Claude's `passive_capture` flips + /// on only when both `agent-adapters` and `claudecode` are enabled; + /// Backends with no adapter shipped (Cursor, Gemini, Aikit) are always + /// false. The two `#[cfg]` arms mirror the production const definition + /// in `backends/claude.rs`, so a future feature-add won't silently + /// regress this test. + #[cfg(all(feature = "agent-adapters", feature = "claudecode"))] + #[test] + fn claude_passive_capture_on_when_feature_enabled() { + assert!( + Backend::Claude.capabilities().passive_capture, + "Claude.passive_capture MUST be true with agent-adapters + claudecode features" + ); + // Backends with no adapter shipped — stays false (spec 010 §5 table). + assert!( + !Backend::Cursor.capabilities().passive_capture, + "Cursor has no adapter; passive_capture MUST be false" + ); + assert!( + !Backend::Gemini.capabilities().passive_capture, + "Gemini has no adapter; passive_capture MUST be false" + ); + } + + #[cfg(not(all(feature = "agent-adapters", feature = "claudecode")))] + #[test] + fn claude_passive_capture_off_when_feature_disabled() { + // Without the feature, Claude's passive_capture stays off — even + // though Claude is a known Backend. The capability is honest: + // "unsupported" rather than "supported but no data". + assert!( + !Backend::Claude.capabilities().passive_capture, + "Claude.passive_capture MUST be false without agent-adapters feature" + ); + } + + /// Spec 010 Phase 3: Codex passive_capture follows the same pattern. + #[cfg(all(feature = "agent-adapters", feature = "codex"))] + #[test] + fn codex_passive_capture_on_when_feature_enabled() { + assert!( + Backend::Codex.capabilities().passive_capture, + "Codex.passive_capture MUST be true with agent-adapters + codex features" + ); + } + + #[cfg(not(all(feature = "agent-adapters", feature = "codex")))] + #[test] + fn codex_passive_capture_off_when_feature_disabled() { + assert!( + !Backend::Codex.capabilities().passive_capture, + "Codex.passive_capture MUST be false without agent-adapters + codex features" + ); + } + + /// Spec 010 Phase 3: OpenCode passive_capture follows the same pattern. + #[cfg(all(feature = "agent-adapters", feature = "opencode"))] + #[test] + fn opencode_passive_capture_on_when_feature_enabled() { + assert!( + Backend::OpenCode.capabilities().passive_capture, + "OpenCode.passive_capture MUST be true with agent-adapters + opencode features" + ); + } + + #[cfg(not(all(feature = "agent-adapters", feature = "opencode")))] + #[test] + fn opencode_passive_capture_off_when_feature_disabled() { + assert!( + !Backend::OpenCode.capabilities().passive_capture, + "OpenCode.passive_capture MUST be false without agent-adapters + opencode features" + ); + } } diff --git a/aikit-sdk/src/runner/backends/claude.rs b/aikit-sdk/src/runner/backends/claude.rs index 097c5a2..860e5d6 100644 --- a/aikit-sdk/src/runner/backends/claude.rs +++ b/aikit-sdk/src/runner/backends/claude.rs @@ -21,6 +21,23 @@ pub(crate) const KEY: &str = "claude"; pub(crate) const BINARY_CANDIDATES: &[&str] = &["claude"]; +// `passive_capture` flips on only when both `agent-adapters` and the +// `claudecode` adapter feature are enabled. Spec 010 §17.2. Two `#[cfg]` +// arms keep the const evaluable without runtime branching. +#[cfg(all(feature = "agent-adapters", feature = "claudecode"))] +pub(crate) const CAPABILITIES: BackendCapabilities = BackendCapabilities::NONE + .with_bidirectional() + .with_structured_tools() + .with_reasoning() + .with_interruptible() + .with_resumable_sessions() + .with_mcp_routing() + .with_hooks() + .with_server_tools() + .with_subagents() + .with_passive_capture(); + +#[cfg(not(all(feature = "agent-adapters", feature = "claudecode")))] pub(crate) const CAPABILITIES: BackendCapabilities = BackendCapabilities::NONE .with_bidirectional() .with_structured_tools() diff --git a/aikit-sdk/src/runner/backends/codex.rs b/aikit-sdk/src/runner/backends/codex.rs index 318645c..5c09751 100644 --- a/aikit-sdk/src/runner/backends/codex.rs +++ b/aikit-sdk/src/runner/backends/codex.rs @@ -18,6 +18,19 @@ pub(crate) const KEY: &str = "codex"; pub(crate) const BINARY_CANDIDATES: &[&str] = &["codex"]; +// `passive_capture` flips on only when both `agent-adapters` and the +// `codex` adapter feature are enabled. Spec 010 §17.2. +#[cfg(all(feature = "agent-adapters", feature = "codex"))] +pub(crate) const CAPABILITIES: BackendCapabilities = BackendCapabilities::NONE + .with_bidirectional() + .with_structured_tools() + .with_reasoning() + .with_file_changes() + .with_interruptible() + .with_resumable_sessions() + .with_passive_capture(); + +#[cfg(not(all(feature = "agent-adapters", feature = "codex")))] pub(crate) const CAPABILITIES: BackendCapabilities = BackendCapabilities::NONE .with_bidirectional() .with_structured_tools() diff --git a/aikit-sdk/src/runner/backends/opencode.rs b/aikit-sdk/src/runner/backends/opencode.rs index 8ae37d9..365c0d7 100644 --- a/aikit-sdk/src/runner/backends/opencode.rs +++ b/aikit-sdk/src/runner/backends/opencode.rs @@ -17,6 +17,14 @@ pub(crate) const KEY: &str = "opencode"; pub(crate) const BINARY_CANDIDATES: &[&str] = &["opencode", "opencode-desktop"]; +// `passive_capture` flips on only when both `agent-adapters` and the +// `opencode` adapter feature are enabled. Spec 010 §17.2. +#[cfg(all(feature = "agent-adapters", feature = "opencode"))] +pub(crate) const CAPABILITIES: BackendCapabilities = BackendCapabilities::NONE + .with_structured_tools() + .with_passive_capture(); + +#[cfg(not(all(feature = "agent-adapters", feature = "opencode")))] pub(crate) const CAPABILITIES: BackendCapabilities = BackendCapabilities::NONE.with_structured_tools(); diff --git a/aikit-sdk/src/runner/capabilities.rs b/aikit-sdk/src/runner/capabilities.rs index 29131f9..237a05a 100644 --- a/aikit-sdk/src/runner/capabilities.rs +++ b/aikit-sdk/src/runner/capabilities.rs @@ -42,6 +42,10 @@ pub struct BackendCapabilities { pub subagents: bool, /// Emits context-compression events. pub context_compression: bool, + /// The Backend's on-disk session format is parseable by `aikit-session-capture`. + /// `false` ⇒ no adapter is registered for this Backend; passive capture + /// is unavailable, not merely empty. Spec 010. + pub passive_capture: bool, } impl BackendCapabilities { @@ -58,6 +62,7 @@ impl BackendCapabilities { server_tools: false, subagents: false, context_compression: false, + passive_capture: false, }; pub const fn with_bidirectional(mut self) -> Self { @@ -104,6 +109,12 @@ impl BackendCapabilities { self.context_compression = true; self } + /// Enable passive on-disk capture (spec 010). Flipped per-Backend only + /// when the matching `aikit-session-capture` feature is on. + pub const fn with_passive_capture(mut self) -> Self { + self.passive_capture = true; + self + } } impl Default for BackendCapabilities { diff --git a/aikit-session-capture/Cargo.toml b/aikit-session-capture/Cargo.toml new file mode 100644 index 0000000..a67ac82 --- /dev/null +++ b/aikit-session-capture/Cargo.toml @@ -0,0 +1,46 @@ +[package] +name = "aikit-session-capture" +version = "0.1.0" +edition = "2021" +license = "Apache-2.0" +description = "Session capture for AI coding tools (Claude Code, Codex, OpenCode) — normalized event parsing of on-disk session data with offset cursors, secret scrubbing, and SQLite/JSONL support. Spec 010." +repository = "https://github.com/goaikit/aikit" +keywords = ["ai", "agents", "capture", "observability", "sessions"] +categories = ["development-tools"] + +[features] +default = ["claudecode", "codex", "opencode"] +# One feature per adapter. Feature names match the module name (not the +# Backend name) so adding an adapter for a Backend whose SDK lives elsewhere +# doesn't collide. See spec 010 §17.1. +claudecode = [] +codex = [] +# OpenCode persists state in SQLite, not JSONL — pulls rusqlite only when on. +opencode = ["dep:rusqlite"] +# WSL2 ↔ Windows home enumeration. Opt-in; default HomeResolver is native only. +crossmount = [] +# cli-framework Command registrations for MCP exposure. +mcp-tools = ["dep:cli-framework"] +# notify-based fsnotify-equivalent watch driver. See spec 010 §14.5. +watcher = ["dep:notify", "dep:walkdir"] + +[dependencies] +async-trait = "0.1" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "2" +regex = "1" +dirs = "6" +chrono = { version = "0.4", features = ["serde"] } +tracing = "0.1" +anyhow = "1" +tokio = { version = "1", features = ["rt", "rt-multi-thread", "sync", "macros", "fs", "time"] } +rusqlite = { version = "0.39", features = ["bundled"], optional = true } +cli-framework = { git = "https://github.com/aroff/cli-framework", rev = "eaef0191a64faab76a27267567a0358b0c7af0e1", optional = true } +notify = { version = "6", optional = true } +walkdir = { version = "2", optional = true } + +[dev-dependencies] +tempfile = "3" +insta = { version = "1", features = ["json"] } +rusqlite = { version = "0.39", features = ["bundled"] } diff --git a/aikit-session-capture/src/adapter.rs b/aikit-session-capture/src/adapter.rs new file mode 100644 index 0000000..37e6f4b --- /dev/null +++ b/aikit-session-capture/src/adapter.rs @@ -0,0 +1,140 @@ +//! The `Adapter` trait: turns one AI tool's on-disk session data into +//! normalized events. +//! +//! See spec 010 §6. Implementations live in sibling modules +//! (`claudecode`, `codex`, `opencode`). + +use std::path::Path; + +use async_trait::async_trait; + +use crate::models::{CacheObservation, TokenEvent, ToolEvent, ToolKind}; + +/// Turns one AI coding tool's on-disk session data into normalized events. +/// +/// Implementations MUST (spec 010 §6): +/// - Scrub raw tool inputs before returning — no secrets escape the adapter. +/// The injected [`SecretScrubber`][crate::SecretScrubber] is the chokepoint. +/// - Emit deterministic `source_event_id`s so re-parsing is idempotent. +/// - Advance `new_offset` past the last fully-parsed byte (or, for SQLite +/// adapters, the last-consumed watermark), so the caller can persist it +/// and skip on the next call. +/// +/// The meaning of `from_offset` / `new_offset` is **adapter-defined**: byte +/// offsets for line-oriented JSONL adapters (Claude Code, Codex), or +/// `time_updated` watermarks for the SQLite-based OpenCode adapter. The +/// trait keeps the value opaque as `u64`; the host records `adapter_kind` +/// alongside the cursor so it can disambiguate on re-load. +#[async_trait] +pub trait Adapter: Send + Sync { + /// Stable identifier; one of [`ToolKind`]. Stored in the `tool` column. + fn kind(&self) -> ToolKind; + + /// Directories to monitor for new/changed session files. Paths that do + /// not exist are skipped at registry time — adapters should return their + /// canonical path regardless of installed state. + fn watch_paths(&self) -> Vec; + + /// Filters watcher events. True if `path` is a session file this adapter + /// should parse. + fn is_session_file(&self, path: &Path) -> bool; + + /// Parse `path` from `from_offset` to EOF (or, for SQLite adapters, + /// rows with `time_updated > from_offset`). + /// + /// Malformed records are skipped, not fatal — implementations advance + /// past them so repeated calls make progress. + async fn parse_session_file( + &self, + path: &Path, + from_offset: u64, + ) -> Result; +} + +/// Value returned by [`Adapter::parse_session_file`]. Mirrors +/// `superbased-observer/internal/adapter/adapter.go:40`. +#[derive(Debug, Clone, Default)] +pub struct ParseResult { + pub tool_events: Vec, + pub token_events: Vec, + pub cache_observations: Vec, + /// Offset to persist for the next call. For JSONL adapters this is a + /// byte offset; for SQLite adapters it is a `time_updated` watermark. + pub new_offset: u64, + pub warnings: Vec, + /// Ask the host to keep the file on the poll loop even when the offset + /// didn't advance (e.g. a foreign-mount SQLite file that hit + /// `SQLITE_IOERR_SHORT_READ` and may succeed on retry). + pub retry_suggested: bool, +} + +/// Non-fatal parse issues. Adapters skip the offending record, advance past +/// it, and emit a warning so the host can surface "watcher had trouble with +/// this file" without blocking ingestion. +#[non_exhaustive] +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ParseWarning { + MalformedLine { + line_no: u64, + reason: String, + }, + UnknownToolVariant { + raw: String, + }, + TruncatedRecord { + line_no: u64, + }, + /// OpenCode foreign-mount mirror retry — see spec 010 §12.3. + ForeignMountRetry { + path: String, + reason: String, + }, + Other { + message: String, + }, +} + +/// Fatal conditions only — things that prevent any further progress on this +/// file. Malformed individual records are [`ParseWarning`]s, not errors. +#[non_exhaustive] +#[derive(Debug, thiserror::Error)] +pub enum AdapterError { + #[error("io error reading {path}: {source}")] + Io { + path: std::path::PathBuf, + #[source] + source: std::io::Error, + }, + #[error("malformed session file {path} at byte {offset}: {reason}")] + Malformed { + path: std::path::PathBuf, + offset: u64, + reason: String, + }, + #[error("offset {requested} is past EOF ({file_size}) for {path}")] + OffsetPastEof { + path: std::path::PathBuf, + requested: u64, + file_size: u64, + }, + #[error("scrubber pattern invalid: {0}")] + ScrubberPattern(#[from] regex::Error), + #[error("home resolution failed: {0}")] + HomeResolution(String), + #[cfg(feature = "opencode")] + #[error("sqlite open failed for {path}: {source}")] + SqliteOpen { + path: std::path::PathBuf, + #[source] + source: rusqlite::Error, + }, + #[cfg(feature = "opencode")] + #[error("foreign-mount mirror failed for {path}: {reason}")] + ForeignMountMirror { + path: std::path::PathBuf, + reason: String, + }, + #[error("other: {0}")] + Other(#[from] anyhow::Error), +} diff --git a/aikit-session-capture/src/claudecode/mod.rs b/aikit-session-capture/src/claudecode/mod.rs new file mode 100644 index 0000000..0121f07 --- /dev/null +++ b/aikit-session-capture/src/claudecode/mod.rs @@ -0,0 +1,123 @@ +//! Claude Code adapter: parses `~/.claude/projects//.jsonl`. +//! +//! See spec 010 §12.1. The adapter reads the file from a byte offset, +//! streams line-by-line, and emits normalized `ToolEvent` / `TokenEvent` / +//! `CacheObservation` rows. +//! +//! Watch-path resolution priority (mirrors observer's `codex/adapter.go:60`): +//! 1. Explicit override (CLI flag / config / test fixture). +//! 2. `CLAUDE_HOME` env var. +//! 3. Crossmount-resolved `$HOME/.claude/projects`. + +mod transcript; + +use std::path::{Path, PathBuf}; + +use async_trait::async_trait; + +use crate::adapter::{Adapter, AdapterError, ParseResult}; +use crate::homes::HomeResolver; +use crate::models::ToolKind; +use crate::scrub::SecretScrubber; + +pub struct ClaudeCodeAdapter { + scrubber: SecretScrubber, + homes: Vec, + override_root: Option, +} + +impl ClaudeCodeAdapter { + /// Production constructor: uses `DefaultHomeResolver` + default scrubber. + pub fn new() -> Self { + Self { + scrubber: SecretScrubber::default(), + homes: crate::homes::DefaultHomeResolver.homes(), + override_root: None, + } + } + + /// Test / config constructor: caller supplies a scrubber + home resolver. + pub fn with(scrubber: SecretScrubber, homes: Vec) -> Self { + Self { + scrubber, + homes, + override_root: None, + } + } + + /// Explicit watch-root override. When set, `watch_paths()` returns only + /// this path — env var and crossmount expansion are suppressed. Used by + /// tests and the `[adapter.claudecode] root = "..."` config field. + pub fn with_override_root(mut self, root: PathBuf) -> Self { + self.override_root = Some(root); + self + } +} + +impl Default for ClaudeCodeAdapter { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl Adapter for ClaudeCodeAdapter { + fn kind(&self) -> ToolKind { + ToolKind::ClaudeCode + } + + fn watch_paths(&self) -> Vec { + if let Some(root) = &self.override_root { + return vec![root.clone()]; + } + // CLAUDE_HOME (single explicit path; suppresses crossmount expansion). + if let Ok(env_home) = std::env::var("CLAUDE_HOME") { + if !env_home.is_empty() { + return vec![PathBuf::from(env_home).join("projects")]; + } + } + // Default: every resolved home gets a `.claude/projects` candidate. + // The registry's `detected()` filter skips the ones that don't exist. + self.homes + .iter() + .map(|h| h.path.join(".claude").join("projects")) + .collect() + } + + fn is_session_file(&self, path: &Path) -> bool { + // Claude Code session files are `.jsonl` under a + // watch root. Subagent transcripts live under `/ + // subagents/agent-*.jsonl` and share the parent session_id — they + // also match this predicate. + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { + return false; + }; + if !name.ends_with(".jsonl") { + return false; + } + // Constrain to a watch path so a random `.jsonl` elsewhere on disk + // doesn't get claimed. + let abs = path.to_path_buf(); + self.watch_paths().iter().any(|root| abs.starts_with(root)) + } + + async fn parse_session_file( + &self, + path: &Path, + from_offset: u64, + ) -> Result { + // Read the whole file. Phase 2 simplicity: the file is read into + // memory; for very large transcripts a streaming BufReader would + // reduce peak memory, but observer's transcripts are bounded (~MB + // range, the adapter.go comment at line 437 calls this out). A future + // optimization can swap to memmap or BufRead without changing the + // trait contract. + let bytes = tokio::fs::read(path) + .await + .map_err(|source| AdapterError::Io { + path: path.to_path_buf(), + source, + })?; + transcript::parse(path, &bytes, from_offset, &self.scrubber) + } +} diff --git a/aikit-session-capture/src/claudecode/transcript.rs b/aikit-session-capture/src/claudecode/transcript.rs new file mode 100644 index 0000000..80adc3a --- /dev/null +++ b/aikit-session-capture/src/claudecode/transcript.rs @@ -0,0 +1,918 @@ +//! Claude Code JSONL transcript parser. +//! +//! Reads `~/.claude/projects//.jsonl` line-by-line, +//! resumable from a byte offset, and emits normalized `ToolEvent` / +//! `TokenEvent` rows. See spec 010 §12.1. +//! +//! Reference: `superbased-observer/internal/adapter/claudecode/adapter.go` +//! (1739 lines, many edge cases). This Phase 2 MVP covers the core flow: +//! user/assistant records, tool_use↔tool_result pairing, usage envelopes +//! with msg-id dedup, and sidechain skipping. Compact_boundary / +//! agent-name / permission-mode / tier-2 cache observations land later. + +use std::path::Path; + +use serde::Deserialize; +use serde_json::Value; + +use crate::adapter::{AdapterError, ParseResult, ParseWarning}; +use crate::models::{ + ActionKind, ActionStatus, CacheObservation, CaptureSource, TokenEvent, ToolEvent, ToolKind, +}; +use crate::scrub::SecretScrubber; + +/// Parse Claude Code JSONL bytes from `from_offset` to EOF. +/// +/// The caller (the Adapter impl) supplies the path (for `source_file` stamps), +/// the byte offset to resume from, and the shared scrubber. Returns the +/// normalized events and the new byte offset to persist. +pub(crate) fn parse( + path: &Path, + bytes: &[u8], + from_offset: u64, + scrubber: &SecretScrubber, +) -> Result { + let start = from_offset as usize; + if start > bytes.len() { + return Err(AdapterError::OffsetPastEof { + path: path.to_path_buf(), + requested: from_offset, + file_size: bytes.len() as u64, + }); + } + let tail = &bytes[start..]; + + let mut res = ParseResult { + new_offset: from_offset, + ..Default::default() + }; + // toolu_id → index into res.tool_events, so a later tool_result block can + // back-fill output / status on the matching tool_use event. + let mut pending: std::collections::HashMap = std::collections::HashMap::new(); + // msg.id → index into res.token_events. One API call emits N JSONL records + // (one per content block) sharing the same msg.id and a progressing + // cumulative usage envelope. Last (highest output_tokens) wins. + let mut msg_id_to_idx: std::collections::HashMap = + std::collections::HashMap::new(); + + let mut cursor: usize = 0; + let mut line_no: u64 = 0; + while cursor < tail.len() { + // Find the next '\n'. Match the observer fix: include the terminator + // length (handles CRLF too — ReadString in Go includes '\n', and we + // count every byte consumed so the cursor never strands short of EOF). + let nl = match tail[cursor..].iter().position(|&b| b == b'\n') { + Some(i) => cursor + i + 1, + None => { + // Partial trailing line (no terminating '\n'). The writer may + // still be appending — defer it to the next poll, do NOT + // advance the cursor past it. + break; + } + }; + line_no += 1; + let raw_line = &tail[cursor..nl]; + cursor = nl; + // Commit NewOffset after every complete line we consume, including + // empty / malformed ones — otherwise the watcher repolls forever + // (observer invariant, adapter.go:468). + res.new_offset = start as u64 + cursor as u64; + + // Strip trailing \r\n / \n. + let trimmed = raw_line + .strip_suffix(b"\r\n") + .unwrap_or_else(|| raw_line.strip_suffix(b"\n").unwrap_or(raw_line)); + let trimmed = std::str::from_utf8(trimmed).unwrap_or(""); + let trimmed = trimmed.trim(); + if trimmed.is_empty() { + continue; + } + + // Parse the record. + let rec: RawLine = match serde_json::from_str(trimmed) { + Ok(r) => r, + Err(e) => { + res.warnings.push(ParseWarning::MalformedLine { + line_no, + reason: format!("JSON parse: {e}"), + }); + continue; + } + }; + + // Skip sidechain (sub-agent) lines — they share the parent's + // session_id but belong to a different thread. The parent's tool_use + // that spawned the subagent is captured separately as ActionKind::Subagent. + if rec.is_sidechain { + continue; + } + + // API error envelopes: type="system", subtype="api_error". Map to a + // ToolEvent with kind=Other and status=Failure so failures show up on + // the timeline alongside tool calls. (Observer has a richer + // ActionAPIError type; the spec 010 taxonomy collapses to Other.) + if rec.r#type == "system" && rec.subtype.as_deref() == Some("api_error") { + if let Some(ev) = build_api_error_event(path, &rec, line_no) { + res.tool_events.push(ev); + } + continue; + } + + // Non-message records we don't yet handle (compact_boundary, + // turn_duration, agent-name, permission-mode) are skipped cleanly — + // observer's V7d audit added those; spec 010 Phase 2 MVP leaves them + // for a later phase. No warning, they're not malformed. + if rec.message.is_none() { + continue; + } + let msg: RawMessage = match serde_json::from_value(rec.message.unwrap()) { + Ok(m) => m, + Err(e) => { + res.warnings.push(ParseWarning::MalformedLine { + line_no, + reason: format!("message field: {e}"), + }); + continue; + } + }; + + let ts_ms = parse_ts_ms(rec.timestamp.as_deref().unwrap_or("")); + // git_root resolution is the host's concern (the trait is + // storage-agnostic). For Phase 2 we stamp the cwd as-is and let the + // EventStore layer resolve git roots. Observer does it inside the + // adapter because it has a git.Resolve helper; we keep the crate + // self-contained and stamp the literal cwd. + let git_root = rec.cwd.as_deref().map(std::path::PathBuf::from); + + // Usage envelope → TokenEvent (assistant turns only). + if let Some(usage) = &msg.usage { + // Drop Claude Code's synthetic placeholder rows (compaction / + // subagent stitching; the live install emits zero usage anyway). + if msg.model.as_deref() == Some("") { + continue; + } + // Dedup key: msg.id when present (one API call = N JSONL records + // sharing it); fall back to the record's uuid. + let event_id = msg.id.clone().filter(|s| !s.is_empty()).unwrap_or_else(|| { + rec.uuid + .clone() + .unwrap_or_else(|| format!("line:{line_no}")) + }); + let cache_creation_1h = usage + .cache_creation + .as_ref() + .map(|c| c.ephemeral_1h_input_tokens.unwrap_or(0)) + .unwrap_or(0); + let cache_creation_total = usage.cache_creation_input_tokens.unwrap_or(0); + // Per spec 010 data-model.md: absence means "not reported", never + // "zero". We only get an integer here (serde default on miss is + // None), so this is preserved naturally. + let ev = TokenEvent { + source_event_id: format!("claude_code:{}", event_id), + session_id: rec.session_id.clone().unwrap_or_default(), + tool: ToolKind::ClaudeCode, + model: msg.model.clone(), + request_id: msg.id.clone().filter(|s| !s.is_empty()), + input_tokens: usage.input_tokens, + cache_read_tokens: usage.cache_read_input_tokens, + cache_creation_tokens: if cache_creation_total > 0 { + Some(cache_creation_total) + } else { + usage + .cache_creation + .as_ref() + .and_then(|c| c.ephemeral_5m_input_tokens) + .map(|m| m + cache_creation_1h) + }, + cache_creation_1h_tokens: if cache_creation_1h > 0 { + Some(cache_creation_1h) + } else { + None + }, + output_tokens: usage.output_tokens, + reasoning_tokens: None, + captured_at_ms: ts_ms.unwrap_or(0), + captured_via: CaptureSource::Transcript, + }; + if let Some(id) = &msg.id { + if !id.is_empty() { + if let Some(&idx) = msg_id_to_idx.get(id) { + // Streaming usage progresses monotonically — keep the + // later record (highest output_tokens). Don't `continue` + // the outer loop; content blocks below are still + // distinct and must be processed. + if ev.output_tokens.unwrap_or(0) + >= res.token_events[idx].output_tokens.unwrap_or(0) + { + res.token_events[idx] = ev; + } + } else { + msg_id_to_idx.insert(id.clone(), res.token_events.len()); + res.token_events.push(ev); + } + // Also emit a Tier-2 CacheObservation so the cachetrack + // module (future) can attribute invalidations. Spec 010 + // data-model.md. + res.cache_observations.push(CacheObservation { + source_event_id: format!("cachetrack:{}", id), + session_id: rec.session_id.clone().unwrap_or_default(), + tool: ToolKind::ClaudeCode, + cache_read_input_tokens: usage.cache_read_input_tokens, + cache_creation_input_tokens: if cache_creation_total > 0 { + Some(cache_creation_total) + } else { + None + }, + cache_creation_1h_input_tokens: if cache_creation_1h > 0 { + Some(cache_creation_1h) + } else { + None + }, + assistant_blocks_hash: None, // Phase 2 leaves this for the cachetrack module. + tools_changed: Vec::new(), + observed_at_ms: ts_ms.unwrap_or(0), + }); + continue; + } + } + res.token_events.push(ev); + } + + // Content blocks: tool_use, tool_result, text. + let blocks = decode_content(&msg.content); + for (block_idx, block) in blocks.iter().enumerate() { + match block.r#type.as_str() { + "text" => { + // User text → user_prompt ToolEvent. Assistant text → Think event. + if msg.role.as_deref() == Some("assistant") { + let text = block.text.as_deref().unwrap_or("").trim(); + if !text.is_empty() { + let event_id = rec + .uuid + .clone() + .unwrap_or_else(|| format!("line:{line_no}:block:{block_idx}")); + res.tool_events.push(ToolEvent { + source_event_id: format!("claude_code:{event_id}"), + source_file: path.to_path_buf(), + session_id: rec.session_id.clone().unwrap_or_default(), + tool: ToolKind::ClaudeCode, + kind: ActionKind::Think, + target: Some(truncate_str(text, 200).to_string()), + input: Some(scrubber.scrub(text)), + output: None, + status: ActionStatus::Success, + error_message: None, + started_at_ms: ts_ms, + duration_ms: None, + git_root: git_root.clone(), + metadata: serde_json::Value::Null, + }); + } + } + } + "tool_use" => { + let event_id = block.id.clone().unwrap_or_else(|| { + format!("{}:{}", rec.uuid.as_deref().unwrap_or("uuid"), block_idx) + }); + let raw_input = block + .input + .as_ref() + .map(|v| v.to_string()) + .unwrap_or_default(); + let kind = map_action_kind(block.name.as_deref().unwrap_or("")); + let target = extract_target( + block.name.as_deref().unwrap_or(""), + block.input.as_ref(), + scrubber, + ); + let scrubbed_input = if raw_input.is_empty() { + None + } else { + Some(scrubber.scrub(&raw_input)) + }; + let ev = ToolEvent { + source_event_id: format!("claude_code:{event_id}"), + source_file: path.to_path_buf(), + session_id: rec.session_id.clone().unwrap_or_default(), + tool: ToolKind::ClaudeCode, + kind, + target, + input: scrubbed_input, + output: None, // filled in when the matching tool_result arrives + status: ActionStatus::Success, // default; flipped to Failure on is_error + error_message: None, + started_at_ms: ts_ms, + duration_ms: None, + git_root: git_root.clone(), + metadata: serde_json::json!({ + "raw_tool_name": block.name.as_deref().unwrap_or(""), + "model": msg.model.as_deref().unwrap_or(""), + }), + }; + let idx = res.tool_events.len(); + res.tool_events.push(ev); + if let Some(id) = &block.id { + pending.insert(id.clone(), idx); + } + } + "tool_result" => { + // Back-fill the matching tool_use event. + if let Some(tu_id) = &block.tool_use_id { + if let Some(&idx) = pending.get(tu_id) { + let body = decode_result_content(block.content.as_ref()); + let scrubbed = scrubber.scrub(&body); + res.tool_events[idx].output = Some(scrubbed.clone()); + if block.is_error.unwrap_or(false) { + res.tool_events[idx].status = ActionStatus::Failure; + res.tool_events[idx].error_message = + Some(truncate_str(&scrubbed, 500).to_string()); + } + } + } + } + _ => {} + } + } + } + + Ok(res) +} + +/// Top-level Claude Code JSONL record. See observer's `rawLine` struct. +/// Fields absent on some records are `Option<...>`; serde leaves them None. +#[derive(Debug, Deserialize)] +struct RawLine { + #[serde(default, rename = "sessionId")] + session_id: Option, + /// Present in real transcripts; the Phase 2 MVP doesn't yet surface it + /// on a ToolEvent (no field in spec 010's data-model) but parsers that + /// follow (Phase 3 cachetrack) will consume it. Kept to avoid silently + /// dropping the field. + #[serde(default, rename = "gitBranch")] + #[allow(dead_code)] + git_branch: Option, + #[serde(default)] + cwd: Option, + #[serde(default)] + uuid: Option, + #[serde(default)] + timestamp: Option, + #[serde(default, rename = "type")] + r#type: String, + #[serde(default)] + subtype: Option, + #[serde(default)] + message: Option, + // System/api_error records carry an `error` envelope instead of `message`. + #[serde(default)] + error: Option, + #[serde(default, rename = "isSidechain")] + is_sidechain: bool, +} + +/// Inner `message` object on user/assistant records. +#[derive(Debug, Deserialize)] +struct RawMessage { + #[serde(default)] + id: Option, + #[serde(default)] + role: Option, + #[serde(default)] + model: Option, + #[serde(default)] + content: Option, + #[serde(default)] + usage: Option, +} + +#[derive(Debug, Deserialize)] +struct RawUsage { + #[serde(default)] + input_tokens: Option, + #[serde(default)] + output_tokens: Option, + #[serde(default)] + cache_creation_input_tokens: Option, + #[serde(default)] + cache_read_input_tokens: Option, + #[serde(default)] + cache_creation: Option, +} + +#[derive(Debug, Deserialize)] +struct CacheCreation { + #[serde(default, rename = "ephemeral_5m_input_tokens")] + ephemeral_5m_input_tokens: Option, + #[serde(default, rename = "ephemeral_1h_input_tokens")] + ephemeral_1h_input_tokens: Option, +} + +/// Content block — tool_use / tool_result / text / etc. +#[derive(Debug, Deserialize)] +struct ContentBlock { + #[serde(default, rename = "type")] + r#type: String, + #[serde(default)] + text: Option, + #[serde(default)] + id: Option, + #[serde(default)] + name: Option, + #[serde(default)] + input: Option, + #[serde(default, rename = "tool_use_id")] + tool_use_id: Option, + #[serde(default)] + content: Option, + #[serde(default, rename = "is_error")] + is_error: Option, +} + +/// `content` field is either a JSON array of blocks or a bare string (short +/// text-only messages). Returns an empty vec on decode failure. +fn decode_content(raw: &Option) -> Vec { + let Some(v) = raw else { + return Vec::new(); + }; + match v { + Value::Array(_) => serde_json::from_value(v.clone()).unwrap_or_default(), + Value::String(s) => vec![ContentBlock { + r#type: "text".into(), + text: Some(s.clone()), + id: None, + name: None, + input: None, + tool_use_id: None, + content: None, + is_error: None, + }], + _ => Vec::new(), + } +} + +/// Render a tool_result content payload (string or block array) as plain text. +/// Mirrors observer's `flattenResult`. +fn decode_result_content(raw: Option<&Value>) -> String { + let Some(v) = raw else { + return String::new(); + }; + match v { + Value::String(s) => s.clone(), + Value::Array(arr) => { + let mut parts = Vec::new(); + for blk in arr { + if blk.get("type").and_then(|t| t.as_str()) == Some("text") { + if let Some(t) = blk.get("text").and_then(|t| t.as_str()) { + let trimmed = t.trim(); + if !trimmed.is_empty() { + parts.push(trimmed.to_string()); + } + } + } + } + parts.join(" ") + } + _ => String::new(), + } +} + +/// Claude Code tool name → normalized ActionKind. Mirrors observer's +/// `actionMap` (trimmed to spec 010's taxonomy — no separate Search/Files +/// split, no native-tools distinction). +fn map_action_kind(name: &str) -> ActionKind { + match name { + "Read" => ActionKind::Read, + "Write" => ActionKind::Write, + "Edit" | "MultiEdit" | "NotebookEdit" => ActionKind::Edit, + "Bash" | "PowerShell" | "powershell" | "pwsh" | "cmd" | "cmd.exe" | "sh" => { + ActionKind::Bash + } + "Grep" => ActionKind::Grep, + "Glob" => ActionKind::Glob, + "WebSearch" => ActionKind::WebSearch, + "WebFetch" => ActionKind::WebFetch, + "Agent" => ActionKind::Subagent, + // MCP tools: mcp____ + n if n.starts_with("mcp__") => ActionKind::Mcp, + _ => ActionKind::Other, + } +} + +/// Extract the "target" string for one tool call — the path/command/pattern +/// the action touched. Mirrors observer's `extractTarget`. The target is +/// pre-scrubbed for Bash (commands often carry inline secrets). +fn extract_target( + tool_name: &str, + input: Option<&Value>, + scrubber: &SecretScrubber, +) -> Option { + let input = input?; + let pick = |key: &str| -> Option { + input + .get(key) + .and_then(|v| v.as_str()) + .map(|s| s.to_string()) + }; + let target = match tool_name { + "Read" | "Write" | "Edit" | "MultiEdit" | "NotebookEdit" => { + pick("file_path").or_else(|| pick("notebook_uri")) + } + "Bash" | "PowerShell" | "powershell" | "pwsh" | "cmd" | "cmd.exe" | "sh" => { + pick("command").map(|c| scrubber.scrub(&c)) + } + "Grep" => pick("pattern"), + "Glob" => pick("pattern"), + "WebSearch" => pick("query"), + "WebFetch" => pick("url"), + _ => None, + }; + target.map(|t| truncate_str(&t, 200).to_string()) +} + +/// Build a ToolEvent from a system/api_error record. Mirrors observer's +/// `buildAPIErrorEvent` (simplified — no nested-error-chain walk for Phase 2; +/// we look one level deep). +fn build_api_error_event(path: &Path, rec: &RawLine, line_no: u64) -> Option { + let err = rec.error.as_ref()?; + let request_id = err.get("requestID").and_then(|v| v.as_str()); + let message = err + .get("error") + .and_then(|e| e.get("message")) + .and_then(|m| m.as_str()) + .or_else(|| err.get("message").and_then(|m| m.as_str())); + let err_type = err + .get("error") + .and_then(|e| e.get("type")) + .and_then(|t| t.as_str()) + .filter(|s| !s.is_empty() && *s != "error") // generic "error" → fall back to outer + .or_else(|| err.get("type").and_then(|t| t.as_str())) + .unwrap_or("api_error"); + + // Need at least one identifier to emit a row. + if request_id.is_none() && message.is_none() { + return None; + } + + let event_id = rec + .uuid + .clone() + .unwrap_or_else(|| format!("api_err:{line_no}")); + let ts_ms = parse_ts_ms(&rec.timestamp.clone().unwrap_or_default()); + Some(ToolEvent { + source_event_id: format!("claude_code:{event_id}"), + source_file: path.to_path_buf(), + session_id: rec.session_id.clone().unwrap_or_default(), + tool: ToolKind::ClaudeCode, + kind: ActionKind::Other, + target: request_id.map(|s| s.to_string()), + input: None, + output: message.map(|s| s.to_string()), + status: ActionStatus::Failure, + error_message: message.map(|s| truncate_str(s, 500).to_string()), + started_at_ms: ts_ms, + duration_ms: None, + git_root: rec.cwd.as_deref().map(std::path::PathBuf::from), + metadata: serde_json::json!({ "error_type": err_type }), + }) +} + +fn parse_ts_ms(s: &str) -> Option { + if s.is_empty() { + return None; + } + // RFC3339 / RFC3339Nano. chrono's parse handles both. + chrono::DateTime::parse_from_rfc3339(s) + .ok() + .map(|dt| dt.timestamp_millis()) +} + +fn truncate_str(s: &str, max: usize) -> &str { + if s.len() <= max { + s + } else { + // Walk to char boundary at or before `max`. + let mut end = max; + while end > 0 && !s.is_char_boundary(end) { + end -= 1; + } + &s[..end] + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture(name: &str) -> Vec { + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + let path = format!("{manifest_dir}/tests/fixtures/claudecode/{name}"); + std::fs::read(&path).unwrap_or_else(|e| panic!("fixture {name} not found at {path}: {e}")) + } + + fn scrubber() -> SecretScrubber { + SecretScrubber::default() + } + + // ---- Task 10: core parsing ----------------------------------------- + + #[test] + fn parses_simple_session_into_events() { + let bytes = fixture("simple-session.jsonl"); + let path = Path::new("/tmp/sess-001.jsonl"); + let res = parse(path, &bytes, 0, &scrubber()).unwrap(); + + // The fixture has 5 lines: + // 1. user text + // 2. assistant with text + tool_use(Read) + usage + // 3. user with tool_result for Read + // 4. assistant with tool_use(Bash) + // 5. user with tool_result for Bash (error) + // Expect: + // - ≥1 TokenEvent (line 2 has usage) + // - ≥2 ToolEvents with kind Read/Bash (the two tool_use blocks) + // - the Bash result is_error=true → status=Failure + let tool_kinds: Vec = res.tool_events.iter().map(|e| e.kind).collect(); + assert!( + tool_kinds.contains(&ActionKind::Read), + "expected a Read ToolEvent, got: {tool_kinds:?}" + ); + assert!( + tool_kinds.contains(&ActionKind::Bash), + "expected a Bash ToolEvent, got: {tool_kinds:?}" + ); + assert!( + !res.token_events.is_empty(), + "expected at least one TokenEvent from the usage envelope" + ); + // Bash result was an error → status Failure on the matching event. + let bash_ev = res + .tool_events + .iter() + .find(|e| e.kind == ActionKind::Bash) + .expect("Bash event exists"); + assert_eq!(bash_ev.status, ActionStatus::Failure); + assert!(bash_ev.error_message.is_some()); + // Read event got its output back-filled from the tool_result. + let read_ev = res + .tool_events + .iter() + .find(|e| e.kind == ActionKind::Read) + .expect("Read event exists"); + assert!( + read_ev.output.is_some(), + "Read tool_result should back-fill output" + ); + } + + #[test] + fn parses_multi_tool_turn_pairs_results_correctly() { + let bytes = fixture("multi-tool-turn.jsonl"); + let path = Path::new("/tmp/sess-002.jsonl"); + let res = parse(path, &bytes, 0, &scrubber()).unwrap(); + + // 3 tool_use blocks: Grep, Glob, WebSearch. All paired with results. + let grep_ev = res + .tool_events + .iter() + .find(|e| e.kind == ActionKind::Grep) + .expect("Grep event"); + assert_eq!(grep_ev.target.as_deref(), Some("func Handle")); + assert_eq!(grep_ev.output.as_deref(), Some("found 12 matches")); + + let glob_ev = res + .tool_events + .iter() + .find(|e| e.kind == ActionKind::Glob) + .expect("Glob event"); + assert_eq!(glob_ev.target.as_deref(), Some("**/*.go")); + + let web_ev = res + .tool_events + .iter() + .find(|e| e.kind == ActionKind::WebSearch) + .expect("WebSearch event"); + assert_eq!(web_ev.status, ActionStatus::Failure); + assert_eq!(web_ev.error_message.as_deref(), Some("network error")); + } + + #[test] + fn parses_api_errors_as_failure_events() { + let bytes = fixture("api-error.jsonl"); + let path = Path::new("/tmp/sess-err.jsonl"); + let res = parse(path, &bytes, 0, &scrubber()).unwrap(); + + // 3 api_error records → 3 ToolEvents with status=Failure. + let failures: Vec<&ToolEvent> = res + .tool_events + .iter() + .filter(|e| e.status == ActionStatus::Failure) + .collect(); + assert_eq!(failures.len(), 3, "expected 3 API error events"); + // The 400 / 429 / 529 trio carry distinct error types. + let error_types: Vec<&str> = failures + .iter() + .map(|e| { + e.metadata + .get("error_type") + .and_then(|v| v.as_str()) + .unwrap_or("") + }) + .collect(); + assert!(error_types.contains(&"invalid_request_error")); + assert!(error_types.contains(&"rate_limit_error")); + assert!(error_types.contains(&"overloaded_error")); + } + + #[test] + fn malformed_line_is_skipped_not_fatal() { + let bytes = fixture("malformed-line.jsonl"); + let path = Path::new("/tmp/sess-003.jsonl"); + let res = parse(path, &bytes, 0, &scrubber()).unwrap(); + + // Line 1 (Read tool_use) and line 3 (Bash tool_use) parse; line 2 + // is "this is not valid json at all }}" and should produce a warning. + assert!( + !res.warnings.is_empty(), + "expected a ParseWarning for the malformed line" + ); + let tool_kinds: Vec = res.tool_events.iter().map(|e| e.kind).collect(); + assert!(tool_kinds.contains(&ActionKind::Read)); + assert!(tool_kinds.contains(&ActionKind::Bash)); + } + + #[test] + fn msg_id_dedup_chooses_highest_output_tokens() { + let bytes = fixture("multi-block-dedup.jsonl"); + let path = Path::new("/tmp/sess-dedup.jsonl"); + let res = parse(path, &bytes, 0, &scrubber()).unwrap(); + + // The fixture emits multiple JSONL records sharing one msg_id with + // progressing usage. The TokenEvent count for that msg_id MUST be 1, + // and it MUST carry the highest output_tokens seen. + let mut by_id: std::collections::HashMap<&str, &TokenEvent> = + std::collections::HashMap::new(); + for ev in &res.token_events { + let key = ev + .request_id + .as_deref() + .unwrap_or(ev.source_event_id.as_str()); + by_id.insert(key, ev); + } + // Every distinct msg_id appears exactly once. + for ev in by_id.values() { + let count = res + .token_events + .iter() + .filter(|e| e.request_id == ev.request_id) + .count(); + assert_eq!( + count, + 1, + "msg_id {} should dedup to one event", + ev.request_id.as_deref().unwrap_or("?") + ); + } + } + + // ---- Task 11: invariants ------------------------------------------- + + #[test] + fn parse_twice_from_zero_produces_identical_source_event_ids() { + // The idempotency invariant: a full re-walk produces byte-identical + // source_event_ids, so the host's upsert is a no-op for seen rows. + let bytes = fixture("simple-session.jsonl"); + let path = Path::new("/tmp/sess-idem.jsonl"); + let r1 = parse(path, &bytes, 0, &scrubber()).unwrap(); + let r2 = parse(path, &bytes, 0, &scrubber()).unwrap(); + let ids1: std::collections::HashSet<&str> = r1 + .tool_events + .iter() + .map(|e| e.source_event_id.as_str()) + .collect(); + let ids2: std::collections::HashSet<&str> = r2 + .tool_events + .iter() + .map(|e| e.source_event_id.as_str()) + .collect(); + assert_eq!(ids1, ids2, "idempotency: source_event_id sets must match"); + let token_ids1: std::collections::HashSet<&str> = r1 + .token_events + .iter() + .map(|e| e.source_event_id.as_str()) + .collect(); + let token_ids2: std::collections::HashSet<&str> = r2 + .token_events + .iter() + .map(|e| e.source_event_id.as_str()) + .collect(); + assert_eq!( + token_ids1, token_ids2, + "idempotency: token_event id sets must match" + ); + } + + #[test] + fn parse_from_nonzero_offset_skips_consumed_bytes() { + let bytes = fixture("simple-session.jsonl"); + let path = Path::new("/tmp/sess-offset.jsonl"); + // First parse consumes everything. + let r1 = parse(path, &bytes, 0, &scrubber()).unwrap(); + // Second parse from the advanced offset should emit nothing new. + let r2 = parse(path, &bytes, r1.new_offset, &scrubber()).unwrap(); + assert!(r2.tool_events.is_empty()); + assert!(r2.token_events.is_empty()); + assert_eq!(r2.new_offset, r1.new_offset); + } + + #[test] + fn offset_advances_past_every_complete_line_including_malformed() { + // The watcher-repoll-forever bug class: if the cursor doesn't advance + // past a malformed line (or an empty trailing line), the poll loops. + let bytes = fixture("malformed-line.jsonl"); + let path = Path::new("/tmp/sess-adv.jsonl"); + let res = parse(path, &bytes, 0, &scrubber()).unwrap(); + // NewOffset must equal total bytes consumed (everything up to the + // last '\n' in the fixture). + let last_nl = bytes + .iter() + .rposition(|&b| b == b'\n') + .map(|i| i as u64 + 1) + .unwrap_or(0); + assert_eq!( + res.new_offset, last_nl, + "cursor must advance past malformed line" + ); + } + + #[test] + fn crlf_line_endings_consume_both_bytes() { + // A line written with '\r\n' must consume both bytes for the cursor, + // otherwise the watcher strands one byte short of EOF per CRLF line. + let bytes = b"{\"type\":\"user\",\"sessionId\":\"s\",\"uuid\":\"u\",\"timestamp\":\"2026-01-01T00:00:00Z\",\"message\":{\"role\":\"user\",\"content\":\"hi\"}}\r\n"; + let path = Path::new("/tmp/crlf.jsonl"); + let res = parse(path, bytes, 0, &scrubber()).unwrap(); + assert_eq!( + res.new_offset, + bytes.len() as u64, + "CRLF must consume both bytes" + ); + } + + #[test] + fn secrets_are_scrubbed_from_input_and_output() { + let bytes = fixture("with-secrets.jsonl"); + let path = Path::new("/tmp/sess-scrub.jsonl"); + let res = parse(path, &bytes, 0, &scrubber()).unwrap(); + + // Every input + output field must be free of the known secret patterns. + let forbidden = [ + "AKIAIOSFODNN7EXAMPLE", + "ghp_0123456789012345678901234567890abcdefgh", + "eyJhbGciOiJIUzI1NiJ9", + "sk-ant-api03-abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJ", + "sk-proj-abcdef1234567890ABCDEFGHIJabcdefghij", + "secretpass123", + ]; + for ev in &res.tool_events { + if let Some(s) = &ev.input { + for f in forbidden { + assert!( + !s.contains(f), + "SCRUB FAILURE: input contains '{f}'\n full: {s}" + ); + } + } + if let Some(s) = &ev.output { + for f in forbidden { + assert!( + !s.contains(f), + "SCRUB FAILURE: output contains '{f}'\n full: {s}" + ); + } + } + if let Some(s) = &ev.target { + for f in forbidden { + assert!( + !s.contains(f), + "SCRUB FAILURE: target contains '{f}'\n full: {s}" + ); + } + } + } + } + + #[test] + fn concatenated_records_on_one_line_treated_as_malformed() { + // The observer has a recovery path for writer-corrupted "two JSON + // records on one physical line" patterns. Phase 2 MVP treats it as + // one malformed line + warning, NOT a fatal error. + let bytes = fixture("concatenated-records.jsonl"); + let path = Path::new("/tmp/sess-concat.jsonl"); + let res = parse(path, &bytes, 0, &scrubber()).unwrap(); + // At least one warning emitted, parsing did not abort. + assert!( + !res.warnings.is_empty() || !res.tool_events.is_empty(), + "concatenated records should produce warnings or partial events, not a silent no-op" + ); + } +} diff --git a/aikit-session-capture/src/codex/mod.rs b/aikit-session-capture/src/codex/mod.rs new file mode 100644 index 0000000..652b609 --- /dev/null +++ b/aikit-session-capture/src/codex/mod.rs @@ -0,0 +1,104 @@ +//! Codex CLI adapter: parses `~/.codex/sessions/rollout-*.jsonl`. +//! +//! See spec 010 §12.2. Watch-path resolution priority mirrors Claude Code: +//! 1. Explicit override. +//! 2. `CODEX_HOME` env var (single explicit path; suppresses crossmount). +//! 3. Crossmount-resolved `$HOME/.codex/sessions`. + +mod transcript; + +use std::path::{Path, PathBuf}; + +use async_trait::async_trait; + +use crate::adapter::{Adapter, AdapterError, ParseResult}; +use crate::homes::HomeResolver; +use crate::models::ToolKind; +use crate::scrub::SecretScrubber; + +pub struct CodexAdapter { + scrubber: SecretScrubber, + homes: Vec, + override_root: Option, +} + +impl CodexAdapter { + pub fn new() -> Self { + Self { + scrubber: SecretScrubber::default(), + homes: crate::homes::DefaultHomeResolver.homes(), + override_root: None, + } + } + + pub fn with(scrubber: SecretScrubber, homes: Vec) -> Self { + Self { + scrubber, + homes, + override_root: None, + } + } + + pub fn with_override_root(mut self, root: PathBuf) -> Self { + self.override_root = Some(root); + self + } +} + +impl Default for CodexAdapter { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl Adapter for CodexAdapter { + fn kind(&self) -> ToolKind { + ToolKind::Codex + } + + fn watch_paths(&self) -> Vec { + if let Some(root) = &self.override_root { + return vec![root.clone()]; + } + // CODEX_HOME — single explicit path; suppresses crossmount expansion + // (mirrors observer's codex/adapter.go:60). + if let Ok(env_home) = std::env::var("CODEX_HOME") { + if !env_home.is_empty() { + return vec![PathBuf::from(env_home).join("sessions")]; + } + } + self.homes + .iter() + .map(|h| h.path.join(".codex").join("sessions")) + .collect() + } + + fn is_session_file(&self, path: &Path) -> bool { + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { + return false; + }; + // Codex session files are `rollout-*.jsonl` under a watch root. + // Be permissive on the prefix (some builds use `session-*.jsonl`) + // but require the `.jsonl` suffix + watch-root constraint. + if !name.ends_with(".jsonl") { + return false; + } + let abs = path.to_path_buf(); + self.watch_paths().iter().any(|root| abs.starts_with(root)) + } + + async fn parse_session_file( + &self, + path: &Path, + from_offset: u64, + ) -> Result { + let bytes = tokio::fs::read(path) + .await + .map_err(|source| AdapterError::Io { + path: path.to_path_buf(), + source, + })?; + transcript::parse(path, &bytes, from_offset, &self.scrubber) + } +} diff --git a/aikit-session-capture/src/codex/transcript.rs b/aikit-session-capture/src/codex/transcript.rs new file mode 100644 index 0000000..3e26355 --- /dev/null +++ b/aikit-session-capture/src/codex/transcript.rs @@ -0,0 +1,1228 @@ +//! Codex CLI rollout JSONL parser. +//! +//! Reads `~/.codex/sessions/rollout-*.jsonl` line-by-line, resumable from a +//! byte offset, and emits normalized `ToolEvent` / `TokenEvent` rows. +//! See spec 010 §12.2. +//! +//! Reference: `superbased-observer/internal/adapter/codex/adapter.go` +//! (3122 lines — full coverage of every event variant). This Phase 3 MVP +//! covers the core rollout flows: session_configured / session_meta / +//! turn_context (context capture), user_message, agent_message, tool_call / +//! tool_output pairing, response_item.function_call / custom_tool_call / +//! function_call_output, event_msg.exec_command_end, web_search_end, +//! token_count (cumulative → per-turn delta), and `error` envelopes. Less +//! common variants (compacted, dynamic_tool_call_*, view_image, mcp_tool_call, +//! reasoning summary) are recognized and skipped cleanly with no warning; +//! they land in a later phase. + +use std::path::Path; + +use serde::Deserialize; +use serde_json::Value; + +use crate::adapter::{AdapterError, ParseResult, ParseWarning}; +use crate::models::{ActionKind, ActionStatus, CaptureSource, TokenEvent, ToolEvent, ToolKind}; +use crate::scrub::SecretScrubber; + +/// Parse Codex rollout JSONL bytes from `from_offset` to EOF. +pub(crate) fn parse( + path: &Path, + bytes: &[u8], + from_offset: u64, + scrubber: &SecretScrubber, +) -> Result { + let start = from_offset as usize; + if start > bytes.len() { + return Err(AdapterError::OffsetPastEof { + path: path.to_path_buf(), + requested: from_offset, + file_size: bytes.len() as u64, + }); + } + let tail = &bytes[start..]; + + let mut res = ParseResult { + new_offset: from_offset, + ..Default::default() + }; + + // File-level context, refreshed by session_configured / session_meta / + // turn_context. Mirrors observer's `sessionContext` + `applyContext`. + let mut ctx = SessionCtx::default(); + // Filename-stem fallback when no session-bearing envelope lands in the + // chunk being parsed (incremental resume case). + let fallback_session_id = session_id_from_path(path); + + // tool_call/tool_output pairing: call_id → index into tool_events. + let mut pending: std::collections::HashMap = std::collections::HashMap::new(); + // Cumulative token tracking — Codex emits session-wide totals; we compute + // per-turn deltas. Keyed by session_id (a single rollout file is one + // session, but we keep the key for parity with observer). + let mut last_net_input: std::collections::HashMap = + std::collections::HashMap::new(); + // Track the most recent token_count's session_id so the delta math knows + // which baseline to subtract from. + let mut current_session_id = String::new(); + + let mut cursor: usize = 0; + let mut line_no: u64 = 0; + while cursor < tail.len() { + let nl = match tail[cursor..].iter().position(|&b| b == b'\n') { + Some(i) => cursor + i + 1, + None => break, // partial trailing line — defer to next poll + }; + line_no += 1; + let raw_line = &tail[cursor..nl]; + cursor = nl; + res.new_offset = start as u64 + cursor as u64; + + let trimmed = std::str::from_utf8(raw_line) + .unwrap_or("") + .trim_end_matches(['\r', '\n']) + .trim(); + if trimmed.is_empty() { + continue; + } + + let rec: RawLine = match serde_json::from_str(trimmed) { + Ok(r) => r, + Err(e) => { + res.warnings.push(ParseWarning::MalformedLine { + line_no, + reason: format!("JSON parse: {e}"), + }); + continue; + } + }; + + let ts_ms = parse_ts_ms(rec.timestamp.as_deref().unwrap_or("")); + + // Dispatch on top-level type. Mirrors observer's switch at line 959+. + let rec_type = rec.r#type.as_str(); + let payload_type = rec + .payload + .as_ref() + .and_then(|p| p.get("type")) + .and_then(|t| t.as_str()) + .unwrap_or(""); + + match rec_type { + // ---- Context-bearing envelopes (sticky state) --------------- + "session_configured" | "session_start" | "session_meta" | "turn_context" => { + apply_context(&rec.payload, &mut ctx); + if ctx.session_id.is_empty() { + ctx.session_id = fallback_session_id.clone(); + } + if !ctx.session_id.is_empty() { + current_session_id = ctx.session_id.clone(); + } + } + + // ---- event_msg dispatch (the modern rollout path) ---------- + "event_msg" => match payload_type { + "task_started" => { + if let Some(turn) = rec + .payload + .as_ref() + .and_then(|p| p.get("turn_id")) + .and_then(|v| v.as_str()) + { + ctx.turn_id = turn.to_string(); + } + } + "agent_message" => { + if let Some(msg) = rec + .payload + .as_ref() + .and_then(|p| p.get("message")) + .and_then(|v| v.as_str()) + { + let msg = msg.trim(); + if !msg.is_empty() { + let event_id = format!("codex:{}:L{line_no}", short_hash(msg)); + res.tool_events.push(ToolEvent { + source_event_id: event_id, + source_file: path.to_path_buf(), + session_id: ctx.session_id.clone(), + tool: ToolKind::Codex, + kind: ActionKind::Think, + target: Some(truncate_str(msg, 200).to_string()), + input: Some(scrubber.scrub(msg)), + output: None, + status: ActionStatus::Success, + error_message: None, + started_at_ms: ts_ms, + duration_ms: None, + git_root: ctx.cwd.as_ref().map(std::path::PathBuf::from), + metadata: serde_json::json!({ "model": ctx.model }), + }); + } + } + } + "user_message" => { + let msg = rec + .payload + .as_ref() + .and_then(|p| { + p.get("message") + .and_then(|v| v.as_str()) + .or_else(|| p.get("content").and_then(|v| v.as_str())) + }) + .unwrap_or("") + .trim(); + if !msg.is_empty() { + let turn = if ctx.turn_id.is_empty() { + format!("L{line_no}") + } else { + ctx.turn_id.clone() + }; + let event_id = format!("codex:user:{turn}"); + res.tool_events.push(ToolEvent { + source_event_id: event_id, + source_file: path.to_path_buf(), + session_id: ctx.session_id.clone(), + tool: ToolKind::Codex, + kind: ActionKind::Other, + target: Some(truncate_str(msg, 200).to_string()), + input: Some(scrubber.scrub(msg)), + output: None, + status: ActionStatus::Success, + error_message: None, + started_at_ms: ts_ms, + duration_ms: None, + git_root: ctx.cwd.as_ref().map(std::path::PathBuf::from), + metadata: serde_json::json!({ "kind": "user_prompt" }), + }); + } + } + "exec_command_end" => { + let call_id = rec + .payload + .as_ref() + .and_then(|p| p.get("call_id")) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let cwd = rec + .payload + .as_ref() + .and_then(|p| p.get("cwd")) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let command = render_command_field(rec.payload.as_ref(), "command"); + let stdout = rec + .payload + .as_ref() + .and_then(|p| p.get("aggregated_output").or_else(|| p.get("stdout"))) + .and_then(|v| v.as_str()) + .unwrap_or(""); + let exit_code = rec + .payload + .as_ref() + .and_then(|p| p.get("exit_code")) + .and_then(|v| v.as_i64()) + .unwrap_or(0); + let status_str = rec + .payload + .as_ref() + .and_then(|p| p.get("status")) + .and_then(|v| v.as_str()) + .unwrap_or(""); + let failed = exit_code != 0 || status_str == "failed"; + // Merge into the pending tool_call if one exists; else emit standalone. + if let Some(&idx) = pending.get(&call_id) { + let scrubbed_out = scrubber.scrub(stdout); + res.tool_events[idx].output = Some(scrubbed_out); + if failed { + res.tool_events[idx].status = ActionStatus::Failure; + } + pending.remove(&call_id); + } else { + let event_id = format!("codex:exec:{call_id}"); + let target = scrubber.scrub(&command); + res.tool_events.push(ToolEvent { + source_event_id: event_id, + source_file: path.to_path_buf(), + session_id: ctx.session_id.clone(), + tool: ToolKind::Codex, + kind: ActionKind::Bash, + target: Some(truncate_str(&target, 200).to_string()), + input: Some(scrubber.scrub(&command)), + output: Some(scrubber.scrub(stdout)), + status: if failed { + ActionStatus::Failure + } else { + ActionStatus::Success + }, + error_message: if failed { + Some(truncate_str(stdout, 500).to_string()) + } else { + None + }, + started_at_ms: ts_ms, + duration_ms: duration_ms_from(rec.payload.as_ref()), + git_root: if cwd.is_empty() { + ctx.cwd.as_ref().map(std::path::PathBuf::from) + } else { + Some(std::path::PathBuf::from(cwd)) + }, + metadata: serde_json::json!({ "exit_code": exit_code }), + }); + } + } + "web_search_end" => { + let call_id = rec + .payload + .as_ref() + .and_then(|p| p.get("call_id")) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let query = rec + .payload + .as_ref() + .and_then(|p| { + // Try action.query first, then top-level query. + p.get("action") + .and_then(|a| a.get("query")) + .and_then(|v| v.as_str()) + .or_else(|| p.get("query").and_then(|v| v.as_str())) + }) + .unwrap_or("") + .to_string(); + if let Some(&idx) = pending.get(&call_id) { + res.tool_events[idx].kind = ActionKind::WebSearch; + res.tool_events[idx].target = Some(query.clone()); + pending.remove(&call_id); + } else { + let event_id = format!("codex:web:{call_id}"); + res.tool_events.push(ToolEvent { + source_event_id: event_id, + source_file: path.to_path_buf(), + session_id: ctx.session_id.clone(), + tool: ToolKind::Codex, + kind: ActionKind::WebSearch, + target: Some(query), + input: None, + output: None, + status: ActionStatus::Success, + error_message: None, + started_at_ms: ts_ms, + duration_ms: None, + git_root: ctx.cwd.as_ref().map(std::path::PathBuf::from), + metadata: serde_json::Value::Null, + }); + } + } + "error" => { + let message = rec + .payload + .as_ref() + .and_then(|p| p.get("message")) + .and_then(|v| v.as_str()) + .unwrap_or(""); + if !message.is_empty() { + let event_id = format!("codex:err:L{line_no}"); + res.tool_events.push(ToolEvent { + source_event_id: event_id, + source_file: path.to_path_buf(), + session_id: ctx.session_id.clone(), + tool: ToolKind::Codex, + kind: ActionKind::Other, + target: None, + input: None, + output: Some(scrubber.scrub(message)), + status: ActionStatus::Failure, + error_message: Some(truncate_str(message, 500).to_string()), + started_at_ms: ts_ms, + duration_ms: None, + git_root: ctx.cwd.as_ref().map(std::path::PathBuf::from), + metadata: serde_json::Value::Null, + }); + } + } + // task_complete / turn_aborted / mcp_tool_call_end / token_count + // are handled by their own top-level branches or below. + "token_count" => handle_token_count( + rec.payload.as_ref(), + &mut ctx, + &mut current_session_id, + &mut last_net_input, + from_offset, + &mut res.token_events, + path, + ts_ms, + line_no, + ), + _ => { + // Other event_msg subtypes we don't yet handle: dynamic_tool_call_*, + // view_image_tool_call, mcp_tool_call_end, context_compacted. + // Skip cleanly — no warning. + } + }, + + // ---- response_item dispatch (Codex Desktop / newer builds) -- + "response_item" => match payload_type { + "function_call" => { + let name = rec + .payload + .as_ref() + .and_then(|p| p.get("name")) + .and_then(|v| v.as_str()) + .unwrap_or(""); + let call_id = rec + .payload + .as_ref() + .and_then(|p| p.get("call_id")) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + // arguments is a JSON-encoded string in this envelope. + let args_str = rec + .payload + .as_ref() + .and_then(|p| p.get("arguments")) + .and_then(|v| v.as_str()) + .unwrap_or(""); + let (target, kind) = interpret_function_call(name, args_str); + let event_id = format!("codex:fn:{call_id}"); + let scrubbed_args = if args_str.is_empty() { + None + } else { + Some(scrubber.scrub(args_str)) + }; + let ev = ToolEvent { + source_event_id: event_id, + source_file: path.to_path_buf(), + session_id: ctx.session_id.clone(), + tool: ToolKind::Codex, + kind, + target: target.map(|t| scrubber.scrub(&t)), + input: scrubbed_args, + output: None, + status: ActionStatus::Success, + error_message: None, + started_at_ms: ts_ms, + duration_ms: None, + git_root: ctx.cwd.as_ref().map(std::path::PathBuf::from), + metadata: serde_json::json!({ "function": name }), + }; + let idx = res.tool_events.len(); + res.tool_events.push(ev); + if !call_id.is_empty() { + pending.insert(call_id, idx); + } + } + "function_call_output" | "custom_tool_call_output" => { + let call_id = rec + .payload + .as_ref() + .and_then(|p| p.get("call_id")) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let output = rec + .payload + .as_ref() + .and_then(|p| p.get("output")) + .and_then(|v| v.as_str()) + .unwrap_or(""); + if let Some(&idx) = pending.get(&call_id) { + let scrubbed = scrubber.scrub(output); + res.tool_events[idx].output = Some(scrubbed.clone()); + pending.remove(&call_id); + } + } + "custom_tool_call" => { + // apply_patch in current Codex Desktop. Treat as Edit. + let call_id = rec + .payload + .as_ref() + .and_then(|p| p.get("call_id")) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let name = rec + .payload + .as_ref() + .and_then(|p| p.get("name")) + .and_then(|v| v.as_str()) + .unwrap_or("apply_patch"); + let input = rec + .payload + .as_ref() + .and_then(|p| p.get("input")) + .and_then(|v| v.as_str()) + .unwrap_or(""); + let event_id = format!("codex:custom:{call_id}"); + let ev = ToolEvent { + source_event_id: event_id, + source_file: path.to_path_buf(), + session_id: ctx.session_id.clone(), + tool: ToolKind::Codex, + kind: ActionKind::Edit, + target: None, + input: if input.is_empty() { + None + } else { + Some(scrubber.scrub(input)) + }, + output: None, + status: ActionStatus::Success, + error_message: None, + started_at_ms: ts_ms, + duration_ms: None, + git_root: ctx.cwd.as_ref().map(std::path::PathBuf::from), + metadata: serde_json::json!({ "tool": name }), + }; + let idx = res.tool_events.len(); + res.tool_events.push(ev); + if !call_id.is_empty() { + pending.insert(call_id, idx); + } + } + "web_search_call" => { + let query = rec + .payload + .as_ref() + .and_then(|p| p.get("action")) + .and_then(|a| a.get("query")) + .and_then(|v| v.as_str()) + .unwrap_or("") + .to_string(); + let call_id = format!("ws:L{line_no}"); + let event_id = format!("codex:ws:{call_id}"); + res.tool_events.push(ToolEvent { + source_event_id: event_id, + source_file: path.to_path_buf(), + session_id: ctx.session_id.clone(), + tool: ToolKind::Codex, + kind: ActionKind::WebSearch, + target: Some(query), + input: None, + output: None, + status: ActionStatus::Success, + error_message: None, + started_at_ms: ts_ms, + duration_ms: None, + git_root: ctx.cwd.as_ref().map(std::path::PathBuf::from), + metadata: serde_json::Value::Null, + }); + } + _ => { + // message / reasoning / etc. — not handled in MVP. + } + }, + + // ---- Top-level tool_call / tool_output (older rollout path) -- + "tool_call" | "function_call" => { + let call_id = rec + .payload + .as_ref() + .and_then(|p| { + p.get("call_id") + .or_else(|| p.get("id")) + .and_then(|v| v.as_str()) + }) + .map(|s| s.to_string()) + .unwrap_or_else(|| format!("L{line_no}")); + let tool_name = rec + .payload + .as_ref() + .and_then(|p| { + p.get("tool") + .or_else(|| p.get("name")) + .and_then(|v| v.as_str()) + }) + .unwrap_or(""); + let input_value = rec.payload.as_ref().and_then(|p| p.get("input")); + let (target, kind) = interpret_tool_call(tool_name, input_value, scrubber); + let scrubbed_input = input_value.map(|v| scrubber.scrub(&v.to_string())); + let event_id = format!("codex:tc:{call_id}"); + let ev = ToolEvent { + source_event_id: event_id, + source_file: path.to_path_buf(), + session_id: ctx.session_id.clone(), + tool: ToolKind::Codex, + kind, + target: target.map(|t| truncate_str(&t, 200).to_string()), + input: scrubbed_input, + output: None, + status: ActionStatus::Success, + error_message: None, + started_at_ms: ts_ms, + duration_ms: None, + git_root: ctx.cwd.as_ref().map(std::path::PathBuf::from), + metadata: serde_json::json!({ "tool": tool_name }), + }; + let idx = res.tool_events.len(); + res.tool_events.push(ev); + pending.insert(call_id, idx); + } + "tool_output" | "function_call_output" => { + let call_id = rec + .payload + .as_ref() + .and_then(|p| { + p.get("call_id") + .or_else(|| p.get("id")) + .and_then(|v| v.as_str()) + }) + .map(|s| s.to_string()) + .unwrap_or_default(); + if call_id.is_empty() { + continue; + } + let body = decode_output(rec.payload.as_ref().and_then(|p| p.get("output"))); + let failed = rec + .payload + .as_ref() + .and_then(|p| { + // is_error=true → failed. success=false → failed. + // Prefer is_error when present; fall back to !success. + p.get("is_error") + .and_then(|v| v.as_bool()) + .or_else(|| p.get("success").and_then(|v| v.as_bool()).map(|b| !b)) + }) + .unwrap_or(false); + if let Some(&idx) = pending.get(&call_id) { + let scrubbed = scrubber.scrub(&body); + res.tool_events[idx].output = Some(scrubbed.clone()); + if failed { + res.tool_events[idx].status = ActionStatus::Failure; + res.tool_events[idx].error_message = + Some(truncate_str(&scrubbed, 500).to_string()); + } + pending.remove(&call_id); + } + } + + // ---- token_count (older rollout path: top-level, not under event_msg) + "token_count" | "usage" => handle_token_count( + rec.payload.as_ref(), + &mut ctx, + &mut current_session_id, + &mut last_net_input, + from_offset, + &mut res.token_events, + path, + ts_ms, + line_no, + ), + + // ---- top-level compacted marker (paired with event_msg/ + // context_compacted) — skip cleanly in MVP. + "compacted" => {} + + // ---- top-level user_message (older rollout shape, no event_msg + // wrapper). MVP emits it as a user-prompt ToolEvent. + "user_message" => { + let msg = rec + .payload + .as_ref() + .and_then(|p| { + p.get("message") + .and_then(|v| v.as_str()) + .or_else(|| p.get("content").and_then(|v| v.as_str())) + }) + .unwrap_or("") + .trim(); + if !msg.is_empty() { + let event_id = format!("codex:user:L{line_no}"); + res.tool_events.push(ToolEvent { + source_event_id: event_id, + source_file: path.to_path_buf(), + session_id: ctx.session_id.clone(), + tool: ToolKind::Codex, + kind: ActionKind::Other, + target: Some(truncate_str(msg, 200).to_string()), + input: Some(scrubber.scrub(msg)), + output: None, + status: ActionStatus::Success, + error_message: None, + started_at_ms: ts_ms, + duration_ms: None, + git_root: ctx.cwd.as_ref().map(std::path::PathBuf::from), + metadata: serde_json::json!({ "kind": "user_prompt" }), + }); + } + } + + _ => { + // Unknown top-level type — skip cleanly. Phase 3 MVP does + // not emit a warning for unknown shapes; the parser makes + // progress via the offset advance above. + } + } + } + + Ok(res) +} + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +#[derive(Default)] +struct SessionCtx { + session_id: String, + turn_id: String, + model: String, + cwd: Option, +} + +fn apply_context(payload: &Option, ctx: &mut SessionCtx) { + let Some(p) = payload else { return }; + if let Some(s) = p.get("session_id").and_then(|v| v.as_str()) { + if !s.is_empty() { + ctx.session_id = s.to_string(); + } + } + // session_meta uses "id" instead of "session_id". + if ctx.session_id.is_empty() { + if let Some(s) = p.get("id").and_then(|v| v.as_str()) { + if !s.is_empty() { + ctx.session_id = s.to_string(); + } + } + } + if let Some(s) = p.get("turn_id").and_then(|v| v.as_str()) { + if !s.is_empty() { + ctx.turn_id = s.to_string(); + } + } + if let Some(s) = p.get("model").and_then(|v| v.as_str()) { + if !s.is_empty() { + ctx.model = s.to_string(); + } + } + if let Some(s) = p.get("cwd").and_then(|v| v.as_str()) { + if !s.is_empty() { + ctx.cwd = Some(s.to_string()); + } + } +} + +#[allow(clippy::too_many_arguments)] +fn handle_token_count( + payload: Option<&Value>, + ctx: &mut SessionCtx, + current_session_id: &mut String, + last_net_input: &mut std::collections::HashMap, + from_offset: u64, + token_events: &mut Vec, + _path: &Path, + ts_ms: Option, + line_no: u64, +) { + let Some(p) = payload else { return }; + let input_tokens = p.get("input_tokens").and_then(|v| v.as_i64()).unwrap_or(0); + let output_tokens = p.get("output_tokens").and_then(|v| v.as_i64()).unwrap_or(0); + let cached = p + .get("cached_input_tokens") + .and_then(|v| v.as_i64()) + .unwrap_or(0); + let model = p + .get("model") + .and_then(|v| v.as_str()) + .map(|s| s.to_string()); + if let Some(m) = &model { + if ctx.model.is_empty() { + ctx.model = m.clone(); + } + } + let session = if !current_session_id.is_empty() { + current_session_id.clone() + } else if !ctx.session_id.is_empty() { + ctx.session_id.clone() + } else { + format!("line:{line_no}") + }; + if current_session_id.is_empty() && !ctx.session_id.is_empty() { + *current_session_id = ctx.session_id.clone(); + } + + // Cumulative → per-turn delta. Net input = gross - cached (Anthropic-shape + // convention; cost-engine TokenBundle.Input is NET non-cached). The first + // event after an incremental resume (from_offset > 0) has no in-memory + // baseline — emit 0 input so we don't over-report. + let net_cum = (input_tokens - cached).max(0); + let net_in = match last_net_input.get(&session) { + None if from_offset == 0 => net_cum, + None => 0, // resume cold-start + Some(&prev) if net_cum >= prev => net_cum - prev, + Some(_) => net_cum, // negative delta — reset / resequencing + }; + last_net_input.insert(session.clone(), net_cum); + + // Dedup: Codex re-emits identical token_count records. Observer's + // `seenModernTotal` invariant — the cumulative totals match exactly. + // Compare the raw (pre-delta) values: input_tokens, output_tokens, + // cached_input_tokens. If the prior event for this session has the same + // cumulative totals, this is a re-emission → skip. + let last_event_match = token_events.iter().rev().take(3).any(|e| { + e.session_id == session + && e.cache_read_tokens == Some(cached as u64) + && e.output_tokens == Some(output_tokens as u64) + // The cumulative input = delta + (cumulative - prior_cumulative). + // For dedup we need the *gross* totals, but TokenEvent stores + // per-turn deltas. Match on output + cached (the two fields that + // ARE the cumulative value verbatim) — those alone uniquely + // identify the re-emission. + }); + if last_event_match { + return; + } + + let event_id = format!("codex:tokens:{session}:L{line_no}"); + token_events.push(TokenEvent { + source_event_id: event_id, + session_id: session, + tool: ToolKind::Codex, + model, + request_id: None, + input_tokens: Some(net_in as u64), + cache_read_tokens: Some(cached as u64), + cache_creation_tokens: None, + cache_creation_1h_tokens: None, + output_tokens: Some(output_tokens as u64), + reasoning_tokens: None, + captured_at_ms: ts_ms.unwrap_or(0), + captured_via: CaptureSource::Transcript, + }); +} + +/// `command` field can be a string or array of strings (the argv form). +fn render_command_field(payload: Option<&Value>, key: &str) -> String { + let Some(v) = payload.and_then(|p| p.get(key)) else { + return String::new(); + }; + match v { + Value::String(s) => s.clone(), + Value::Array(arr) => arr + .iter() + .filter_map(|v| v.as_str()) + .collect::>() + .join(" "), + _ => String::new(), + } +} + +fn duration_ms_from(payload: Option<&Value>) -> Option { + let d = payload.and_then(|p| p.get("duration"))?; + let secs = d.get("secs").and_then(|v| v.as_i64()).unwrap_or(0); + let nanos = d.get("nanos").and_then(|v| v.as_i64()).unwrap_or(0); + if secs == 0 && nanos == 0 { + return None; + } + Some((secs * 1000 + nanos / 1_000_000) as u64) +} + +fn decode_output(raw: Option<&Value>) -> String { + let Some(v) = raw else { + return String::new(); + }; + match v { + Value::String(s) => s.clone(), + Value::Array(arr) => arr + .iter() + .filter_map(|v| { + if v.get("type").and_then(|t| t.as_str()) == Some("text") { + v.get("text").and_then(|t| t.as_str()).map(String::from) + } else { + None + } + }) + .collect::>() + .join(" "), + _ => v.to_string(), + } +} + +fn interpret_function_call(name: &str, args_str: &str) -> (Option, ActionKind) { + // Try to parse args as JSON for target extraction. + let args_v: Option = serde_json::from_str(args_str).ok(); + let target = args_v.as_ref().and_then(|a| { + a.get("command") + .and_then(|v| { + v.as_str().map(String::from).or_else(|| { + v.as_array().map(|arr| { + arr.iter() + .filter_map(|x| x.as_str()) + .collect::>() + .join(" ") + }) + }) + }) + .or_else(|| a.get("path").and_then(|v| v.as_str()).map(String::from)) + .or_else(|| a.get("filePath").and_then(|v| v.as_str()).map(String::from)) + .or_else(|| a.get("query").and_then(|v| v.as_str()).map(String::from)) + }); + let kind = match name { + "shell_command" | "exec_command" | "shell" | "exec" | "execute" | "command" => { + ActionKind::Bash + } + "file_read" | "read_file" | "open_file" | "view_image" => ActionKind::Read, + "file_write" | "write_file" | "create_file" => ActionKind::Write, + "apply_patch" | "edit_file" | "patch" | "replace" => ActionKind::Edit, + // web_search is a server-side web query (OpenAI/Anthropic hosted), + // not a codebase grep — classify distinctly from content search. + "web_search" => ActionKind::WebSearch, + "search" | "grep" | "find_text" | "find_in_files" => ActionKind::Search, + "web_fetch" | "fetch_url" => ActionKind::WebFetch, + "glob" | "find" | "list_files" | "list_directory" | "file_search" => ActionKind::Glob, + "update_plan" => ActionKind::Plan, + n if n.starts_with("mcp") || n.contains("mcp_") => ActionKind::Mcp, + _ => ActionKind::Other, + }; + (target, kind) +} + +fn interpret_tool_call( + tool_name: &str, + input: Option<&Value>, + scrubber: &SecretScrubber, +) -> (Option, ActionKind) { + let target = input.and_then(|i| { + i.get("path") + .and_then(|v| v.as_str()) + .map(String::from) + .or_else(|| i.get("filePath").and_then(|v| v.as_str()).map(String::from)) + .or_else(|| i.get("query").and_then(|v| v.as_str()).map(String::from)) + .or_else(|| { + i.get("command").and_then(|v| { + v.as_str().map(|s| scrubber.scrub(s)).or_else(|| { + v.as_array().map(|arr| { + scrubber.scrub( + &arr.iter() + .filter_map(|x| x.as_str()) + .collect::>() + .join(" "), + ) + }) + }) + }) + }) + }); + let kind = match tool_name { + "shell" | "exec" | "execute" | "command" | "shell_command" | "exec_command" => { + ActionKind::Bash + } + "file_read" | "read_file" | "open_file" | "view_image" => ActionKind::Read, + "file_write" | "write_file" | "create_file" => ActionKind::Write, + "apply_patch" | "edit_file" | "patch" | "replace" => ActionKind::Edit, + "web_search" => ActionKind::WebSearch, + "search" | "grep" | "find_text" | "find_in_files" => ActionKind::Search, + "web_fetch" | "fetch_url" => ActionKind::WebFetch, + "glob" | "find" | "list_files" | "list_directory" | "file_search" => ActionKind::Glob, + "update_plan" => ActionKind::Plan, + n if n.starts_with("mcp") || n.contains("mcp_") => ActionKind::Mcp, + _ => ActionKind::Other, + }; + (target, kind) +} + +fn session_id_from_path(path: &Path) -> String { + // Use filename stem. Codex rollout files are `rollout--.jsonl`. + path.file_stem() + .and_then(|s| s.to_str()) + .map(|s| s.to_string()) + .unwrap_or_default() +} + +fn parse_ts_ms(s: &str) -> Option { + if s.is_empty() { + return None; + } + chrono::DateTime::parse_from_rfc3339(s) + .ok() + .map(|dt| dt.timestamp_millis()) +} + +fn truncate_str(s: &str, max: usize) -> &str { + if s.len() <= max { + s + } else { + let mut end = max; + while end > 0 && !s.is_char_boundary(end) { + end -= 1; + } + &s[..end] + } +} + +/// Stable short hash (8 hex chars) for content-derived event IDs. NOT a +/// security primitive — just a deterministic discriminator so two identical +/// user prompts in one file dedup cleanly. +fn short_hash(s: &str) -> String { + // FNV-1a 64-bit, top 32 bits to hex. No extra dep. + let mut hash: u64 = 0xcbf29ce484222325; + for &b in s.as_bytes() { + hash ^= b as u64; + hash = hash.wrapping_mul(0x100000001b3); + } + format!("{:08x}", (hash >> 32) as u32) +} + +// --------------------------------------------------------------------------- +// Raw types +// --------------------------------------------------------------------------- + +#[derive(Debug, Deserialize)] +struct RawLine { + /// Record id; not currently consumed by the MVP emitter (event IDs are + /// derived from `call_id` / `line_no` for stable cross-reparse + /// determinism). Kept on the struct so a future phase can read it for + /// observer-parity event IDs without a schema change. + #[serde(default)] + #[allow(dead_code)] + id: Option, + #[serde(default)] + timestamp: Option, + #[serde(default, rename = "type")] + r#type: String, + #[serde(default)] + payload: Option, +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +#[cfg(test)] +mod tests { + use super::*; + + fn fixture(name: &str) -> Vec { + let manifest_dir = env!("CARGO_MANIFEST_DIR"); + let path = format!("{manifest_dir}/tests/fixtures/codex/{name}"); + std::fs::read(&path).unwrap_or_else(|e| panic!("fixture {name} not found at {path}: {e}")) + } + + fn scrubber() -> SecretScrubber { + SecretScrubber::default() + } + + // ---- Task 14: core parsing ----------------------------------------- + + #[test] + fn parses_rollout_session_into_events() { + let bytes = fixture("rollout-session.jsonl"); + let path = Path::new("/tmp/rollout-001.jsonl"); + let res = parse(path, &bytes, 0, &scrubber()).unwrap(); + + // Expected: 3 tool_use events (file_read, shell, web_search), + // ≥2 token events, ≥1 user prompt. + let tool_kinds: Vec = res.tool_events.iter().map(|e| e.kind).collect(); + assert!( + tool_kinds.contains(&ActionKind::Read), + "expected a Read, got: {tool_kinds:?}" + ); + assert!( + tool_kinds.contains(&ActionKind::Bash), + "expected a Bash, got: {tool_kinds:?}" + ); + assert!( + tool_kinds.contains(&ActionKind::WebSearch), + "expected a WebSearch, got: {tool_kinds:?}" + ); + + // file_read result paired back. + let read_ev = res + .tool_events + .iter() + .find(|e| e.kind == ActionKind::Read) + .expect("Read event"); + assert_eq!( + read_ev.output.as_deref(), + Some("package main\n\nfunc main() {}") + ); + + // shell result was failure (success:false). + let bash_ev = res + .tool_events + .iter() + .find(|e| e.kind == ActionKind::Bash) + .expect("Bash event"); + assert_eq!(bash_ev.status, ActionStatus::Failure); + + // ≥2 token events (the fixture has two token_count lines). + assert!( + res.token_events.len() >= 2, + "expected ≥2 token events, got {}", + res.token_events.len() + ); + // Session ID inherited from session_configured. + assert_eq!(read_ev.session_id, "cx-001"); + } + + #[test] + fn parses_response_item_dispatch_path() { + let bytes = fixture("rollout-response-item.jsonl"); + let path = Path::new("/tmp/rollout-resp.jsonl"); + let res = parse(path, &bytes, 0, &scrubber()).unwrap(); + + // The fixture exercises the response_item.function_call path. + // We should see at least one shell/bash event from "shell_command" + // function calls. + let tool_kinds: Vec = res.tool_events.iter().map(|e| e.kind).collect(); + assert!( + tool_kinds.contains(&ActionKind::Bash), + "expected a Bash event from function_call shell_command, got: {tool_kinds:?}" + ); + // And at least one web_search_call → WebSearch. + assert!( + tool_kinds.contains(&ActionKind::WebSearch), + "expected a WebSearch event, got: {tool_kinds:?}" + ); + } + + // ---- Task 14: invariants ------------------------------------------- + + #[test] + fn parse_twice_from_zero_produces_identical_source_event_ids() { + let bytes = fixture("rollout-session.jsonl"); + let path = Path::new("/tmp/rollout-idem.jsonl"); + let r1 = parse(path, &bytes, 0, &scrubber()).unwrap(); + let r2 = parse(path, &bytes, 0, &scrubber()).unwrap(); + let ids1: std::collections::HashSet<&str> = r1 + .tool_events + .iter() + .map(|e| e.source_event_id.as_str()) + .collect(); + let ids2: std::collections::HashSet<&str> = r2 + .tool_events + .iter() + .map(|e| e.source_event_id.as_str()) + .collect(); + assert_eq!(ids1, ids2, "idempotency: tool_event id sets must match"); + let token_ids1: std::collections::HashSet<&str> = r1 + .token_events + .iter() + .map(|e| e.source_event_id.as_str()) + .collect(); + let token_ids2: std::collections::HashSet<&str> = r2 + .token_events + .iter() + .map(|e| e.source_event_id.as_str()) + .collect(); + assert_eq!( + token_ids1, token_ids2, + "idempotency: token_event id sets must match" + ); + } + + #[test] + fn parse_from_nonzero_offset_skips_consumed_bytes() { + let bytes = fixture("rollout-session.jsonl"); + let path = Path::new("/tmp/rollout-offset.jsonl"); + let r1 = parse(path, &bytes, 0, &scrubber()).unwrap(); + let r2 = parse(path, &bytes, r1.new_offset, &scrubber()).unwrap(); + assert!(r2.tool_events.is_empty()); + assert!(r2.token_events.is_empty()); + assert_eq!(r2.new_offset, r1.new_offset); + } + + #[test] + fn offset_advances_to_eof_on_complete_parse() { + let bytes = fixture("rollout-session.jsonl"); + let path = Path::new("/tmp/rollout-eof.jsonl"); + let res = parse(path, &bytes, 0, &scrubber()).unwrap(); + let last_nl = bytes + .iter() + .rposition(|&b| b == b'\n') + .map(|i| i as u64 + 1) + .unwrap_or(0); + assert_eq!( + res.new_offset, last_nl, + "cursor must reach EOF after parsing all lines" + ); + } + + #[test] + fn malformed_line_is_skipped_not_fatal() { + // session_configured → malformed line → top-level user_message. + let bytes = b"{\"id\":\"sc\",\"type\":\"session_configured\",\"payload\":{\"session_id\":\"s1\"}}\nthis is not json }}\n{\"id\":\"um1\",\"timestamp\":\"2026-01-01T00:00:00Z\",\"type\":\"user_message\",\"payload\":{\"message\":\"hi\"}}\n"; + let path = Path::new("/tmp/rollout-malformed.jsonl"); + let res = parse(path, bytes, 0, &scrubber()).unwrap(); + assert!( + !res.warnings.is_empty(), + "expected a ParseWarning for the malformed line" + ); + // The user_message after the malformed line parsed. + assert!( + res.tool_events.iter().any(|e| e.kind == ActionKind::Other), + "expected the user_message after the malformed line to parse" + ); + // Cursor reached EOF. + let last_nl = bytes.iter().rposition(|&b| b == b'\n').unwrap() as u64 + 1; + assert_eq!(res.new_offset, last_nl); + } + + #[test] + fn secrets_are_scrubbed_from_input_and_output() { + let bytes = fixture("with-secrets.jsonl"); + let path = Path::new("/tmp/rollout-scrub.jsonl"); + let res = parse(path, &bytes, 0, &scrubber()).unwrap(); + let forbidden = [ + "AKIAIOSFODNN7EXAMPLE", + "ghp_0123456789012345678901234567890abcdefgh", + "eyJhbGciOiJIUzI1NiJ9", + "sk-ant-api03-abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJ", + "sk-proj-abcdef1234567890ABCDEFGHIJabcdefghij", + "secretpass123", + ]; + for ev in &res.tool_events { + for s in [&ev.input, &ev.output, &ev.target].into_iter().flatten() { + for f in forbidden { + assert!(!s.contains(f), "SCRUB FAILURE: {s:?} contains '{f}'"); + } + } + } + } + + #[test] + fn token_count_dedups_identical_envelopes() { + // Codex periodically re-emits identical token_count records. The + // adapter MUST dedup them (observer's seenModernTotal invariant). + let bytes = b"{\"id\":\"sc\",\"type\":\"session_configured\",\"payload\":{\"session_id\":\"s1\"}}\n{\"id\":\"tk1\",\"type\":\"token_count\",\"payload\":{\"input_tokens\":1000,\"output_tokens\":200,\"cached_input_tokens\":800,\"model\":\"gpt-5-codex\"}}\n{\"id\":\"tk2\",\"type\":\"token_count\",\"payload\":{\"input_tokens\":1000,\"output_tokens\":200,\"cached_input_tokens\":800,\"model\":\"gpt-5-codex\"}}\n"; + let path = Path::new("/tmp/rollout-dedup.jsonl"); + let res = parse(path, bytes, 0, &scrubber()).unwrap(); + // Two identical token_count records, but only one event survives. + assert_eq!( + res.token_events.len(), + 1, + "identical token_count envelopes MUST dedup to one event" + ); + } + + #[test] + fn token_count_cumulative_converts_to_per_turn_delta() { + // First token_count with cumulative input=1000,cached=800 → net 200. + // Second with cumulative input=1600,cached=1200 → net 400 → delta 200. + let bytes = b"{\"id\":\"sc\",\"type\":\"session_configured\",\"payload\":{\"session_id\":\"s1\"}}\n{\"id\":\"tk1\",\"type\":\"token_count\",\"payload\":{\"input_tokens\":1000,\"output_tokens\":200,\"cached_input_tokens\":800,\"model\":\"m\"}}\n{\"id\":\"tk2\",\"type\":\"token_count\",\"payload\":{\"input_tokens\":1600,\"output_tokens\":400,\"cached_input_tokens\":1200,\"model\":\"m\"}}\n"; + let path = Path::new("/tmp/rollout-delta.jsonl"); + let res = parse(path, bytes, 0, &scrubber()).unwrap(); + assert_eq!(res.token_events.len(), 2); + // First event: net input = 1000-800 = 200. + assert_eq!(res.token_events[0].input_tokens, Some(200)); + // Second event: net cumulative 400, prior 200 → delta 200. + assert_eq!(res.token_events[1].input_tokens, Some(200)); + } + + #[test] + fn unknown_top_level_type_skipped_cleanly() { + let bytes = b"{\"id\":\"x\",\"type\":\"future_event\",\"payload\":{\"foo\":\"bar\"}}\n"; + let path = Path::new("/tmp/rollout-unknown.jsonl"); + let res = parse(path, bytes, 0, &scrubber()).unwrap(); + assert!(res.tool_events.is_empty()); + assert!( + res.warnings.is_empty(), + "unknown shapes skip silently (no warning)" + ); + // Cursor advanced. + let last_nl = bytes.iter().rposition(|&b| b == b'\n').unwrap() as u64 + 1; + assert_eq!(res.new_offset, last_nl); + } +} diff --git a/aikit-session-capture/src/cursor_offset.rs b/aikit-session-capture/src/cursor_offset.rs new file mode 100644 index 0000000..85c0580 --- /dev/null +++ b/aikit-session-capture/src/cursor_offset.rs @@ -0,0 +1,166 @@ +//! Offset cursors: persist where to resume parsing from on the next call. +//! +//! See spec 010 §10. Two reference impls ship in this crate; production +//! SQLite impls live in the host (aikit-serve). + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::Mutex; + +use async_trait::async_trait; +use chrono::{DateTime, Utc}; +use serde::{Deserialize, Serialize}; + +use crate::models::ToolKind; + +/// One persisted cursor row. +#[derive(Debug, Clone)] +pub struct ParseCursor { + pub source_file: PathBuf, + /// Offset semantics are adapter-defined: byte offset for JSONL adapters, + /// `time_updated` watermark for the SQLite-based OpenCode adapter. + pub offset: u64, + pub adapter_kind: ToolKind, + pub updated_at: DateTime, +} + +/// On-disk JSON shape. `source_file` is the map key, not a field here. +#[derive(Serialize, Deserialize)] +struct StoredCursor { + offset: u64, + adapter_kind: ToolKind, + updated_at: DateTime, +} + +/// Persists parse offsets so the next call resumes from the right place. +#[async_trait] +pub trait CursorStore: Send + Sync { + async fn load(&self, source_file: &Path) -> Option; + async fn save(&self, cursor: ParseCursor); +} + +/// In-memory cursor store. For tests and ephemeral hosts. +#[derive(Default)] +pub struct InMemoryCursorStore { + inner: Mutex>, +} + +#[async_trait] +impl CursorStore for InMemoryCursorStore { + async fn load(&self, source_file: &Path) -> Option { + self.inner.lock().unwrap().get(source_file).cloned() + } + async fn save(&self, cursor: ParseCursor) { + self.inner + .lock() + .unwrap() + .insert(cursor.source_file.clone(), cursor); + } +} + +/// JSON-on-disk cursor store. Atomic writes via temp-file + rename. +pub struct JsonSidecarCursorStore { + path: PathBuf, +} + +impl JsonSidecarCursorStore { + pub fn open() -> std::io::Result { + let root = dirs::config_dir() + .or_else(dirs::home_dir) + .unwrap_or_else(|| PathBuf::from(".")); + let dir = root.join(".aikit").join("adapters"); + std::fs::create_dir_all(&dir)?; + Ok(Self { + path: dir.join("cursors.json"), + }) + } + + pub fn at(path: PathBuf) -> Self { + Self { path } + } + + fn load_all(&self) -> HashMap { + match std::fs::read(&self.path) { + Ok(bytes) => serde_json::from_slice(&bytes).unwrap_or_default(), + Err(_) => HashMap::new(), + } + } + + fn store_all(&self, map: &HashMap) -> std::io::Result<()> { + if let Some(parent) = self.path.parent() { + std::fs::create_dir_all(parent)?; + } + let tmp = self.path.with_extension("json.tmp"); + let bytes = serde_json::to_vec(map)?; + std::fs::write(&tmp, &bytes)?; + std::fs::rename(&tmp, &self.path)?; + Ok(()) + } +} + +#[async_trait] +impl CursorStore for JsonSidecarCursorStore { + async fn load(&self, source_file: &Path) -> Option { + let map = self.load_all(); + let key = source_file.to_string_lossy(); + let stored = map.get(key.as_ref())?; + Some(ParseCursor { + source_file: source_file.to_path_buf(), + offset: stored.offset, + adapter_kind: stored.adapter_kind, + updated_at: stored.updated_at, + }) + } + + async fn save(&self, cursor: ParseCursor) { + let key = cursor.source_file.to_string_lossy().into_owned(); + let stored = StoredCursor { + offset: cursor.offset, + adapter_kind: cursor.adapter_kind, + updated_at: cursor.updated_at, + }; + let mut map = self.load_all(); + map.insert(key, stored); + if let Err(e) = self.store_all(&map) { + tracing::warn!(target: "aikit_session_capture::cursor", "cursor sidecar write failed: {e}"); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn in_memory_roundtrip() { + let store = InMemoryCursorStore::default(); + let path = PathBuf::from("/tmp/sess.jsonl"); + let cursor = ParseCursor { + source_file: path.clone(), + offset: 4096, + adapter_kind: ToolKind::ClaudeCode, + updated_at: Utc::now(), + }; + store.save(cursor.clone()).await; + let loaded = store.load(&path).await.unwrap(); + assert_eq!(loaded.offset, 4096); + assert_eq!(loaded.adapter_kind, ToolKind::ClaudeCode); + } + + #[tokio::test] + async fn json_sidecar_roundtrip() { + let tmp = tempfile::tempdir().unwrap(); + let store = JsonSidecarCursorStore::at(tmp.path().join("cursors.json")); + let path = PathBuf::from("/tmp/sess.jsonl"); + let cursor = ParseCursor { + source_file: path.clone(), + offset: 8192, + adapter_kind: ToolKind::Codex, + updated_at: Utc::now(), + }; + store.save(cursor.clone()).await; + let loaded = store.load(&path).await.unwrap(); + assert_eq!(loaded.offset, 8192); + assert_eq!(loaded.adapter_kind, ToolKind::Codex); + } +} diff --git a/aikit-session-capture/src/event_store.rs b/aikit-session-capture/src/event_store.rs new file mode 100644 index 0000000..e1da218 --- /dev/null +++ b/aikit-session-capture/src/event_store.rs @@ -0,0 +1,393 @@ +//! `EventStore`: the trait the host implements to persist and query parsed +//! events. Distinct from [`CursorStore`][crate::CursorStore] — see spec 010 §11. +//! +//! The crate ships [`InMemoryEventStore`] for tests. Production SQLite impl +//! lives in `aikit-serve/src/storage/`. + +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::sync::RwLock; + +use async_trait::async_trait; + +use crate::models::{ActionKind, CacheObservation, TokenEvent, ToolEvent, ToolKind}; + +/// Argument to [`EventStore::upsert_events`]. Each field may be empty. +#[derive(Debug, Clone, Default)] +pub struct EventBatch { + pub tool_events: Vec, + pub token_events: Vec, + pub cache_observations: Vec, +} + +/// One row returned by [`EventStore::sessions_for`]. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct SessionSummary { + pub tool: ToolKind, + pub session_id: String, + pub source_file: PathBuf, + pub first_event_at_ms: i64, + pub last_event_at_ms: i64, + pub action_count: u64, + pub tool_kinds: Vec, + #[serde(skip_serializing_if = "Option::is_none")] + pub git_root: Option, +} + +/// Return value of [`EventStore::last_file_touch`]. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct FileTouch { + pub path: PathBuf, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_read_at_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_modified_at_ms: Option, +} + +#[non_exhaustive] +#[derive(Debug, thiserror::Error)] +pub enum StoreError { + #[error("io: {0}")] + Io(#[from] std::io::Error), + /// Backend-specific failure (SQLite locked, rusqlite error, etc.). The + /// string is for diagnostics; callers do not pattern-match on it. + #[error("backend: {0}")] + Backend(String), +} + +/// Persists parsed events and answers queries over them. Host-implemented; +/// the trait is the contract. +#[async_trait] +pub trait EventStore: Send + Sync { + /// Upsert events. `(source_file, source_event_id)` is the unique key; + /// re-inserting the same key MUST be a no-op (idempotent upsert). + /// Returns the number of rows actually inserted (excluding deduplicated + /// upserts). Callers compare against batch size to compute + /// `deduplicated_count`. + async fn upsert_events(&self, events: EventBatch) -> Result; + + /// List parsed sessions for one Backend, optionally filtered by cwd. + async fn sessions_for( + &self, + tool: ToolKind, + cwd: Option<&Path>, + limit: u32, + offset: u32, + ) -> Result, StoreError>; + + /// Full action stream for one session, paginated. + async fn actions_for_session( + &self, + tool: ToolKind, + session_id: &str, + limit: u32, + offset: u32, + ) -> Result, StoreError>; + + /// Substring/regex search over `ToolEvent.output`. Used by MCP + /// `search_past_outputs`. Hosts with FTS5 should override. + async fn search_outputs(&self, query: &str, limit: u32) -> Result, StoreError>; + + /// Cost-engine pull accessor (spec 009 integration, spec 010 §16). + async fn token_events_for_session( + &self, + tool: ToolKind, + session_id: &str, + ) -> Result, StoreError>; + + /// Freshness join: most recent `Read`/`Edit`/`Write` of `path` across + /// every session. Used by MCP `check_file_freshness`. + async fn last_file_touch(&self, path: &Path) -> Result, StoreError>; +} + +/// In-memory event store. For tests and the `CliTestHarness` MCP integration +/// tests (spec 010 Phase 5). Production hosts use SQLite (lives in +/// aikit-serve, not this crate). +#[derive(Default)] +pub struct InMemoryEventStore { + tool_events: RwLock>, + token_events: RwLock>, + cache_observations: RwLock>, + // Index: source_event_id → position in tool_events. Enforces idempotency. + seen: RwLock>, + seen_cache: RwLock>, +} + +impl InMemoryEventStore { + pub fn new() -> Self { + Self::default() + } +} + +#[async_trait] +impl EventStore for InMemoryEventStore { + async fn upsert_events(&self, events: EventBatch) -> Result { + let mut seen = self.seen.write().unwrap(); + let mut te = self.tool_events.write().unwrap(); + let mut tk = self.token_events.write().unwrap(); + let mut inserted = 0u64; + + for ev in events.tool_events { + let key = format!("{}:{}", ev.source_file.display(), ev.source_event_id); + if seen.contains_key(&key) { + continue; + } + seen.insert(key, te.len()); + te.push(ev); + inserted += 1; + } + // Token events dedupe by their own source_event_id within the session. + let mut tk_seen: HashMap = HashMap::new(); + for (i, e) in tk.iter().enumerate() { + tk_seen.insert(format!("{}:{}", e.session_id, e.source_event_id), i); + } + for ev in events.token_events { + let key = format!("{}:{}", ev.session_id, ev.source_event_id); + if tk_seen.contains_key(&key) { + continue; + } + tk_seen.insert(key, tk.len()); + tk.push(ev); + inserted += 1; + } + let mut cache_seen = self.seen_cache.write().unwrap(); + let mut cache = self.cache_observations.write().unwrap(); + for ev in events.cache_observations { + let key = format!("{}:{}", ev.session_id, ev.source_event_id); + if cache_seen.contains_key(&key) { + continue; + } + cache_seen.insert(key, cache.len()); + cache.push(ev); + inserted += 1; + } + Ok(inserted) + } + + async fn sessions_for( + &self, + tool: ToolKind, + cwd: Option<&Path>, + limit: u32, + offset: u32, + ) -> Result, StoreError> { + let te = self.tool_events.read().unwrap(); + let mut by_session: HashMap<(ToolKind, String), Vec<&ToolEvent>> = HashMap::new(); + for ev in te.iter() { + if ev.tool != tool { + continue; + } + if let Some(cwd) = cwd { + let root = ev + .git_root + .as_deref() + .unwrap_or_else(|| ev.source_file.parent().unwrap_or(Path::new(""))); + if root != cwd { + continue; + } + } + by_session + .entry((ev.tool, ev.session_id.clone())) + .or_default() + .push(ev); + } + let mut summaries: Vec = by_session + .into_iter() + .map(|((tool, session_id), evs)| { + let action_count = evs.len() as u64; + let first = evs + .iter() + .filter_map(|e| e.started_at_ms) + .min() + .unwrap_or(0); + let last = evs + .iter() + .filter_map(|e| e.started_at_ms) + .max() + .unwrap_or(0); + let mut kinds: Vec = evs.iter().map(|e| e.kind).collect(); + // `Discriminant` is not `Ord`; sort by variant + // tag string for a stable, deterministic order. + kinds.sort_by_key(|k| k.as_str()); + kinds.dedup_by(|a, b| a.as_str() == b.as_str()); + SessionSummary { + tool, + session_id, + source_file: evs[0].source_file.clone(), + first_event_at_ms: first, + last_event_at_ms: last, + action_count, + tool_kinds: kinds, + git_root: evs[0].git_root.clone(), + } + }) + .collect(); + summaries.sort_by_key(|s| s.last_event_at_ms); + let off = (offset as usize).min(summaries.len()); + let end = (off + limit as usize).min(summaries.len()); + Ok(summaries[off..end].to_vec()) + } + + async fn actions_for_session( + &self, + tool: ToolKind, + session_id: &str, + limit: u32, + offset: u32, + ) -> Result, StoreError> { + let te = self.tool_events.read().unwrap(); + let mut matches: Vec<&ToolEvent> = te + .iter() + .filter(|e| e.tool == tool && e.session_id == session_id) + .collect(); + matches.sort_by_key(|e| e.started_at_ms.unwrap_or(0)); + let off = (offset as usize).min(matches.len()); + let end = (off + limit as usize).min(matches.len()); + Ok(matches[off..end].iter().map(|e| (*e).clone()).collect()) + } + + async fn search_outputs(&self, query: &str, limit: u32) -> Result, StoreError> { + let te = self.tool_events.read().unwrap(); + let q = query.to_lowercase(); + let matches: Vec = te + .iter() + .filter(|e| { + e.output + .as_deref() + .map(|o| o.to_lowercase().contains(&q)) + .unwrap_or(false) + }) + .take(limit as usize) + .cloned() + .collect(); + Ok(matches) + } + + async fn token_events_for_session( + &self, + tool: ToolKind, + session_id: &str, + ) -> Result, StoreError> { + let tk = self.token_events.read().unwrap(); + Ok(tk + .iter() + .filter(|e| e.tool == tool && e.session_id == session_id) + .cloned() + .collect()) + } + + async fn last_file_touch(&self, path: &Path) -> Result, StoreError> { + let te = self.tool_events.read().unwrap(); + let last_read = te + .iter() + .filter(|e| { + matches!( + e.kind, + ActionKind::Read | ActionKind::Edit | ActionKind::Write + ) && e.target.as_deref() == Some(path.to_string_lossy().as_ref()) + }) + .filter_map(|e| e.started_at_ms) + .max(); + let last_modified = std::fs::metadata(path) + .and_then(|m| m.modified()) + .ok() + .and_then(|t| { + t.duration_since(std::time::UNIX_EPOCH) + .ok() + .map(|d| d.as_millis() as i64) + }); + if last_read.is_none() && last_modified.is_none() { + return Ok(None); + } + Ok(Some(FileTouch { + path: path.to_path_buf(), + last_read_at_ms: last_read, + last_modified_at_ms: last_modified, + })) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::models::{ActionStatus, CaptureSource}; + + fn ev(id: &str, sess: &str, kind: ActionKind, started: i64) -> ToolEvent { + ToolEvent { + source_event_id: id.into(), + source_file: PathBuf::from("/tmp/sess.jsonl"), + session_id: sess.into(), + tool: ToolKind::ClaudeCode, + kind, + target: None, + input: None, + output: None, + status: ActionStatus::Success, + error_message: None, + started_at_ms: Some(started), + duration_ms: None, + git_root: None, + metadata: serde_json::Value::Null, + } + } + + #[tokio::test] + async fn upsert_is_idempotent() { + let store = InMemoryEventStore::new(); + let batch = EventBatch { + tool_events: vec![ev("1", "s1", ActionKind::Read, 100)], + token_events: vec![], + cache_observations: vec![], + }; + let n1 = store.upsert_events(batch.clone()).await.unwrap(); + let n2 = store.upsert_events(batch).await.unwrap(); + assert_eq!(n1, 1); + assert_eq!(n2, 0, "second upsert of same batch must dedupe to zero"); + } + + #[tokio::test] + async fn sessions_for_filters_by_tool() { + let store = InMemoryEventStore::new(); + store + .upsert_events(EventBatch { + tool_events: vec![ + ev("1", "s1", ActionKind::Read, 100), + ev("2", "s1", ActionKind::Edit, 200), + ev("3", "s2", ActionKind::Read, 50), + ], + token_events: vec![], + cache_observations: vec![], + }) + .await + .unwrap(); + let got = store + .sessions_for(ToolKind::ClaudeCode, None, 10, 0) + .await + .unwrap(); + assert_eq!(got.len(), 2); + } + + #[tokio::test] + async fn search_outputs_matches_substring() { + let store = InMemoryEventStore::new(); + let mut e = ev("1", "s1", ActionKind::Bash, 100); + e.output = Some("go test ./...\nok main 0.1s".into()); + store + .upsert_events(EventBatch { + tool_events: vec![e], + token_events: vec![], + cache_observations: vec![], + }) + .await + .unwrap(); + let got = store.search_outputs("main", 10).await.unwrap(); + assert_eq!(got.len(), 1); + } + + #[tokio::test] + async fn token_events_absence_is_none() { + // Sanity-check the CaptureSource variant deserializes cleanly. + let s = serde_json::to_string(&CaptureSource::Transcript).unwrap(); + assert_eq!(s, "\"transcript\""); + } +} diff --git a/aikit-session-capture/src/homes.rs b/aikit-session-capture/src/homes.rs new file mode 100644 index 0000000..dc64506 --- /dev/null +++ b/aikit-session-capture/src/homes.rs @@ -0,0 +1,317 @@ +//! Home-directory resolution. The trait is the seam; the strategy is +//! pluggable. See spec 010 §8. +//! +//! The `Adapter` trait has zero opinions about how `watch_paths()` are +//! discovered — it only consumes `Vec`. A `HomeResolver` strategy +//! produces them, with optional cross-mount enumeration behind the +//! `crossmount` feature. + +#[cfg(feature = "crossmount")] +use std::path::Path; +use std::path::PathBuf; + +/// One candidate `$HOME`-equivalent directory the adapter should treat as a +/// watch-root candidate. +#[derive(Debug, Clone)] +pub struct HomeRoot { + /// Absolute filesystem path, in whatever form is reachable from the + /// running process. + pub path: PathBuf, + /// The LOGICAL OS of the home, decoupled from `std::env::consts::OS`. + /// A native home on Linux has `Linux`; a `/mnt/c/Users/` entry on + /// the same host has `Windows`. Adapters use this to pick per-OS + /// subpaths (e.g. OpenCode Desktop's per-platform location). + pub os: HomeOs, + /// Where this candidate came from: `"native"` | `"wsl-mnt:"` | + /// `"wslhost:/"`. + pub origin: &'static str, +} + +/// The logical OS of a [`HomeRoot`]. Decoupled from the runtime OS so the +/// crossmount resolver can tag `/mnt/c/Users/*` as `Windows` while running +/// on Linux. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HomeOs { + Linux, + Windows, + Darwin, +} + +/// Produces candidate home directories. Native home is always first; extra +/// homes (when the resolver supports them) follow in stable order. +pub trait HomeResolver: Send + Sync { + fn homes(&self) -> Vec; +} + +/// Default: native home only. No cross-mount enumeration. Mirrors +/// `superbased-observer`'s `DefaultHomeResolver` semantics when the +/// `crossmount` package is disabled. +pub struct DefaultHomeResolver; + +impl HomeResolver for DefaultHomeResolver { + fn homes(&self) -> Vec { + let native = dirs::home_dir().map(|path| HomeRoot { + path, + os: native_os(), + origin: "native", + }); + native.into_iter().collect() + } +} + +fn native_os() -> HomeOs { + #[cfg(target_os = "linux")] + { + HomeOs::Linux + } + #[cfg(target_os = "macos")] + { + HomeOs::Darwin + } + #[cfg(target_os = "windows")] + { + HomeOs::Windows + } + #[cfg(not(any(target_os = "linux", target_os = "macos", target_os = "windows")))] + { + HomeOs::Linux + } +} + +/// Cross-mount resolver: enumerates WSL2 ↔ Windows home pairs so observer +/// running on one side picks up sessions written on the other. Behind the +/// `crossmount` feature; default-off because most users are on a single OS. +/// +/// On Linux: walks `/mnt/c/Users/*` for Windows homes. +/// On Windows: walks `\\wsl.localhost\\home\` for Linux homes. +/// On macOS / pure Linux / pure Windows: native only (the walks no-op). +/// +/// The [`DirLister`] trait seam (mirrors `crossmount.go:37-51` `detector`) +/// lets unit tests inject fake filesystems so the enumeration logic runs +/// deterministically on any host. +#[cfg(feature = "crossmount")] +pub struct CrossmountResolver { + lister: Box, +} + +/// Filesystem probe seam for the crossmount resolver. Production code uses +/// [`NativeDirLister`]; tests inject fakes to simulate `/mnt/c/Users` or +/// `\\wsl.localhost\` without actually having them on disk. +#[cfg(feature = "crossmount")] +pub trait DirLister: Send + Sync { + fn runtime_os(&self) -> &'static str; + fn native_home(&self) -> Option; + fn is_dir(&self, path: &Path) -> bool; + fn read_dir_names(&self, path: &Path) -> Vec; +} + +/// Production [`DirLister`] backed by `std::fs`. +#[cfg(feature = "crossmount")] +pub struct NativeDirLister; + +#[cfg(feature = "crossmount")] +impl DirLister for NativeDirLister { + fn runtime_os(&self) -> &'static str { + std::env::consts::OS + } + fn native_home(&self) -> Option { + dirs::home_dir() + } + fn is_dir(&self, path: &Path) -> bool { + path.is_dir() + } + fn read_dir_names(&self, path: &Path) -> Vec { + std::fs::read_dir(path) + .map(|entries| { + entries + .flatten() + .map(|e| e.file_name().to_string_lossy().into_owned()) + .collect() + }) + .unwrap_or_default() + } +} + +#[cfg(feature = "crossmount")] +impl CrossmountResolver { + /// Create with [`NativeDirLister`] — the production path. + pub fn new() -> Self { + Self { + lister: Box::new(NativeDirLister), + } + } + + /// Create with a custom [`DirLister`] — for tests. + pub fn with_lister(lister: Box) -> Self { + Self { lister } + } +} + +#[cfg(feature = "crossmount")] +impl Default for CrossmountResolver { + fn default() -> Self { + Self::new() + } +} + +#[cfg(feature = "crossmount")] +impl HomeResolver for CrossmountResolver { + fn homes(&self) -> Vec { + let mut out = DefaultHomeResolver.homes(); + out.extend(extra_homes(&*self.lister)); + out + } +} + +#[cfg(feature = "crossmount")] +fn extra_homes(lister: &dyn DirLister) -> Vec { + let os = lister.runtime_os(); + if os == "linux" { + wsl_windows_homes(lister) + } else if os == "windows" { + windows_wsl_homes(lister) + } else { + Vec::new() + } +} + +/// Linux side: enumerate `/mnt/c/Users/`. +#[cfg(feature = "crossmount")] +fn wsl_windows_homes(lister: &dyn DirLister) -> Vec { + let root = Path::new("/mnt/c/Users"); + if !lister.is_dir(root) { + return Vec::new(); + } + lister + .read_dir_names(root) + .into_iter() + .filter_map(|name| { + let path = root.join(&name); + if !lister.is_dir(&path) { + return None; + } + Some(HomeRoot { + path, + os: HomeOs::Windows, + origin: leak_origin("wsl-mnt", &name), + }) + }) + .collect() +} + +/// Windows side: enumerate `\\wsl.localhost\\home\`. +#[cfg(feature = "crossmount")] +fn windows_wsl_homes(lister: &dyn DirLister) -> Vec { + let root = Path::new(r"\\wsl.localhost\"); + if !lister.is_dir(root) { + return Vec::new(); + } + let mut out = Vec::new(); + for distro in lister.read_dir_names(root) { + let home_dir = root.join(&distro).join("home"); + if !lister.is_dir(&home_dir) { + continue; + } + for user in lister.read_dir_names(&home_dir) { + let path = home_dir.join(&user); + if lister.is_dir(&path) { + out.push(HomeRoot { + path, + os: HomeOs::Linux, + origin: leak_origin("wslhost", &format!("{distro}/{user}")), + }); + } + } + } + out +} + +/// Intern a dynamic origin string into a `&'static str`. The origin is a +/// diagnostic label — not data — so leaking is acceptable and bounded by the +/// number of home directories on the host (typically ≤5). +#[cfg(feature = "crossmount")] +fn leak_origin(prefix: &str, suffix: &str) -> &'static str { + Box::leak(format!("{prefix}:{suffix}").into_boxed_str()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn default_resolver_returns_native_home() { + let r = DefaultHomeResolver; + let homes = r.homes(); + // On a real host this returns ≥1 entry. On a stripped CI container + // without HOME set it may return 0; don't hard-assert. + for h in &homes { + assert_eq!(h.origin, "native"); + assert_eq!(h.os, native_os()); + } + } + + #[cfg(feature = "crossmount")] + #[test] + fn crossmount_faked_wsl_returns_native_plus_extras() { + // Simulate a Linux host with /mnt/c/Users/alice and /mnt/c/Users/bob. + struct FakeLister; + impl DirLister for FakeLister { + fn runtime_os(&self) -> &'static str { + "linux" + } + fn native_home(&self) -> Option { + Some(PathBuf::from("/home/me")) + } + fn is_dir(&self, path: &Path) -> bool { + let s = path.to_string_lossy(); + s == "/mnt/c/Users" + || s == "/mnt/c/Users/alice" + || s == "/mnt/c/Users/bob" + || s == "/home/me" + } + fn read_dir_names(&self, path: &Path) -> Vec { + match path.to_string_lossy().as_ref() { + "/mnt/c/Users" => vec!["alice".into(), "bob".into()], + _ => vec![], + } + } + } + + let resolver = CrossmountResolver::with_lister(Box::new(FakeLister)); + let homes = resolver.homes(); + + // Should include native + 2 WSL homes. + assert!(homes.len() >= 2, "expected at least 2 WSL homes"); + let wsl: Vec<_> = homes.iter().filter(|h| h.os == HomeOs::Windows).collect(); + assert_eq!(wsl.len(), 2, "should find 2 Windows homes via /mnt/c"); + // Verify origin format: wsl-mnt: + assert!(wsl.iter().any(|h| h.origin == "wsl-mnt:alice")); + assert!(wsl.iter().any(|h| h.origin == "wsl-mnt:bob")); + } + + #[cfg(feature = "crossmount")] + #[test] + fn crossmount_no_mnt_c_returns_native_only() { + struct NoMntC; + impl DirLister for NoMntC { + fn runtime_os(&self) -> &'static str { + "linux" + } + fn native_home(&self) -> Option { + Some(PathBuf::from("/home/me")) + } + fn is_dir(&self, _: &Path) -> bool { + false // /mnt/c/Users doesn't exist + } + fn read_dir_names(&self, _: &Path) -> Vec { + vec![] + } + } + + let resolver = CrossmountResolver::with_lister(Box::new(NoMntC)); + let homes = resolver.homes(); + let extras: Vec<_> = homes.iter().filter(|h| h.origin != "native").collect(); + assert!(extras.is_empty(), "no extras when /mnt/c/Users absent"); + } +} diff --git a/aikit-session-capture/src/lib.rs b/aikit-session-capture/src/lib.rs new file mode 100644 index 0000000..fa7d733 --- /dev/null +++ b/aikit-session-capture/src/lib.rs @@ -0,0 +1,60 @@ +//! `aikit-session-capture`: on-disk session adapters for AI coding tools. +//! +//! Turns one AI tool's persisted session data (Claude Code JSONL, Codex +//! rollout JSONL, OpenCode SQLite, …) into normalized `ToolEvent` / +//! `TokenEvent` rows that the host can store, query, and surface over MCP. +//! +//! The crate is **read-only and side-effect-free**: it parses what the tool +//! already wrote; it never spawns or drives the tool. Spawning remains the +//! runner's job (spec 006). +//! +//! # Status +//! +//! Scaffold — types and traits land per spec 010 phasing. This module +//! re-exports the public surface as it is implemented; today everything is +//! a stub that compiles green with `--no-default-features`. +//! +//! # Contracts (spec 010 §6) +//! +//! Implementations MUST: +//! - Scrub raw tool inputs before returning — no secrets escape the adapter. +//! The injected [`SecretScrubber`] is the single chokepoint. +//! - Emit deterministic `source_event_id`s so re-parsing is idempotent. +//! - Advance `new_offset` past the last fully-parsed byte (or, for SQLite +//! adapters, the last-consumed watermark), so the caller can persist it +//! and skip on the next call. +//! +//! [`SecretScrubber`]: scrub::SecretScrubber + +pub mod adapter; +pub mod cursor_offset; +pub mod event_store; +pub mod homes; +pub mod models; +pub mod registry; +pub mod scrub; + +#[cfg(feature = "claudecode")] +pub mod claudecode; +#[cfg(feature = "codex")] +pub mod codex; +#[cfg(feature = "opencode")] +pub mod opencode; + +#[cfg(feature = "mcp-tools")] +pub mod mcp; + +#[cfg(feature = "watcher")] +pub mod watch; + +pub use adapter::{Adapter, AdapterError, ParseResult, ParseWarning}; +pub use cursor_offset::{CursorStore, InMemoryCursorStore, JsonSidecarCursorStore, ParseCursor}; +pub use event_store::{ + EventBatch, EventStore, FileTouch, InMemoryEventStore, SessionSummary, StoreError, +}; +pub use homes::{DefaultHomeResolver, HomeOs, HomeResolver, HomeRoot}; +pub use models::{ + ActionKind, ActionStatus, CacheObservation, CaptureSource, TokenEvent, ToolEvent, ToolKind, +}; +pub use registry::Registry; +pub use scrub::SecretScrubber; diff --git a/aikit-session-capture/src/mcp.rs b/aikit-session-capture/src/mcp.rs new file mode 100644 index 0000000..f7e9c62 --- /dev/null +++ b/aikit-session-capture/src/mcp.rs @@ -0,0 +1,652 @@ +//! MCP tool functions for session capture (spec 010 §15). +//! +//! Four tools backed by [`EventStore`](crate::EventStore): +//! - `check_file_freshness` — freshness join: most recent Read/Edit/Write of a path +//! - `search_past_outputs` — substring search over `ToolEvent.output` +//! - `get_session_summary` — rule-based summary (no LLM, §15.1) +//! - `list_actions_around` — chronological ±N actions around a given action +//! +//! Behind the `mcp-tools` feature. Each function is a plain async callable; +//! the host wraps them into cli-framework `Command` registrations (spec §15.2). + +use std::collections::HashMap; +use std::path::Path; +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +use crate::event_store::EventStore; +use crate::models::{ToolEvent, ToolKind}; + +use cli_framework::app::AppBuilder; +use cli_framework::command::Command; +use cli_framework::spec::arg_spec::{ArgKind, ArgSpec, ArgValueType, Cardinality}; +use cli_framework::spec::command_tree::CommandSpec; +use cli_framework::spec::value::ArgValue; + +/// Register session-capture MCP commands with cli-framework. +/// +/// Commands are root-level and MCP-exportable; hosts may mount them into an +/// app that only exposes MCP, chat tools, or both. +pub fn register_commands(builder: AppBuilder, store: Arc) -> AppBuilder { + try_register_commands(builder, store).expect("session-capture MCP command registration") +} + +/// Fallible variant for hosts that want to surface command registration +/// collisions instead of panicking. +pub fn try_register_commands( + mut builder: AppBuilder, + store: Arc, +) -> anyhow::Result { + for command in [ + command_check_file_freshness(store.clone()), + command_search_past_outputs(store.clone()), + command_get_session_summary(store.clone()), + command_list_actions_around(store), + ] { + builder = builder.register_command(command)?; + } + Ok(builder) +} + +fn command_check_file_freshness(store: Arc) -> Command { + command( + "check_file_freshness", + CommandSpec { + summary: "Check whether an agent has read the current file content", + args: vec![string_arg( + "path", + Cardinality::Required, + "Absolute file path to check", + )], + category: Some("session-capture"), + ..CommandSpec::default() + }, + move |args| { + let store = store.clone(); + async move { + let result = check_file_freshness( + store.as_ref(), + CheckFileFreshnessArgs { + path: str_arg(&args, "path").unwrap_or_default(), + }, + ) + .await?; + Ok(result) + } + }, + ) +} + +fn command_search_past_outputs(store: Arc) -> Command { + command( + "search_past_outputs", + CommandSpec { + summary: "Search prior captured tool outputs", + args: vec![ + string_arg("query", Cardinality::Required, "Substring to search for"), + int_arg("limit", 20, "Maximum result count"), + ], + category: Some("session-capture"), + ..CommandSpec::default() + }, + move |args| { + let store = store.clone(); + async move { + let result = search_past_outputs( + store.as_ref(), + SearchPastOutputsArgs { + query: str_arg(&args, "query").unwrap_or_default(), + limit: u32_arg(&args, "limit", 20), + }, + ) + .await?; + Ok(result) + } + }, + ) +} + +fn command_get_session_summary(store: Arc) -> Command { + command( + "get_session_summary", + CommandSpec { + summary: "Summarize a captured session deterministically", + args: vec![string_arg( + "session_id", + Cardinality::Required, + "Captured session id", + )], + category: Some("session-capture"), + ..CommandSpec::default() + }, + move |args| { + let store = store.clone(); + async move { + let result = get_session_summary( + store.as_ref(), + GetSessionSummaryArgs { + session_id: str_arg(&args, "session_id").unwrap_or_default(), + }, + ) + .await?; + Ok(result) + } + }, + ) +} + +fn command_list_actions_around(store: Arc) -> Command { + command( + "list_actions_around", + CommandSpec { + summary: "List captured actions around an anchor action", + args: vec![ + string_arg("action_id", Cardinality::Required, "Anchor source event id"), + string_arg("session_id", Cardinality::Required, "Captured session id"), + int_arg("n", 5, "Number of actions before and after"), + ], + category: Some("session-capture"), + ..CommandSpec::default() + }, + move |args| { + let store = store.clone(); + async move { + let result = list_actions_around( + store.as_ref(), + ListActionsAroundArgs { + action_id: str_arg(&args, "action_id").unwrap_or_default(), + session_id: str_arg(&args, "session_id").unwrap_or_default(), + n: u32_arg(&args, "n", 5), + }, + ) + .await?; + Ok(result) + } + }, + ) +} + +fn command(id: &'static str, spec: CommandSpec, f: F) -> Command +where + F: Fn(HashMap) -> Fut + Send + Sync + 'static, + Fut: std::future::Future> + Send + 'static, + T: Serialize, +{ + let f = Arc::new(f); + Command { + id: Arc::from(id), + spec: Arc::new(spec), + validator: None, + expose_mcp: true, + expose_chat: true, + execute: Arc::new(move |ctx, args| { + let f = f.clone(); + Box::pin(async move { + let result = f(args).await?; + ctx.framework_println(&serde_json::to_string(&result)?); + Ok(()) + }) + }), + } +} + +fn string_arg(name: &'static str, cardinality: Cardinality, help: &'static str) -> ArgSpec { + ArgSpec { + name, + kind: ArgKind::Option, + value_type: ArgValueType::String, + cardinality, + help, + ..ArgSpec::default() + } +} + +fn int_arg(name: &'static str, default: i64, help: &'static str) -> ArgSpec { + ArgSpec { + name, + kind: ArgKind::Option, + value_type: ArgValueType::Int, + cardinality: Cardinality::Optional, + default: Some(ArgValue::Int(default)), + min: Some(0), + help, + ..ArgSpec::default() + } +} + +fn str_arg(args: &HashMap, name: &str) -> Option { + match args.get(name) { + Some(ArgValue::Str(s)) | Some(ArgValue::Enum(s)) => Some(s.clone()), + _ => None, + } +} + +fn u32_arg(args: &HashMap, name: &str, default: u32) -> u32 { + match args.get(name) { + Some(ArgValue::Int(i)) if *i >= 0 => (*i).min(u32::MAX as i64) as u32, + _ => default, + } +} + +// ── check_file_freshness ────────────────────────────────────────────────────── + +/// Args for [`check_file_freshness`]. +#[derive(Debug, Deserialize)] +pub struct CheckFileFreshnessArgs { + pub path: String, +} + +/// Result of [`check_file_freshness`]. +#[derive(Debug, Serialize)] +pub struct FileFreshness { + pub fresh: bool, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_read_at_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub last_modified_at_ms: Option, +} + +/// Check if a file was read by an agent more recently than it was last +/// modified on disk. Returns `fresh: true` when the agent has seen the +/// current content. +pub async fn check_file_freshness( + store: &dyn EventStore, + args: CheckFileFreshnessArgs, +) -> anyhow_result::Result { + let path = Path::new(&args.path); + let touch = store.last_file_touch(path).await?; + match touch { + Some(t) => { + let fresh = match (t.last_read_at_ms, t.last_modified_at_ms) { + (Some(read), Some(modified)) => read >= modified, + _ => false, + }; + Ok(FileFreshness { + fresh, + last_read_at_ms: t.last_read_at_ms, + last_modified_at_ms: t.last_modified_at_ms, + }) + } + None => Ok(FileFreshness { + fresh: false, + last_read_at_ms: None, + last_modified_at_ms: None, + }), + } +} + +// ── search_past_outputs ─────────────────────────────────────────────────────── + +/// Args for [`search_past_outputs`]. +#[derive(Debug, Deserialize)] +pub struct SearchPastOutputsArgs { + pub query: String, + #[serde(default = "default_search_limit")] + pub limit: u32, +} + +fn default_search_limit() -> u32 { + 20 +} + +/// Result of [`search_past_outputs`]. +#[derive(Debug, Serialize)] +pub struct SearchResult { + pub results: Vec, +} + +#[derive(Debug, Serialize)] +pub struct SearchHit { + pub source_event_id: String, + pub session_id: String, + pub tool: String, + pub kind: String, + pub target: Option, + pub output_excerpt: String, + pub started_at_ms: Option, +} + +/// Substring search over all `ToolEvent.output` fields. Used by agents to +/// recall past tool outputs ("what did the test suite say last time?"). +pub async fn search_past_outputs( + store: &dyn EventStore, + args: SearchPastOutputsArgs, +) -> anyhow_result::Result { + let events = store.search_outputs(&args.query, args.limit).await?; + let results = events + .into_iter() + .map(|ev| SearchHit { + source_event_id: ev.source_event_id, + session_id: ev.session_id, + tool: ev.tool.as_str().to_string(), + kind: ev.kind.as_str().to_string(), + target: ev.target, + output_excerpt: ev.output.unwrap_or_default().chars().take(500).collect(), + started_at_ms: ev.started_at_ms, + }) + .collect(); + Ok(SearchResult { results }) +} + +// ── get_session_summary ─────────────────────────────────────────────────────── + +/// Args for [`get_session_summary`]. +#[derive(Debug, Deserialize)] +pub struct GetSessionSummaryArgs { + pub session_id: String, +} + +/// Rule-based summary (spec §15.1). No LLM — deterministic format-string +/// composition. Pinned by test against a fixture. +#[derive(Debug, Serialize)] +pub struct SessionSummaryResult { + pub summary: String, + pub session_id: String, + pub action_count: u64, + pub tool_histogram: Vec<(String, u64)>, +} + +/// Build a deterministic summary of one session: first user message (≤200 +/// chars), top-3 ActionKind histogram, final assistant text (≤200 chars). +pub async fn get_session_summary( + store: &dyn EventStore, + args: GetSessionSummaryArgs, +) -> anyhow_result::Result { + // Gather sessions across all tools to find this one. + let tools = [ToolKind::ClaudeCode, ToolKind::Codex, ToolKind::OpenCode]; + for tool in &tools { + let actions = store + .actions_for_session(*tool, &args.session_id, u32::MAX, 0) + .await?; + if actions.is_empty() { + continue; + } + let action_count = actions.len() as u64; + + // Build histogram of ActionKind. + let mut hist: HashMap<&'static str, u64> = HashMap::new(); + for a in &actions { + *hist.entry(a.kind.as_str()).or_default() += 1; + } + let mut hist_vec: Vec<(String, u64)> = + hist.iter().map(|(k, v)| ((*k).to_string(), *v)).collect(); + hist_vec.sort_by(|a, b| b.1.cmp(&a.1)); + let top3: Vec<(String, u64)> = hist_vec.into_iter().take(3).collect(); + let hist_str = top3 + .iter() + .map(|(k, v)| format!("{k}({v})")) + .collect::>() + .join(", "); + + // First action's input is a proxy for "first user message". + let first_msg = actions + .first() + .and_then(|a| a.input.as_deref()) + .unwrap_or("(no input)") + .chars() + .take(200) + .collect::(); + + // Last action's output is a proxy for "conclusion." + let last_msg = actions + .last() + .and_then(|a| a.output.as_deref()) + .unwrap_or("(no output)") + .chars() + .take(200) + .collect::(); + + let summary = + format!("User asked: \"{first_msg}\"\nUsed: {hist_str}\nConcluded: \"{last_msg}\""); + + return Ok(SessionSummaryResult { + summary, + session_id: args.session_id, + action_count, + tool_histogram: top3, + }); + } + Ok(SessionSummaryResult { + summary: "(session not found)".to_string(), + session_id: args.session_id, + action_count: 0, + tool_histogram: vec![], + }) +} + +// ── list_actions_around ─────────────────────────────────────────────────────── + +/// Args for [`list_actions_around`]. +#[derive(Debug, Deserialize)] +pub struct ListActionsAroundArgs { + pub action_id: String, + pub session_id: String, + #[serde(default = "default_around_n")] + pub n: u32, +} + +fn default_around_n() -> u32 { + 5 +} + +/// Result of [`list_actions_around`]. +#[derive(Debug, Serialize)] +pub struct ActionsAroundResult { + pub actions: Vec, +} + +/// Return the chronological ±N actions around the given `action_id` in one +/// session. Used by agents to get context around a specific past action. +pub async fn list_actions_around( + store: &dyn EventStore, + args: ListActionsAroundArgs, +) -> anyhow_result::Result { + let tools = [ToolKind::ClaudeCode, ToolKind::Codex, ToolKind::OpenCode]; + for tool in &tools { + let all = store + .actions_for_session(*tool, &args.session_id, u32::MAX, 0) + .await?; + if all.is_empty() { + continue; + } + // Find the index of the anchor action. + let idx = all.iter().position(|a| a.source_event_id == args.action_id); + let idx = match idx { + Some(i) => i, + None => continue, + }; + let start = idx.saturating_sub(args.n as usize); + let end = (idx + args.n as usize + 1).min(all.len()); + return Ok(ActionsAroundResult { + actions: all[start..end].to_vec(), + }); + } + Ok(ActionsAroundResult { actions: vec![] }) +} + +/// Convenience: build the JSON Schema for one tool's args. Hosts use this +/// when registering tools via cli-framework's `CommandSpec`. +pub fn tool_args_schema(tool: &str) -> Value { + match tool { + "check_file_freshness" => serde_json::json!({ + "type": "object", + "properties": { + "path": { "type": "string", "description": "Absolute file path to check" } + }, + "required": ["path"] + }), + "search_past_outputs" => serde_json::json!({ + "type": "object", + "properties": { + "query": { "type": "string", "description": "Substring to search for" }, + "limit": { "type": "integer", "default": 20, "description": "Max results" } + }, + "required": ["query"] + }), + "get_session_summary" => serde_json::json!({ + "type": "object", + "properties": { + "session_id": { "type": "string", "description": "Session ID to summarize" } + }, + "required": ["session_id"] + }), + "list_actions_around" => serde_json::json!({ + "type": "object", + "properties": { + "action_id": { "type": "string", "description": "Source event ID of the anchor action" }, + "session_id": { "type": "string", "description": "Session containing the action" }, + "n": { "type": "integer", "default": 5, "description": "Number of actions before and after" } + }, + "required": ["action_id", "session_id"] + }), + _ => serde_json::json!({}), + } +} + +/// Lightweight wrapper around `anyhow::Result` to avoid a hard dependency +/// on `anyhow` for MCP consumers. We re-export the standard `anyhow::Result` +/// since it's already in the crate's dependency tree. +mod anyhow_result { + pub type Result = std::result::Result; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::event_store::{EventBatch, InMemoryEventStore}; + use crate::models::{ActionKind, ActionStatus, ToolEvent}; + + fn ev(id: &str, sess: &str, kind: ActionKind, started: i64, out: &str) -> ToolEvent { + ToolEvent { + source_event_id: id.into(), + source_file: std::path::PathBuf::from("/tmp/sess.jsonl"), + session_id: sess.into(), + tool: ToolKind::ClaudeCode, + kind, + target: None, + input: Some("Help me fix the bug".into()), + output: Some(out.into()), + status: ActionStatus::Success, + error_message: None, + started_at_ms: Some(started), + duration_ms: None, + git_root: None, + metadata: serde_json::Value::Null, + } + } + + #[tokio::test] + async fn check_file_freshness_unknown_path() { + let store = InMemoryEventStore::new(); + let result = check_file_freshness( + &store, + CheckFileFreshnessArgs { + path: "/nonexistent/file.go".into(), + }, + ) + .await + .unwrap(); + assert!(!result.fresh); + } + + #[tokio::test] + async fn search_past_outputs_finds_match() { + let store = InMemoryEventStore::new(); + store + .upsert_events(EventBatch { + tool_events: vec![ev("1", "s1", ActionKind::Bash, 100, "go test PASSED")], + token_events: vec![], + cache_observations: vec![], + }) + .await + .unwrap(); + let result = search_past_outputs( + &store, + SearchPastOutputsArgs { + query: "PASSED".into(), + limit: 10, + }, + ) + .await + .unwrap(); + assert_eq!(result.results.len(), 1); + assert_eq!(result.results[0].kind, "bash"); + } + + #[tokio::test] + async fn get_session_summary_is_deterministic() { + let store = InMemoryEventStore::new(); + store + .upsert_events(EventBatch { + tool_events: vec![ + ev("1", "s1", ActionKind::Read, 100, ""), + ev("2", "s1", ActionKind::Read, 200, ""), + ev("3", "s1", ActionKind::Edit, 300, ""), + ev("4", "s1", ActionKind::Bash, 400, "tests passed"), + ], + token_events: vec![], + cache_observations: vec![], + }) + .await + .unwrap(); + let result = get_session_summary( + &store, + GetSessionSummaryArgs { + session_id: "s1".into(), + }, + ) + .await + .unwrap(); + assert_eq!(result.action_count, 4); + assert!(result.summary.contains("User asked:")); + assert!(result.summary.contains("Used:")); + assert!(result.summary.contains("Concluded:")); + assert!(result.summary.contains("read(2)")); + assert_eq!(result.tool_histogram.len(), 3); + } + + #[tokio::test] + async fn list_actions_around_returns_neighbors() { + let store = InMemoryEventStore::new(); + let events: Vec = (0..10) + .map(|i| ev(&format!("act_{i}"), "s1", ActionKind::Read, i * 100, "")) + .collect(); + store + .upsert_events(EventBatch { + tool_events: events, + token_events: vec![], + cache_observations: vec![], + }) + .await + .unwrap(); + let result = list_actions_around( + &store, + ListActionsAroundArgs { + action_id: "act_5".into(), + session_id: "s1".into(), + n: 2, + }, + ) + .await + .unwrap(); + // ±2 = 5 actions (act_3, act_4, act_5, act_6, act_7) + assert_eq!(result.actions.len(), 5); + assert_eq!(result.actions[0].source_event_id, "act_3"); + assert_eq!(result.actions[2].source_event_id, "act_5"); + } + + #[test] + fn tool_args_schemas_are_valid() { + for tool in [ + "check_file_freshness", + "search_past_outputs", + "get_session_summary", + "list_actions_around", + ] { + let schema = tool_args_schema(tool); + assert_eq!(schema["type"], "object"); + } + } +} diff --git a/aikit-session-capture/src/models.rs b/aikit-session-capture/src/models.rs new file mode 100644 index 0000000..654bdf9 --- /dev/null +++ b/aikit-session-capture/src/models.rs @@ -0,0 +1,227 @@ +//! Normalized event types emitted by every adapter. +//! +//! See spec 010 `data-model.md` for the full field tables and invariants. +//! All enums/structs are `#[non_exhaustive]` and +//! `#[serde(rename_all = "snake_case")]`. + +use std::path::PathBuf; + +use serde::{Deserialize, Serialize}; + +/// Stable identifier for one AI coding tool's session format. Stored in the +/// `tool` column of the host's event store. Mirrors +/// `superbased-observer/internal/models.Tool*`. +// This enum intentionally reserves arms for adapters not implemented in this +// spec (Cursor, Gemini, Cline, …) so adding a new adapter is a non-breaking +// change for downstream consumers. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ToolKind { + ClaudeCode, + Codex, + OpenCode, + /// Reserved for a future Cursor hook-driven adapter (spec 011). + Cursor, + /// Reserved for a future Gemini adapter. + Gemini, +} + +impl ToolKind { + /// Stable string identifier (matches the serde representation). + pub fn as_str(self) -> &'static str { + match self { + ToolKind::ClaudeCode => "claude_code", + ToolKind::Codex => "codex", + ToolKind::OpenCode => "open_code", + ToolKind::Cursor => "cursor", + ToolKind::Gemini => "gemini", + } + } +} + +/// Normalized action category. Mirrors observer's 28-category taxonomy, +/// trimmed to the kinds adapters in this spec emit. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ActionKind { + Read, + Write, + Edit, + Delete, + Bash, + Glob, + Grep, + Search, + WebFetch, + WebSearch, + Mcp, + Think, + Plan, + Subagent, + Other, +} + +impl ActionKind { + /// Stable string tag; used as a sort key where `Discriminant` is not `Ord`. + pub fn as_str(self) -> &'static str { + match self { + ActionKind::Read => "read", + ActionKind::Write => "write", + ActionKind::Edit => "edit", + ActionKind::Delete => "delete", + ActionKind::Bash => "bash", + ActionKind::Glob => "glob", + ActionKind::Grep => "grep", + ActionKind::Search => "search", + ActionKind::WebFetch => "web_fetch", + ActionKind::WebSearch => "web_search", + ActionKind::Mcp => "mcp", + ActionKind::Think => "think", + ActionKind::Plan => "plan", + ActionKind::Subagent => "subagent", + ActionKind::Other => "other", + } + } +} + +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum ActionStatus { + Success, + Failure, + Cancelled, + Unknown, +} + +/// Where a [`TokenEvent`] was captured. Distinguishes exact upstream +/// numbers (proxy) from approximations derived from on-disk transcripts. +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum CaptureSource { + /// Sourced from the on-disk JSONL/SQLite transcript. Approximate. + Transcript, + /// Sourced from the API reverse proxy (future spec). Exact. + Proxy, +} + +/// One normalized tool call. The unit the host stores, queries, and surfaces +/// over MCP. +/// +/// **Contract**: `input` and `output` MUST be scrubbed before this struct +/// leaves the adapter. The injected [`SecretScrubber`][crate::SecretScrubber] +/// is the single chokepoint. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ToolEvent { + pub source_event_id: String, + pub source_file: PathBuf, + pub session_id: String, + pub tool: ToolKind, + pub kind: ActionKind, + #[serde(skip_serializing_if = "Option::is_none")] + pub target: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub input: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub output: Option, + pub status: ActionStatus, + #[serde(skip_serializing_if = "Option::is_none")] + pub error_message: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub started_at_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub duration_ms: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub git_root: Option, + #[serde(default, skip_serializing_if = "serde_json::Value::is_null")] + pub metadata: serde_json::Value, +} + +/// Usage envelope for one assistant turn. All token fields are `Option`: +/// absence means "not reported", never "zero". See spec 010 data-model.md. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TokenEvent { + pub source_event_id: String, + pub session_id: String, + pub tool: ToolKind, + #[serde(skip_serializing_if = "Option::is_none")] + pub model: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub request_id: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub input_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_read_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_creation_tokens: Option, + /// Anthropic 1h ephemeral tier; absent on non-Anthropic traffic. + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_creation_1h_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub output_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub reasoning_tokens: Option, + pub captured_at_ms: i64, + pub captured_via: CaptureSource, +} + +/// Tier-2 (transcript-derived) prompt-cache view emitted by adapters that +/// can see content blocks + usage envelopes. Additive — adapters that don't +/// populate it leave it empty. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CacheObservation { + pub source_event_id: String, + pub session_id: String, + pub tool: ToolKind, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_read_input_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_creation_input_tokens: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub cache_creation_1h_input_tokens: Option, + /// Content-hash of assistant-side blocks; the host uses this for + /// prefix-cache attribution. + #[serde(skip_serializing_if = "Option::is_none")] + pub assistant_blocks_hash: Option, + /// Tool names that differ from the prior turn (a cache invalidation cause). + #[serde(skip_serializing_if = "Vec::is_empty", default)] + pub tools_changed: Vec, + pub observed_at_ms: i64, +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn toolkind_serde_roundtrip() { + for k in [ + ToolKind::ClaudeCode, + ToolKind::Codex, + ToolKind::OpenCode, + ToolKind::Cursor, + ToolKind::Gemini, + ] { + let s = serde_json::to_string(&k).unwrap(); + let back: ToolKind = serde_json::from_str(&s).unwrap(); + assert_eq!(k, back, "serde roundtrip failed for {:?}", k); + assert_eq!(k.as_str(), s.trim_matches('"')); + } + } + + #[test] + fn token_event_absence_is_not_zero() { + // Absent fields deserialize to None, not Some(0). This is the + // invariant from spec 010 data-model.md and spec 009 §5. + let ev: TokenEvent = serde_json::from_str( + r#"{"source_event_id":"x","session_id":"s","tool":"codex", + "captured_at_ms":1,"captured_via":"transcript"}"#, + ) + .unwrap(); + assert_eq!(ev.input_tokens, None); + assert_eq!(ev.cache_read_tokens, None); + } +} diff --git a/aikit-session-capture/src/opencode/db.rs b/aikit-session-capture/src/opencode/db.rs new file mode 100644 index 0000000..21065f1 --- /dev/null +++ b/aikit-session-capture/src/opencode/db.rs @@ -0,0 +1,65 @@ +//! SQLite read-only open + table-exists guard for the OpenCode adapter. +//! +//! See spec 010 §12.3. OpenCode persists state in `opencode.db` (plus +//! `-wal`/`-shm` siblings). The adapter opens it read-only with +//! `query_only(1)` so even a bug in the parser cannot corrupt the live DB. +//! +//! Reference: `superbased-observer/internal/adapter/opencode/adapter.go` +//! (`openReadOnlyDB`, `tableExists`, `latestWatermark`). + +use std::path::Path; + +use rusqlite::{Connection, OpenFlags}; + +use crate::adapter::AdapterError; + +/// Open `opencode.db` read-only with `query_only(1)` and a 2s busy timeout. +/// +/// `busy_timeout(2000)` matters because OpenCode may be actively writing the +/// WAL while we read. The read-only + query_only combo is belt-and-braces: +/// even a stray `UPDATE` from the parser would be rejected at the SQLite +/// layer, never touching the underlying file. +/// +/// `SQLITE_OPEN_URI` enables the URL-form pragma directives +/// (`?mode=ro&query_only=1&busy_timeout=2000`). +pub(crate) fn open_read_only(path: &Path) -> Result { + let flags = OpenFlags::SQLITE_OPEN_READ_ONLY + | OpenFlags::SQLITE_OPEN_URI + | OpenFlags::SQLITE_OPEN_NO_MUTEX; + let url = format!( + "file:{}?mode=ro&query_only=1&busy_timeout=2000", + path.to_string_lossy() + ); + Connection::open_with_flags(&url, flags).map_err(|source| AdapterError::SqliteOpen { + path: path.to_path_buf(), + source, + }) +} + +/// `true` when `table_name` exists in the DB. Used to gracefully degrade on +/// schemas that don't have every table (e.g. older builds without `todo`). +#[allow(dead_code)] // Reserved for future phases (todo / subtask tables). +pub(crate) fn table_exists(db: &Connection, table_name: &str) -> bool { + match db.query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name=?", + [table_name], + |row| row.get::<_, i64>(0), + ) { + Ok(n) => n > 0, + Err(_) => false, + } +} + +/// The high-water `time_updated` across `message`, `part`, and `session` — +/// the adapter's resumable offset. Mirrors observer's `latestWatermark` +/// (`opencode/adapter.go:1125`). +pub(crate) fn latest_watermark(db: &Connection) -> i64 { + let q = "SELECT MAX(v) FROM ( + SELECT COALESCE(MAX(time_updated), 0) AS v FROM message + UNION ALL + SELECT COALESCE(MAX(time_updated), 0) AS v FROM part + UNION ALL + SELECT COALESCE(MAX(time_updated), 0) AS v FROM session + )"; + db.query_row(q, [], |row| row.get::<_, i64>(0)).unwrap_or(0) +} diff --git a/aikit-session-capture/src/opencode/mirror.rs b/aikit-session-capture/src/opencode/mirror.rs new file mode 100644 index 0000000..7046534 --- /dev/null +++ b/aikit-session-capture/src/opencode/mirror.rs @@ -0,0 +1,202 @@ +//! Foreign-mount mirror staging for the OpenCode adapter. +//! +//! See spec 010 §12.3 + §19.3. When `opencode.db` lives on a foreign mount +//! (e.g. `/mnt/c/Users//.local/share/opencode/opencode.db` on a WSL2 +//! Linux host reading the Windows side), `rusqlite` hits +//! `SQLITE_IOERR_SHORT_READ` while the Windows process is actively writing +//! the WAL. The fix: stage a local mirror — copy the trio (`.db` + `-wal` + +//! `-shm`) into a per-source cache dir and open the mirror read-only. +//! +//! Reference: `superbased-observer/internal/adapter/opencode/adapter.go:1176`. +//! +//! # Priority chain (spec 010 §19.3) +//! +//! 1. `dirs::cache_dir()` — preferred; persists across runs. +//! 2. `std::env::temp_dir()` — works on read-only hosts; may not persist. +//! 3. None — fall through to direct-open with retry semantics. + +use std::path::{Path, PathBuf}; + +use crate::adapter::AdapterError; + +/// Pick the mirror root directory according to the §19.3 priority chain. +/// Returns `Err(MirrorError::NoWritableCache)` when no writable candidate +/// is found; callers then open the source directly with retry semantics. +pub(crate) fn pick_mirror_root() -> Result { + // 1. `dirs::cache_dir()` — preferred. + if let Some(c) = dirs::cache_dir() { + if dir_is_writable(&c) { + return Ok(c); + } + } + // 2. `std::env::temp_dir()` — works on read-only hosts. + let t = std::env::temp_dir(); + if dir_is_writable(&t) { + return Ok(t); + } + Err(MirrorError::NoWritableCache) +} + +/// Error returned by `pick_mirror_root`. The adapter turns this into a +/// `ParseResult { retry_suggested: true, warnings: [ForeignMountRetry] }` +/// and the host's poll loop re-attempts on the next tick. +#[derive(Debug)] +pub(crate) enum MirrorError { + NoWritableCache, +} + +/// Stage a mirror for `src_db` (an `opencode.db` path) when it lives on a +/// foreign mount; return the mirror path when one was created or already +/// up-to-date, or `src_db` unchanged when the source is native. +/// +/// Native-mount sources short-circuit: `is_foreign_mount_path` returns +/// false, and the function returns `src_db` without copying anything. +pub(crate) fn stage_mirror_if_foreign(src_db: &Path) -> Result { + if !is_foreign_mount_path(src_db) { + return Ok(src_db.to_path_buf()); + } + let cache_root = pick_mirror_root().map_err(|reason| AdapterError::ForeignMountMirror { + path: src_db.to_path_buf(), + reason: format!("mirror root pick failed: {reason:?}"), + })?; + // Per-source cache subdir keyed by a short hash of the source path so + // multiple foreign-mount sources don't collide. + let hash = short_hash(&src_db.to_string_lossy()); + let mirror_dir = cache_root + .join("aikit-session-capture") + .join("opencode-mirror") + .join(&hash[..8]); + std::fs::create_dir_all(&mirror_dir).map_err(|e| AdapterError::ForeignMountMirror { + path: src_db.to_path_buf(), + reason: format!("mkdir mirror: {e}"), + })?; + let dst_db = mirror_dir.join("opencode.db"); + + if mirror_up_to_date(src_db, &dst_db) { + return Ok(dst_db); + } + + // Copy the trio. Missing siblings are removed from the mirror so a + // stale `-wal` doesn't shadow a freshly-checkpointed source. + for suffix in &["", "-wal", "-shm"] { + let src = format!("{}{suffix}", src_db.to_string_lossy()); + let dst = format!("{}{suffix}", dst_db.to_string_lossy()); + let src_path = Path::new(&src); + match std::fs::read(src_path) { + Ok(data) => { + std::fs::write(&dst, data).map_err(|e| AdapterError::ForeignMountMirror { + path: src_db.to_path_buf(), + reason: format!("write {}: {e}", dst), + })?; + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + let _ = std::fs::remove_file(&dst); // stale sibling cleanup + continue; + } + Err(e) => { + return Err(AdapterError::ForeignMountMirror { + path: src_db.to_path_buf(), + reason: format!("read {src}: {e}"), + }); + } + } + } + Ok(dst_db) +} + +/// `true` when every trio sibling's `(size, mtime)` on the source matches +/// the mirror. Cheap stat check; the WAL mtime is the fast-moving signal. +fn mirror_up_to_date(src_db: &Path, dst_db: &Path) -> bool { + for suffix in &["", "-wal", "-shm"] { + let src_string = format!("{}{suffix}", src_db.to_string_lossy()); + let dst_string = format!("{}{suffix}", dst_db.to_string_lossy()); + let src = Path::new(&src_string); + let dst = Path::new(&dst_string); + if !files_match(src, dst) { + return false; + } + } + true +} + +fn files_match(src: &Path, dst: &Path) -> bool { + let s = match std::fs::metadata(src) { + Ok(m) => m, + Err(_) => return false, + }; + let d = match std::fs::metadata(dst) { + Ok(m) => m, + Err(_) => return false, + }; + if s.len() != d.len() { + return false; + } + use std::time::SystemTime; + let s_mod = s.modified().unwrap_or(SystemTime::UNIX_EPOCH); + let d_mod = d.modified().unwrap_or(SystemTime::UNIX_EPOCH); + // `true` when the source is newer than the destination → not up-to-date. + s_mod <= d_mod +} + +/// Detect whether `path` lives on a foreign mount — a filesystem whose +/// device id differs from the cache dir's device id. Linux/WSL2 /mnt/c is +/// the canonical case. +/// +/// On platforms where `device_id` extraction isn't supported, returns +/// `false` (assume native). The cost of a false negative is that foreign- +/// mount sources open directly and may hit `SQLITE_IOERR_SHORT_READ`; the +/// adapter emits `ParseWarning::ForeignMountRetry` and the poll loop +/// re-attempts on the next tick. +pub(crate) fn is_foreign_mount_path(path: &Path) -> bool { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let src_dev = match std::fs::metadata(path) { + Ok(m) => m.dev(), + Err(_) => return false, + }; + let cache_dev = match dirs::cache_dir().and_then(|c| std::fs::metadata(&c).ok()) { + Some(m) => m.dev(), + None => return false, + }; + src_dev != cache_dev + } + #[cfg(not(unix))] + { + // Non-Unix (Windows, wasm, …) — assume native. The mirror logic is + // only load-bearing on Linux/WSL2 reading /mnt/c. + let _ = path; + false + } +} + +fn dir_is_writable(p: &Path) -> bool { + std::fs::metadata(p) + .and_then(|m| { + if m.is_dir() { + // Try writing a probe file. Cheaper than a full fsync. + let probe = p.join(".aikit-session-capture-write-probe"); + std::fs::write(&probe, b"")?; + let _ = std::fs::remove_file(&probe); + Ok(()) + } else { + // Not a directory. Use a generic error kind so the MSRV + // check stays clean (ErrorKind::NotADirectory is 1.83+). + Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "not a directory", + )) + } + }) + .is_ok() +} + +fn short_hash(s: &str) -> String { + // FNV-1a 64-bit, full hex. Deterministic per path. + let mut hash: u64 = 0xcbf29ce484222325; + for &b in s.as_bytes() { + hash ^= b as u64; + hash = hash.wrapping_mul(0x100000001b3); + } + format!("{:016x}", hash) +} diff --git a/aikit-session-capture/src/opencode/mod.rs b/aikit-session-capture/src/opencode/mod.rs new file mode 100644 index 0000000..e418921 --- /dev/null +++ b/aikit-session-capture/src/opencode/mod.rs @@ -0,0 +1,701 @@ +//! OpenCode adapter: parses `~/.opencode/opencode.db` (SQLite). +//! +//! See spec 010 §12.3. **Different shape from Claude/Codex** — OpenCode +//! persists state in SQLite, not JSONL. The adapter queries the DB rather +//! than streaming bytes; `from_offset` is a `time_updated` watermark, NOT a +//! byte offset. The trait signature tolerates this because `from_offset: +//! u64` is opaque. +//! +//! Watch paths (cross-mount union, mirrors `opencode/adapter.go:1323`): +//! - sst/opencode CLI (canonical): `~/.opencode/` +//! - XDG_DATA fallback: `~/.local/share/opencode/` +//! - Desktop variant (per-OS): `~/AppData/Roaming/ai.opencode.desktop/` +//! (Windows), `~/Library/Application Support/ai.opencode.desktop/` +//! (macOS), `~/.config/ai.opencode.desktop/` (Linux) + +mod db; +mod mirror; + +use std::path::{Path, PathBuf}; + +use async_trait::async_trait; +use rusqlite::params; + +#[allow(unused_imports)] +use crate::adapter::ParseWarning; // retained for future ForeignMountRetry emission +use crate::adapter::{Adapter, AdapterError, ParseResult}; +use crate::homes::HomeResolver; +use crate::models::{ActionKind, ActionStatus, CaptureSource, TokenEvent, ToolEvent, ToolKind}; +use crate::scrub::SecretScrubber; + +pub struct OpenCodeAdapter { + scrubber: SecretScrubber, + homes: Vec, + override_roots: Option>, +} + +impl OpenCodeAdapter { + pub fn new() -> Self { + Self { + scrubber: SecretScrubber::default(), + homes: crate::homes::DefaultHomeResolver.homes(), + override_roots: None, + } + } + + pub fn with(scrubber: SecretScrubber, homes: Vec) -> Self { + Self { + scrubber, + homes, + override_roots: None, + } + } + + pub fn with_override_roots(mut self, roots: Vec) -> Self { + self.override_roots = Some(roots); + self + } +} + +impl Default for OpenCodeAdapter { + fn default() -> Self { + Self::new() + } +} + +#[async_trait] +impl Adapter for OpenCodeAdapter { + fn kind(&self) -> ToolKind { + ToolKind::OpenCode + } + + fn watch_paths(&self) -> Vec { + if let Some(roots) = &self.override_roots { + return roots.clone(); + } + // Crossmount-resolved per-home expansion. Mirrors observer's + // `defaultRoots()` (opencode/adapter.go:1323). + let mut roots = Vec::new(); + for h in &self.homes { + roots.push(h.path.join(".opencode")); + roots.push(h.path.join(".local").join("share").join("opencode")); + match h.os { + crate::homes::HomeOs::Windows => { + roots.push( + h.path + .join("AppData") + .join("Roaming") + .join("ai.opencode.desktop"), + ); + } + crate::homes::HomeOs::Darwin => { + roots.push( + h.path + .join("Library") + .join("Application Support") + .join("ai.opencode.desktop"), + ); + } + crate::homes::HomeOs::Linux => { + roots.push(h.path.join(".config").join("ai.opencode.desktop")); + } + } + } + roots + } + + fn is_session_file(&self, path: &Path) -> bool { + let Some(name) = path.file_name().and_then(|n| n.to_str()) else { + return false; + }; + let lower = name.to_ascii_lowercase(); + if lower != "opencode.db" && lower != "opencode.db-wal" { + return false; + } + self.watch_paths().iter().any(|root| path.starts_with(root)) + } + + async fn parse_session_file( + &self, + path: &Path, + from_offset: u64, + ) -> Result { + // Stage a mirror for foreign-mount sources (spec §19.3). Native + // sources short-circuit and open directly. + let db_path = mirror::stage_mirror_if_foreign(path)?; + // Open inside spawn_blocking — rusqlite is sync. + let scrubber = self.scrubber.clone(); + let path_owned = path.to_path_buf(); + let db_path = tokio::task::spawn_blocking(move || -> Result { + let conn = db::open_read_only(&db_path)?; + let mut res = ParseResult { + new_offset: from_offset, + ..Default::default() + }; + + // High-water mark → if no new rows, short-circuit. + let latest = db::latest_watermark(&conn); + if latest <= from_offset as i64 { + // No new rows since the last parse. Return the actual + // high-water mark so the host stores the truth (not the + // inflated requested offset). + res.new_offset = latest as u64; + return Ok(res); + } + + // Tool events from the `part` table (type='tool'). + res.tool_events.extend(load_tool_events( + &conn, + &path_owned, + from_offset as i64, + &scrubber, + )?); + + // User-prompt events from `message` table (role='user'). + res.tool_events.extend(load_user_prompts( + &conn, + &path_owned, + from_offset as i64, + &scrubber, + )?); + + // Token events from `message` (role='assistant'). + res.token_events = load_token_events(&conn, &path_owned, from_offset as i64)?; + + res.new_offset = latest as u64; + Ok(res) + }) + .await + .map_err(|e| AdapterError::Other(anyhow::anyhow!("spawn_blocking join: {e}")))??; + + Ok(db_path) + } +} + +// --------------------------------------------------------------------------- +// SQL query helpers — mirror observer's loadToolEvents / loadUserPromptEvents +// / loadTokenEvents (opencode/adapter.go:268, 360, 823). +// --------------------------------------------------------------------------- + +fn load_tool_events( + db: &rusqlite::Connection, + source_file: &Path, + from_offset: i64, + scrubber: &SecretScrubber, +) -> Result, AdapterError> { + let mut stmt = db + .prepare( + "SELECT p.id, p.message_id, p.session_id, \ + COALESCE(s.directory, ''), p.time_created, p.time_updated, p.data, m.data \ + FROM part p \ + JOIN message m ON m.id = p.message_id \ + LEFT JOIN session s ON s.id = p.session_id \ + WHERE p.time_updated > ?1 \ + AND json_extract(p.data, '$.type') = 'tool' \ + ORDER BY p.time_updated ASC, p.id ASC", + ) + .map_err(|e| AdapterError::SqliteOpen { + path: source_file.to_path_buf(), + source: e, + })?; + let rows = stmt + .query_map(params![from_offset], |row| { + Ok(PartRow { + id: row.get(0)?, + message_id: row.get(1)?, + session_id: row.get(2)?, + directory: row.get::<_, Option>(3)?.unwrap_or_default(), + time_created: row.get(4)?, + time_updated: row.get(5)?, + data: row.get(6)?, + message: row.get(7)?, + }) + }) + .map_err(|e| AdapterError::SqliteOpen { + path: source_file.to_path_buf(), + source: e, + })?; + let mut out = Vec::new(); + for row in rows { + let row = row.map_err(|e| AdapterError::SqliteOpen { + path: source_file.to_path_buf(), + source: e, + })?; + if let Some(ev) = build_tool_event(source_file, &row, scrubber) { + out.push(ev); + } + } + Ok(out) +} + +fn build_tool_event( + source_file: &Path, + row: &PartRow, + scrubber: &SecretScrubber, +) -> Option { + let msg: MessageData = serde_json::from_str(&row.message).ok()?; + let part: ToolPartData = serde_json::from_str(&row.data).ok()?; + if part.r#type != "tool" { + return None; + } + let (kind, target, success, err_msg) = map_tool(&part); + let when_ms = if part.state.time.start > 0 { + Some(part.state.time.start) + } else if msg.time.created > 0 { + Some(msg.time.created) + } else { + row.time_created + }; + let duration_ms = if part.state.time.start > 0 && part.state.time.end > part.state.time.start { + Some((part.state.time.end - part.state.time.start) as u64) + } else { + None + }; + let raw_input = part + .state + .input + .as_ref() + .map(|v| v.to_string()) + .unwrap_or_default(); + let scrubbed_input = if raw_input.is_empty() { + None + } else { + Some(scrubber.scrub(&raw_input)) + }; + let output = if !part.state.output.is_empty() { + part.state.output.clone() + } else { + part.state.metadata.output.clone() + }; + let scrubbed_output = if output.is_empty() { + None + } else { + Some(scrubber.scrub(&output)) + }; + Some(ToolEvent { + source_event_id: format!("opencode:part:{}", row.id), + source_file: source_file.to_path_buf(), + session_id: row.session_id.clone(), + tool: ToolKind::OpenCode, + kind, + target: target.map(|t| truncate_str(&scrubber.scrub(&t), 200).to_string()), + input: scrubbed_input, + output: scrubbed_output, + status: if success { + ActionStatus::Success + } else { + ActionStatus::Failure + }, + error_message: err_msg.map(|m| truncate_str(&m, 500).to_string()), + started_at_ms: when_ms, + duration_ms, + git_root: if msg.path.cwd.is_empty() { + if row.directory.is_empty() { + None + } else { + Some(PathBuf::from(&row.directory)) + } + } else { + Some(PathBuf::from(&msg.path.cwd)) + }, + metadata: serde_json::json!({ + "tool": part.tool, + "model": msg.model.model_id, + "variant": msg.variant, + }), + }) +} + +fn map_tool(part: &ToolPartData) -> (ActionKind, Option, bool, Option) { + let input: ToolInput = part + .state + .input + .as_ref() + .and_then(|v| serde_json::from_value(v.clone()).ok()) + .unwrap_or_default(); + let tool_lower = part.tool.to_ascii_lowercase(); + let mut success = + part.state.status.is_empty() || part.state.status.eq_ignore_ascii_case("completed"); + let mut err_msg = None; + let kind = match tool_lower.as_str() { + "bash" | "shell" | "command" | "powershell" | "pwsh" | "cmd" | "cmd.exe" => { + if part.state.metadata.exit != 0 { + success = false; + err_msg = if !part.state.output.is_empty() { + Some(part.state.output.clone()) + } else { + Some(part.state.metadata.output.clone()) + }; + } + ActionKind::Bash + } + "read" | "cat" | "view" => ActionKind::Read, + "write" | "create" => ActionKind::Write, + "edit" | "patch" | "replace" | "multiedit" | "applypatch" | "apply_patch" => { + ActionKind::Edit + } + "grep" | "search" | "rg" => ActionKind::Search, + "glob" | "find" | "ls" => ActionKind::Glob, + "webfetch" | "fetch" | "http" => ActionKind::WebFetch, + "websearch" => ActionKind::WebSearch, + "task" | "agent" | "subagent" => ActionKind::Subagent, + "todoread" | "todowrite" | "todo" => ActionKind::Plan, + n if n.contains("mcp") => ActionKind::Mcp, + _ => ActionKind::Other, + }; + let target = input + .command + .clone() + .or_else(|| input.file_path.clone()) + .or_else(|| part.state.metadata.filepath.clone()) + .or_else(|| part.state.title.clone()) + .or_else(|| Some(part.tool.clone()).filter(|s| !s.is_empty())); + (kind, target, success, err_msg) +} + +fn load_user_prompts( + db: &rusqlite::Connection, + source_file: &Path, + from_offset: i64, + scrubber: &SecretScrubber, +) -> Result, AdapterError> { + let mut stmt = db + .prepare( + "SELECT m.id, m.session_id, COALESCE(s.directory, ''), \ + m.time_created, m.time_updated, m.data \ + FROM message m \ + LEFT JOIN session s ON s.id = m.session_id \ + WHERE m.time_updated > ?1 \ + AND json_extract(m.data, '$.role') = 'user' \ + ORDER BY m.time_updated ASC, m.id ASC", + ) + .map_err(|e| AdapterError::SqliteOpen { + path: source_file.to_path_buf(), + source: e, + })?; + let rows = stmt + .query_map(params![from_offset], |row| { + Ok(MessageRow { + id: row.get(0)?, + session_id: row.get(1)?, + directory: row.get::<_, Option>(2)?.unwrap_or_default(), + time_created: row.get(3)?, + time_updated: row.get(4)?, + data: row.get(5)?, + }) + }) + .map_err(|e| AdapterError::SqliteOpen { + path: source_file.to_path_buf(), + source: e, + })?; + + let mut out = Vec::new(); + for row in rows { + let row = row.map_err(|e| AdapterError::SqliteOpen { + path: source_file.to_path_buf(), + source: e, + })?; + let msg: MessageData = match serde_json::from_str(&row.data) { + Ok(m) => m, + Err(_) => continue, + }; + // Walk parts to collect text content. + let text = collect_message_text(db, &row.id, scrubber); + if text.trim().is_empty() { + continue; + } + let when_ms = if msg.time.created > 0 { + Some(msg.time.created) + } else { + row.time_created + }; + out.push(ToolEvent { + source_event_id: format!("opencode:message:{}", row.id), + source_file: source_file.to_path_buf(), + session_id: row.session_id.clone(), + tool: ToolKind::OpenCode, + kind: ActionKind::Other, + target: Some(truncate_str(&text, 200).to_string()), + input: Some(scrubber.scrub(&text)), + output: None, + status: ActionStatus::Success, + error_message: None, + started_at_ms: when_ms, + duration_ms: None, + git_root: if msg.path.cwd.is_empty() && row.directory.is_empty() { + None + } else if !msg.path.cwd.is_empty() { + Some(PathBuf::from(&msg.path.cwd)) + } else { + Some(PathBuf::from(&row.directory)) + }, + metadata: serde_json::json!({ "kind": "user_prompt" }), + }); + } + Ok(out) +} + +fn collect_message_text( + db: &rusqlite::Connection, + message_id: &str, + scrubber: &SecretScrubber, +) -> String { + let Ok(mut stmt) = + db.prepare("SELECT data FROM part WHERE message_id = ?1 ORDER BY time_created ASC, id ASC") + else { + return String::new(); + }; + let Ok(rows) = stmt.query_map([message_id], |row| row.get::<_, String>(0)) else { + return String::new(); + }; + let mut parts = Vec::new(); + for row in rows.flatten() { + if let Ok(text_part) = serde_json::from_str::(&row) { + if text_part.r#type == "text" && !text_part.text.trim().is_empty() { + parts.push(text_part.text); + } + } + } + scrubber.scrub(&parts.join("\n")) +} + +fn load_token_events( + db: &rusqlite::Connection, + source_file: &Path, + from_offset: i64, +) -> Result, AdapterError> { + let mut stmt = db + .prepare( + "SELECT m.id, m.session_id, COALESCE(s.directory, ''), \ + m.time_created, m.time_updated, m.data \ + FROM message m \ + LEFT JOIN session s ON s.id = m.session_id \ + WHERE m.time_updated > ?1 \ + AND json_extract(m.data, '$.role') = 'assistant' \ + ORDER BY m.time_updated ASC, m.id ASC", + ) + .map_err(|e| AdapterError::SqliteOpen { + path: source_file.to_path_buf(), + source: e, + })?; + let rows = stmt + .query_map(params![from_offset], |row| { + Ok(MessageRow { + id: row.get(0)?, + session_id: row.get(1)?, + directory: row.get::<_, Option>(2)?.unwrap_or_default(), + time_created: row.get(3)?, + time_updated: row.get(4)?, + data: row.get(5)?, + }) + }) + .map_err(|e| AdapterError::SqliteOpen { + path: source_file.to_path_buf(), + source: e, + })?; + let mut out = Vec::new(); + for row in rows { + let row = row.map_err(|e| AdapterError::SqliteOpen { + path: source_file.to_path_buf(), + source: e, + })?; + let msg: MessageData = match serde_json::from_str(&row.data) { + Ok(m) => m, + Err(_) => continue, + }; + // Skip rows with no token data (in-progress turn). + if msg.tokens.input == 0 + && msg.tokens.output == 0 + && msg.tokens.cache.read == 0 + && msg.tokens.cache.write == 0 + && msg.tokens.reasoning == 0 + { + continue; + } + let when_ms = if msg.time.completed > 0 { + Some(msg.time.completed) + } else { + Some(row.time_updated) + }; + out.push(TokenEvent { + source_event_id: format!("opencode:tokens:{}", row.id), + session_id: row.session_id.clone(), + tool: ToolKind::OpenCode, + model: if !msg.model_id.is_empty() { + Some(msg.model_id.clone()) + } else if !msg.model.model_id.is_empty() { + Some(msg.model.model_id.clone()) + } else { + None + }, + request_id: None, + input_tokens: Some(msg.tokens.input as u64), + cache_read_tokens: Some(msg.tokens.cache.read as u64), + cache_creation_tokens: Some(msg.tokens.cache.write as u64), + cache_creation_1h_tokens: None, // OpenCode doesn't distinguish 1h tier. + output_tokens: Some(msg.tokens.output as u64), + reasoning_tokens: Some(msg.tokens.reasoning as u64), + captured_at_ms: when_ms.unwrap_or(0), + captured_via: CaptureSource::Transcript, + }); + } + Ok(out) +} + +fn truncate_str(s: &str, max: usize) -> &str { + if s.len() <= max { + s + } else { + let mut end = max; + while end > 0 && !s.is_char_boundary(end) { + end -= 1; + } + &s[..end] + } +} + +// --------------------------------------------------------------------------- +// Row + JSON types +// --------------------------------------------------------------------------- + +struct PartRow { + id: String, + #[allow(dead_code)] + message_id: String, + session_id: String, + directory: String, + time_created: Option, + #[allow(dead_code)] + time_updated: i64, + data: String, + message: String, +} + +struct MessageRow { + id: String, + session_id: String, + directory: String, + time_created: Option, + #[allow(dead_code)] + time_updated: i64, + data: String, +} + +#[derive(serde::Deserialize)] +struct MessageData { + #[serde(default)] + #[allow(dead_code)] + role: String, + #[serde(default, rename = "modelID")] + model_id: String, + #[serde(default)] + model: ModelBlock, + #[serde(default)] + path: PathBlock, + #[serde(default)] + time: TimeBlock, + #[serde(default)] + variant: String, + #[serde(default)] + tokens: TokensBlock, +} + +#[derive(serde::Deserialize, Default)] +struct ModelBlock { + #[serde(default, rename = "modelID")] + model_id: String, +} + +#[derive(serde::Deserialize, Default)] +struct PathBlock { + #[serde(default)] + cwd: String, +} + +#[derive(serde::Deserialize, Default)] +struct TimeBlock { + #[serde(default)] + created: i64, + #[serde(default)] + completed: i64, +} + +#[derive(serde::Deserialize, Default)] +struct TokensBlock { + #[serde(default)] + input: i64, + #[serde(default)] + output: i64, + #[serde(default)] + reasoning: i64, + #[serde(default)] + cache: CacheBlock, +} + +#[derive(serde::Deserialize, Default)] +struct CacheBlock { + #[serde(default)] + read: i64, + #[serde(default)] + write: i64, +} + +#[derive(serde::Deserialize)] +struct ToolPartData { + #[serde(rename = "type")] + r#type: String, + tool: String, + #[allow(dead_code)] + #[serde(default, rename = "callID")] + call_id: String, + state: ToolState, +} + +#[derive(serde::Deserialize)] +struct ToolState { + #[serde(default)] + status: String, + #[serde(default)] + input: Option, + #[serde(default)] + output: String, + #[serde(default)] + metadata: ToolMetadata, + #[serde(default)] + title: Option, + #[serde(default)] + time: ToolTime, +} + +#[derive(serde::Deserialize, Default)] +struct ToolMetadata { + #[serde(default)] + output: String, + #[serde(default)] + exit: i64, + #[serde(default)] + filepath: Option, +} + +#[derive(serde::Deserialize, Default)] +struct ToolTime { + #[serde(default)] + start: i64, + #[serde(default)] + end: i64, +} + +#[derive(serde::Deserialize, Default)] +struct ToolInput { + #[serde(default)] + command: Option, + #[serde(default, rename = "filePath")] + file_path: Option, +} + +#[derive(serde::Deserialize)] +struct TextPartData { + #[serde(rename = "type")] + r#type: String, + text: String, +} diff --git a/aikit-session-capture/src/registry.rs b/aikit-session-capture/src/registry.rs new file mode 100644 index 0000000..d5ecf57 --- /dev/null +++ b/aikit-session-capture/src/registry.rs @@ -0,0 +1,163 @@ +//! Adapter registry with auto-detection based on watch-path existence. +//! +//! See spec 010 §7. Mirrors `superbased-observer/internal/adapter/registry.go`. +//! +//! The nil-vs-empty allow-list distinction is load-bearing for TOML configs: +//! `None` = all adapters considered; `Some(&[])` = none; `Some(&[k])` = +//! restrict to those kinds. + +use std::path::Path; + +use crate::adapter::Adapter; +use crate::models::ToolKind; + +/// Holds all known adapters, in registration order. +#[derive(Default)] +pub struct Registry { + adapters: Vec>, +} + +impl Registry { + /// Empty registry. + pub fn new() -> Self { + Self::default() + } + + /// Add an adapter. Registration order is preserved by [`Registry::all`]. + pub fn register(&mut self, adapter: Box) { + self.adapters.push(adapter); + } + + /// Returns the first adapter matching `kind`, or `None`. + pub fn get(&self, kind: ToolKind) -> Option<&dyn Adapter> { + self.adapters + .iter() + .find(|a| a.kind() == kind) + .map(|a| a.as_ref()) + } + + /// Every registered adapter, in registration order. + pub fn all(&self) -> Vec<&dyn Adapter> { + self.adapters.iter().map(|a| a.as_ref()).collect() + } + + /// Filter to adapters whose `watch_paths()` include at least one + /// directory that currently exists. Allow-list semantics (spec §7): + /// + /// - `allow == None` → no filter (all adapters considered). + /// - `allow == Some(empty)` → filter to *zero* adapters. This is the + /// explicit user-intent case: a config `[adapter] enabled = []` + /// disables passive capture rather than silently falling through. + /// - `allow == Some(non-empty)`→ restrict to the named kinds. + pub fn detected(&self, allow: Option<&[ToolKind]>) -> Vec<&dyn Adapter> { + if let Some([]) = allow { + return Vec::new(); + } + let allow_set: Option> = + allow.map(|ks| ks.iter().copied().collect()); + self.adapters + .iter() + .filter(|a| { + if let Some(set) = &allow_set { + if !set.contains(&a.kind()) { + return false; + } + } + any_dir_exists(&a.watch_paths()) + }) + .map(|a| a.as_ref()) + .collect() + } +} + +fn any_dir_exists(paths: &[std::path::PathBuf]) -> bool { + paths.iter().any(|p| p.is_dir()) +} + +// Lint: `Path` import kept for the `any_dir_exists` signature readability; +// the function uses `is_dir()` which is on `PathBuf`/`Path` interchangeably. +#[allow(clippy::needless_borrow)] +fn _path_typecheck(_: &Path) {} + +#[cfg(test)] +mod tests { + use super::*; + use async_trait::async_trait; + use std::path::PathBuf; + + struct FakeAdapter { + kind: ToolKind, + paths: Vec, + } + + #[async_trait] + impl Adapter for FakeAdapter { + fn kind(&self) -> ToolKind { + self.kind + } + fn watch_paths(&self) -> Vec { + self.paths.clone() + } + fn is_session_file(&self, _path: &Path) -> bool { + false + } + async fn parse_session_file( + &self, + _path: &Path, + _from_offset: u64, + ) -> Result { + Ok(crate::ParseResult::default()) + } + } + + #[test] + fn allow_none_considers_all() { + let mut r = Registry::new(); + r.register(Box::new(FakeAdapter { + kind: ToolKind::ClaudeCode, + paths: vec![std::env::temp_dir()], + })); + r.register(Box::new(FakeAdapter { + kind: ToolKind::Codex, + paths: vec![std::env::temp_dir()], + })); + assert_eq!(r.detected(None).len(), 2); + } + + #[test] + fn allow_empty_filters_to_zero() { + let mut r = Registry::new(); + r.register(Box::new(FakeAdapter { + kind: ToolKind::ClaudeCode, + paths: vec![std::env::temp_dir()], + })); + // The explicit "disable everything" case — not a fallback to "all". + assert_eq!(r.detected(Some(&[])).len(), 0); + } + + #[test] + fn allow_specific_restricts() { + let mut r = Registry::new(); + r.register(Box::new(FakeAdapter { + kind: ToolKind::ClaudeCode, + paths: vec![std::env::temp_dir()], + })); + r.register(Box::new(FakeAdapter { + kind: ToolKind::Codex, + paths: vec![std::env::temp_dir()], + })); + let got = r.detected(Some(&[ToolKind::ClaudeCode])); + assert_eq!(got.len(), 1); + assert_eq!(got[0].kind(), ToolKind::ClaudeCode); + } + + #[test] + fn nonexistent_paths_filtered() { + let mut r = Registry::new(); + r.register(Box::new(FakeAdapter { + kind: ToolKind::ClaudeCode, + paths: vec![PathBuf::from("/this/does/not/exist/anywhere/expected")], + })); + assert_eq!(r.detected(None).len(), 0); + } +} diff --git a/aikit-session-capture/src/scrub.rs b/aikit-session-capture/src/scrub.rs new file mode 100644 index 0000000..c855967 --- /dev/null +++ b/aikit-session-capture/src/scrub.rs @@ -0,0 +1,162 @@ +//! `SecretScrubber`: the single chokepoint that enforces "no secrets escape +//! the adapter". Every `ToolEvent`'s `input` and `output` passes through +//! `scrub()` before the parse result is returned. +//! +//! See spec 010 §9. Default patterns cover the common credential shapes; +//! hosts add their own via [`SecretScrubber::with_pattern`]. + +use regex::Regex; + +/// Replaces credential matches in adapter output with `[REDACTED: