diff --git a/Cargo.lock b/Cargo.lock index 624c100..9bde219 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -31,7 +31,7 @@ dependencies = [ [[package]] name = "aikit-agent" version = "0.1.0" -source = "git+https://github.com/goaikit/aikit?branch=main#dff63ec966a6ad9f310f00e31a27d3e6f195a549" +source = "git+https://github.com/goaikit/aikit?branch=main#a22a619fa61d2fce7f7364c18199b75b72d74e4b" dependencies = [ "reqwest", "serde", @@ -41,10 +41,27 @@ dependencies = [ "tracing", ] +[[package]] +name = "aikit-evals" +version = "0.1.0" +source = "git+https://github.com/goaikit/aikit?branch=main#a22a619fa61d2fce7f7364c18199b75b72d74e4b" +dependencies = [ + "aikit-sdk", + "async-trait", + "base64 0.22.1", + "num_cpus", + "serde", + "serde_json", + "thiserror 2.0.18", + "tokio", + "toml 1.1.2+spec-1.1.0", + "tracing", +] + [[package]] name = "aikit-sdk" -version = "0.2.1" -source = "git+https://github.com/goaikit/aikit?branch=main#dff63ec966a6ad9f310f00e31a27d3e6f195a549" +version = "0.3.0" +source = "git+https://github.com/goaikit/aikit?branch=main#a22a619fa61d2fce7f7364c18199b75b72d74e4b" dependencies = [ "aikit-agent", "dirs", @@ -1663,18 +1680,10 @@ dependencies = [ name = "fastskill-evals" version = "0.9.114" dependencies = [ - "aikit-sdk", - "async-trait", - "base64 0.22.1", + "aikit-evals", "fastskill-core", - "num_cpus", - "serde", - "serde_json", "tempfile", - "thiserror 2.0.18", - "tokio", "toml 0.8.23", - "tracing", ] [[package]] @@ -2302,7 +2311,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.3", + "socket2 0.5.10", "tokio", "tower-service", "tracing", @@ -3197,7 +3206,7 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls 0.23.38", - "socket2 0.6.3", + "socket2 0.5.10", "thiserror 2.0.18", "tokio", "tracing", @@ -3234,7 +3243,7 @@ dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.6.3", + "socket2 0.5.10", "tracing", "windows-sys 0.60.2", ] @@ -4833,7 +4842,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.48.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 9f5c441..7063c13 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,6 +17,8 @@ repository = "https://github.com/aroff/fastskill" [workspace.dependencies] # aikit-sdk for agent metadata file resolution aikit-sdk = { git = "https://github.com/goaikit/aikit", branch = "main" } +# aikit-evals provides the generic eval runner infrastructure +aikit-evals = { git = "https://github.com/goaikit/aikit", branch = "main" } # Async runtime tokio = { version = "1.0", features = ["rt-multi-thread", "net", "fs", "io-util", "macros", "process"] } diff --git a/crates/fastskill-cli/src/commands/init.rs b/crates/fastskill-cli/src/commands/init.rs index 9df4362..cf711db 100644 --- a/crates/fastskill-cli/src/commands/init.rs +++ b/crates/fastskill-cli/src/commands/init.rs @@ -480,7 +480,20 @@ mod tests { #[tokio::test] async fn test_execute_init_with_all_args() { + let _lock = fastskill_core::test_utils::DIR_MUTEX + .lock() + .unwrap_or_else(|e| e.into_inner()); let temp_dir = TempDir::new().unwrap(); + let original_dir = std::env::current_dir().ok(); + struct DirGuard(Option); + impl Drop for DirGuard { + fn drop(&mut self) { + if let Some(dir) = &self.0 { + let _ = std::env::set_current_dir(dir); + } + } + } + let _guard = DirGuard(original_dir); std::env::set_current_dir(temp_dir.path()).unwrap(); let args = InitArgs { @@ -505,7 +518,20 @@ mod tests { #[tokio::test] async fn test_execute_init_with_invalid_version() { + let _lock = fastskill_core::test_utils::DIR_MUTEX + .lock() + .unwrap_or_else(|e| e.into_inner()); let temp_dir = TempDir::new().unwrap(); + let original_dir = std::env::current_dir().ok(); + struct DirGuard(Option); + impl Drop for DirGuard { + fn drop(&mut self) { + if let Some(dir) = &self.0 { + let _ = std::env::set_current_dir(dir); + } + } + } + let _guard = DirGuard(original_dir); std::env::set_current_dir(temp_dir.path()).unwrap(); let args = InitArgs { @@ -532,7 +558,20 @@ mod tests { // Note: This test verifies that skill ID validation works // The ID is derived from directory name, so we can't easily test invalid IDs here // Invalid ID validation happens when the directory name is invalid + let _lock = fastskill_core::test_utils::DIR_MUTEX + .lock() + .unwrap_or_else(|e| e.into_inner()); let temp_dir = TempDir::new().unwrap(); + let original_dir = std::env::current_dir().ok(); + struct DirGuard(Option); + impl Drop for DirGuard { + fn drop(&mut self) { + if let Some(dir) = &self.0 { + let _ = std::env::set_current_dir(dir); + } + } + } + let _guard = DirGuard(original_dir); std::env::set_current_dir(temp_dir.path()).unwrap(); let args = InitArgs { diff --git a/crates/fastskill-evals/Cargo.toml b/crates/fastskill-evals/Cargo.toml index 5ed9149..a22710a 100644 --- a/crates/fastskill-evals/Cargo.toml +++ b/crates/fastskill-evals/Cargo.toml @@ -11,17 +11,8 @@ categories = ["development-tools::testing"] [dependencies] fastskill-core = { path = "../fastskill-core", default-features = false, features = ["filesystem-storage"] } -aikit-sdk.workspace = true -tokio.workspace = true -async-trait.workspace = true -serde.workspace = true -serde_json.workspace = true +aikit-evals.workspace = true toml.workspace = true -thiserror.workspace = true -base64.workspace = true -num_cpus.workspace = true -tracing.workspace = true -tempfile.workspace = true [dev-dependencies] tempfile.workspace = true diff --git a/crates/fastskill-evals/src/artifacts.rs b/crates/fastskill-evals/src/artifacts.rs deleted file mode 100644 index a70ea57..0000000 --- a/crates/fastskill-evals/src/artifacts.rs +++ /dev/null @@ -1,433 +0,0 @@ -//! Artifact layout and persistence for eval runs - -use crate::checks::CheckResult; -use serde::{Deserialize, Serialize}; -use std::path::{Path, PathBuf}; -use thiserror::Error; - -/// Status of a single eval case -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(rename_all = "lowercase")] -pub enum CaseStatus { - Passed, - Failed, - Error, - Skipped, -} - -impl std::fmt::Display for CaseStatus { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - CaseStatus::Passed => write!(f, "passed"), - CaseStatus::Failed => write!(f, "failed"), - CaseStatus::Error => write!(f, "error"), - CaseStatus::Skipped => write!(f, "skipped"), - } - } -} - -/// Per-case result -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CaseResult { - pub id: String, - pub status: CaseStatus, - pub command_count: Option, - pub input_tokens: Option, - pub output_tokens: Option, - #[serde(default)] - pub check_results: Vec, - pub error_message: Option, -} - -/// Per-trial result for a case -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TrialResult { - pub trial_id: u32, - pub status: CaseStatus, - pub command_count: Option, - pub input_tokens: Option, - pub output_tokens: Option, - #[serde(default)] - pub check_results: Vec, - pub error_message: Option, -} - -/// Aggregated results for a case across multiple trials -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CaseTrialsResult { - pub id: String, - pub trials: Vec, - pub aggregated_status: CaseStatus, - pub pass_count: u32, - pub total_trials: u32, - pub pass_rate: f64, -} - -/// Aggregated run summary -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct SummaryResult { - pub suite_pass: bool, - #[serde(default)] - pub suite_pass_rate: Option, - pub agent: String, - pub model: Option, - pub total_cases: usize, - pub passed: usize, - pub failed: usize, - #[serde(default)] - pub trials_per_case: Option, - #[serde(default)] - pub parallel: Option, - #[serde(default)] - pub pass_threshold: Option, - pub run_dir: PathBuf, - pub checks_path: Option, - pub skill_project_root: PathBuf, - pub cases: Vec, -} - -/// Per-case summary entry in summary.json -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CaseSummary { - pub id: String, - pub status: CaseStatus, - pub command_count: Option, - pub input_tokens: Option, - pub output_tokens: Option, - #[serde(default)] - pub pass_count: Option, - #[serde(default)] - pub total_trials: Option, - #[serde(default)] - pub pass_rate: Option, - #[serde(default)] - pub trials: Vec, -} - -/// All artifacts from a completed run -#[derive(Debug)] -pub struct RunArtifacts { - pub run_id: String, - pub run_dir: PathBuf, - pub summary: SummaryResult, - pub case_results: Vec, -} - -/// Errors during artifact writing/reading -#[derive(Debug, Error)] -pub enum ArtifactsError { - #[error("EVAL_ARTIFACTS_CORRUPT: IO error: {0}")] - Io(#[from] std::io::Error), - #[error("EVAL_ARTIFACTS_CORRUPT: JSON error: {0}")] - Json(#[from] serde_json::Error), - #[error("EVAL_ARTIFACTS_CORRUPT: Missing required field: {0}")] - MissingField(String), -} - -/// Allocate a run directory under output_dir using ISO 8601 timestamp format -/// Appends numeric suffix if directory already exists -pub fn allocate_run_dir(output_dir: &Path, run_id: &str) -> Result { - let base = output_dir.join(run_id); - if !base.exists() { - std::fs::create_dir_all(&base)?; - return Ok(base); - } - - // Append numeric suffix - for i in 2..=999 { - let candidate = output_dir.join(format!("{}-{}", run_id, i)); - if !candidate.exists() { - std::fs::create_dir_all(&candidate)?; - return Ok(candidate); - } - } - - // Fallback: use the base (it exists, will overwrite) - Ok(base) -} - -/// Write per-trial artifacts (stdout.txt, stderr.txt, trace.jsonl, result.json) under: -/// `{run_dir}/{case_id}/trial-{trial_id}/` -pub fn write_trial_artifacts( - run_dir: &Path, - case_id: &str, - trial_id: u32, - stdout: &[u8], - stderr: &[u8], - trace_jsonl: &str, - result: &TrialResult, -) -> Result { - let trial_dir = run_dir.join(case_id).join(format!("trial-{}", trial_id)); - std::fs::create_dir_all(&trial_dir)?; - - std::fs::write(trial_dir.join("stdout.txt"), stdout)?; - std::fs::write(trial_dir.join("stderr.txt"), stderr)?; - std::fs::write(trial_dir.join("trace.jsonl"), trace_jsonl)?; - - let result_json = serde_json::to_string_pretty(result)?; - std::fs::write(trial_dir.join("result.json"), result_json)?; - - Ok(trial_dir) -} - -/// Write `{run_dir}/{case_id}/aggregated.json` -pub fn write_case_trials_summary( - run_dir: &Path, - case_id: &str, - trials_result: &CaseTrialsResult, -) -> Result<(), ArtifactsError> { - let case_dir = run_dir.join(case_id); - std::fs::create_dir_all(&case_dir)?; - let aggregated_json = serde_json::to_string_pretty(trials_result)?; - std::fs::write(case_dir.join("aggregated.json"), aggregated_json)?; - Ok(()) -} - -fn case_result_to_trial(case: &CaseResult, trial_id: u32) -> TrialResult { - TrialResult { - trial_id, - status: case.status.clone(), - command_count: case.command_count, - input_tokens: case.input_tokens, - output_tokens: case.output_tokens, - check_results: case.check_results.clone(), - error_message: case.error_message.clone(), - } -} - -/// Write per-case artifacts for backwards-compatible callers. -/// -/// Artifacts are written as trial 1 under `{run_dir}/{case_id}/trial-1/`, and an -/// aggregated `{run_dir}/{case_id}/aggregated.json` is also created. -pub fn write_case_artifacts( - run_dir: &Path, - case_id: &str, - stdout: &[u8], - stderr: &[u8], - trace_jsonl: &str, - result: &CaseResult, -) -> Result { - let trial = case_result_to_trial(result, 1); - let trial_dir = - write_trial_artifacts(run_dir, case_id, 1, stdout, stderr, trace_jsonl, &trial)?; - - let pass_count = if result.status == CaseStatus::Passed { - 1 - } else { - 0 - }; - let pass_rate = pass_count as f64; - let aggregated = CaseTrialsResult { - id: result.id.clone(), - trials: vec![trial], - aggregated_status: result.status.clone(), - pass_count, - total_trials: 1, - pass_rate, - }; - write_case_trials_summary(run_dir, case_id, &aggregated)?; - - Ok(trial_dir) -} - -/// Write summary.json -pub fn write_summary(run_dir: &Path, summary: &SummaryResult) -> Result<(), ArtifactsError> { - let summary_json = serde_json::to_string_pretty(summary)?; - std::fs::write(run_dir.join("summary.json"), summary_json)?; - Ok(()) -} - -/// Read summary.json from a run directory -pub fn read_summary(run_dir: &Path) -> Result { - let summary_path = run_dir.join("summary.json"); - let content = std::fs::read_to_string(&summary_path)?; - let summary: SummaryResult = serde_json::from_str(&content)?; - Ok(summary) -} - -/// Read case result.json files from a run directory -pub fn read_case_results(run_dir: &Path) -> Result, ArtifactsError> { - let mut results = Vec::new(); - - let entries = std::fs::read_dir(run_dir)?; - for entry in entries.flatten() { - let path = entry.path(); - if path.is_dir() { - let aggregated_path = path.join("aggregated.json"); - if aggregated_path.exists() { - let content = std::fs::read_to_string(&aggregated_path)?; - let aggregated: CaseTrialsResult = serde_json::from_str(&content)?; - let total_input = aggregated - .trials - .iter() - .filter_map(|t| t.input_tokens) - .fold(None::, |acc, v| { - Some(acc.unwrap_or(0).saturating_add(v)) - }); - let total_output = aggregated - .trials - .iter() - .filter_map(|t| t.output_tokens) - .fold(None::, |acc, v| { - Some(acc.unwrap_or(0).saturating_add(v)) - }); - results.push(CaseResult { - id: aggregated.id, - status: aggregated.aggregated_status, - command_count: None, - input_tokens: total_input, - output_tokens: total_output, - check_results: vec![], - error_message: None, - }); - continue; - } - - // Legacy layout fallback: `{case_id}/result.json` - let result_path = path.join("result.json"); - if result_path.exists() { - let content = std::fs::read_to_string(&result_path)?; - let result: CaseResult = serde_json::from_str(&content)?; - results.push(result); - } - } - } - - Ok(results) -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::TempDir; - - #[test] - fn test_allocate_run_dir_creates_new() { - let dir = TempDir::new().unwrap(); - let run_dir = allocate_run_dir(dir.path(), "2026-04-01T14-00-00Z").unwrap(); - assert!(run_dir.exists()); - assert!(run_dir.ends_with("2026-04-01T14-00-00Z")); - } - - #[test] - fn test_allocate_run_dir_suffix_on_conflict() { - let dir = TempDir::new().unwrap(); - let run_dir1 = allocate_run_dir(dir.path(), "2026-04-01T14-00-00Z").unwrap(); - let run_dir2 = allocate_run_dir(dir.path(), "2026-04-01T14-00-00Z").unwrap(); - assert_ne!(run_dir1, run_dir2); - assert!(run_dir2.to_string_lossy().contains("-2")); - } - - #[test] - fn test_read_case_results_sums_trial_tokens() { - let dir = TempDir::new().unwrap(); - let case_dir = dir.path().join("case-1"); - std::fs::create_dir_all(&case_dir).unwrap(); - - let trials_result = CaseTrialsResult { - id: "case-1".to_string(), - trials: vec![ - TrialResult { - trial_id: 1, - status: CaseStatus::Passed, - command_count: Some(1), - input_tokens: Some(100), - output_tokens: Some(50), - check_results: vec![], - error_message: None, - }, - TrialResult { - trial_id: 2, - status: CaseStatus::Passed, - command_count: Some(1), - input_tokens: Some(200), - output_tokens: Some(80), - check_results: vec![], - error_message: None, - }, - ], - aggregated_status: CaseStatus::Passed, - pass_count: 2, - total_trials: 2, - pass_rate: 1.0, - }; - let json = serde_json::to_string_pretty(&trials_result).unwrap(); - std::fs::write(case_dir.join("aggregated.json"), json).unwrap(); - - let results = read_case_results(dir.path()).unwrap(); - assert_eq!(results.len(), 1); - assert_eq!( - results[0].input_tokens, - Some(300), - "must sum input_tokens across trials" - ); - assert_eq!( - results[0].output_tokens, - Some(130), - "must sum output_tokens across trials" - ); - } - - #[test] - fn test_read_case_results_none_tokens_when_all_trials_none() { - let dir = TempDir::new().unwrap(); - let case_dir = dir.path().join("case-null"); - std::fs::create_dir_all(&case_dir).unwrap(); - - let trials_result = CaseTrialsResult { - id: "case-null".to_string(), - trials: vec![TrialResult { - trial_id: 1, - status: CaseStatus::Error, - command_count: None, - input_tokens: None, - output_tokens: None, - check_results: vec![], - error_message: Some("timeout".to_string()), - }], - aggregated_status: CaseStatus::Error, - pass_count: 0, - total_trials: 1, - pass_rate: 0.0, - }; - let json = serde_json::to_string_pretty(&trials_result).unwrap(); - std::fs::write(case_dir.join("aggregated.json"), json).unwrap(); - - let results = read_case_results(dir.path()).unwrap(); - assert_eq!(results.len(), 1); - assert_eq!( - results[0].input_tokens, None, - "must remain None when all trial tokens are None" - ); - assert_eq!( - results[0].output_tokens, None, - "must remain None when all trial tokens are None" - ); - } - - #[test] - fn test_write_and_read_summary() { - let dir = TempDir::new().unwrap(); - let summary = SummaryResult { - suite_pass: true, - suite_pass_rate: Some(1.0), - agent: "codex".to_string(), - model: None, - total_cases: 2, - passed: 2, - failed: 0, - trials_per_case: Some(1), - parallel: None, - pass_threshold: Some(1.0), - run_dir: dir.path().to_path_buf(), - checks_path: None, - skill_project_root: dir.path().to_path_buf(), - cases: vec![], - }; - - write_summary(dir.path(), &summary).unwrap(); - let read = read_summary(dir.path()).unwrap(); - assert_eq!(read.total_cases, 2); - assert!(read.suite_pass); - } -} diff --git a/crates/fastskill-evals/src/checks.rs b/crates/fastskill-evals/src/checks.rs deleted file mode 100644 index 8d59bcf..0000000 --- a/crates/fastskill-evals/src/checks.rs +++ /dev/null @@ -1,346 +0,0 @@ -//! Deterministic check engine for eval artifact scoring - -use serde::{Deserialize, Serialize}; -use std::path::PathBuf; -use thiserror::Error; - -use crate::trace::{TraceEvent, TracePayload}; - -/// A deterministic check definition loaded from checks.toml -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "name")] -pub enum CheckDefinition { - /// Check whether a pattern appears (or doesn't appear) in the trace - #[serde(rename = "trigger_expectation")] - TriggerExpectation { - pattern: String, - /// true = pattern must appear; false = pattern must NOT appear - expected: bool, - #[serde(default = "default_required")] - required: bool, - }, - /// Check whether the trace contains a line with this pattern - #[serde(rename = "command_contains")] - CommandContains { - pattern: String, - #[serde(default = "default_required")] - required: bool, - }, - /// Check whether a file exists in the working directory after execution - #[serde(rename = "file_exists")] - FileExists { - path: PathBuf, - #[serde(default = "default_required")] - required: bool, - }, - /// Check that the number of raw_json trace lines does not exceed a limit - #[serde(rename = "max_command_count")] - MaxCommandCount { - limit: usize, - #[serde(default = "default_required")] - required: bool, - }, -} - -fn default_required() -> bool { - true -} - -impl CheckDefinition { - pub fn name(&self) -> &str { - match self { - CheckDefinition::TriggerExpectation { .. } => "trigger_expectation", - CheckDefinition::CommandContains { .. } => "command_contains", - CheckDefinition::FileExists { .. } => "file_exists", - CheckDefinition::MaxCommandCount { .. } => "max_command_count", - } - } - - pub fn is_required(&self) -> bool { - match self { - CheckDefinition::TriggerExpectation { required, .. } => *required, - CheckDefinition::CommandContains { required, .. } => *required, - CheckDefinition::FileExists { required, .. } => *required, - CheckDefinition::MaxCommandCount { required, .. } => *required, - } - } -} - -/// Result of a single check -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct CheckResult { - pub check_name: String, - pub passed: bool, - pub message: Option, -} - -/// TOML file structure for checks configuration -#[derive(Debug, Deserialize)] -pub struct ChecksToml { - #[serde(rename = "check", default)] - pub checks: Vec, -} - -/// Errors loading checks configuration -#[derive(Debug, Error)] -pub enum ChecksError { - #[error("EVAL_CHECKS_INVALID: Failed to read checks file: {0}")] - Io(#[from] std::io::Error), - #[error("EVAL_CHECKS_INVALID: Failed to parse checks TOML: {0}")] - Parse(#[from] toml::de::Error), -} - -/// Load check definitions from a TOML file -pub fn load_checks(path: &std::path::Path) -> Result, ChecksError> { - let content = std::fs::read_to_string(path)?; - let parsed: ChecksToml = toml::from_str(&content)?; - Ok(parsed.checks) -} - -/// Run all checks against captured stdout content and working directory -pub fn run_checks( - checks: &[CheckDefinition], - stdout_content: &str, - trace_jsonl: &str, - working_dir: &std::path::Path, -) -> Vec { - checks - .iter() - .map(|check| run_single_check(check, stdout_content, trace_jsonl, working_dir)) - .collect() -} - -/// Count trace events with payload type `raw_json`. -pub fn count_raw_json_events(trace_jsonl: &str) -> usize { - trace_jsonl - .lines() - .filter_map(|line| serde_json::from_str::(line).ok()) - .filter(|event| matches!(event.payload, TracePayload::RawJson { .. })) - .count() -} - -fn run_single_check( - check: &CheckDefinition, - stdout_content: &str, - trace_jsonl: &str, - working_dir: &std::path::Path, -) -> CheckResult { - match check { - CheckDefinition::TriggerExpectation { - pattern, expected, .. - } => { - // Literal substring matching in stdout + trace - let combined = format!("{}\n{}", stdout_content, trace_jsonl); - let found = combined.contains(pattern.as_str()); - let passed = found == *expected; - let message = if passed { - None - } else if *expected { - Some(format!("Pattern '{}' not found but was expected", pattern)) - } else { - Some(format!("Pattern '{}' found but was not expected", pattern)) - }; - CheckResult { - check_name: "trigger_expectation".to_string(), - passed, - message, - } - } - CheckDefinition::CommandContains { pattern, .. } => { - let combined = format!("{}\n{}", stdout_content, trace_jsonl); - let passed = combined.contains(pattern.as_str()); - let message = if passed { - None - } else { - Some(format!("Pattern '{}' not found in output", pattern)) - }; - CheckResult { - check_name: "command_contains".to_string(), - passed, - message, - } - } - CheckDefinition::FileExists { path, .. } => { - let full_path = working_dir.join(path); - let passed = full_path.exists(); - let message = if passed { - None - } else { - Some(format!("File '{}' does not exist", path.display())) - }; - CheckResult { - check_name: "file_exists".to_string(), - passed, - message, - } - } - CheckDefinition::MaxCommandCount { limit, .. } => { - // Count raw_json trace lines - let count = count_raw_json_events(trace_jsonl); - let passed = count <= *limit; - let message = if passed { - None - } else { - Some(format!("Command count {} exceeds limit {}", count, limit)) - }; - CheckResult { - check_name: "max_command_count".to_string(), - passed, - message, - } - } - } -} - -/// Aggregate check results: suite passes if all required checks pass -pub fn suite_passes(results: &[CheckResult]) -> bool { - results.iter().all(|r| r.passed) -} - -#[cfg(test)] -mod tests { - use super::*; - use std::path::Path; - use tempfile::TempDir; - - #[test] - fn test_trigger_expectation_passes_when_pattern_found() { - let check = CheckDefinition::TriggerExpectation { - pattern: "fastskill".to_string(), - expected: true, - required: true, - }; - let results = run_checks(&[check], "fastskill triggered", "", Path::new("/tmp")); - assert!(results[0].passed); - } - - #[test] - fn test_trigger_expectation_fails_when_pattern_missing() { - let check = CheckDefinition::TriggerExpectation { - pattern: "fastskill".to_string(), - expected: true, - required: true, - }; - let results = run_checks(&[check], "nothing here", "", Path::new("/tmp")); - assert!(!results[0].passed); - } - - #[test] - fn test_trigger_expectation_negative_passes_when_pattern_absent() { - let check = CheckDefinition::TriggerExpectation { - pattern: "fastskill".to_string(), - expected: false, - required: true, - }; - let results = run_checks(&[check], "no match", "", Path::new("/tmp")); - assert!(results[0].passed); - } - - #[test] - fn test_file_exists_check_passes() { - let dir = TempDir::new().unwrap(); - let file_path = dir.path().join("output.txt"); - std::fs::write(&file_path, "content").unwrap(); - - let check = CheckDefinition::FileExists { - path: PathBuf::from("output.txt"), - required: true, - }; - let results = run_checks(&[check], "", "", dir.path()); - assert!(results[0].passed); - } - - #[test] - fn test_file_exists_check_fails() { - let dir = TempDir::new().unwrap(); - let check = CheckDefinition::FileExists { - path: PathBuf::from("missing.txt"), - required: true, - }; - let results = run_checks(&[check], "", "", dir.path()); - assert!(!results[0].passed); - } - - #[test] - fn test_max_command_count_passes() { - let check = CheckDefinition::MaxCommandCount { - limit: 5, - required: true, - }; - let trace = r#"{"seq":0,"payload":{"type":"raw_json","data":{"cmd":"a"}}} -{"seq":1,"payload":{"type":"raw_json","data":{"cmd":"b"}}} -{"seq":2,"payload":{"type":"raw_line","line":"ok"}}"#; - let results = run_checks(&[check], "", trace, Path::new("/tmp")); - assert!(results[0].passed); - } - - #[test] - fn test_max_command_count_fails() { - let check = CheckDefinition::MaxCommandCount { - limit: 1, - required: true, - }; - let trace = r#"{"seq":0,"payload":{"type":"raw_json","data":{"cmd":"a"}}} -{"seq":1,"payload":{"type":"raw_json","data":{"cmd":"b"}}} -{"seq":2,"payload":{"type":"raw_json","data":{"cmd":"c"}}}"#; - let results = run_checks(&[check], "", trace, Path::new("/tmp")); - assert!(!results[0].passed); - } - - #[test] - fn test_count_raw_json_events_ignores_substring_only() { - let trace = r#"{"seq":0,"payload":{"type":"raw_line","line":"mentions raw_json text"}} -{"seq":1,"payload":{"type":"raw_json","data":{"cmd":"x"}}}"#; - assert_eq!(count_raw_json_events(trace), 1); - } - - #[test] - fn test_suite_passes_all_passed() { - let results = vec![ - CheckResult { - check_name: "a".to_string(), - passed: true, - message: None, - }, - CheckResult { - check_name: "b".to_string(), - passed: true, - message: None, - }, - ]; - assert!(suite_passes(&results)); - } - - #[test] - fn test_suite_passes_any_failed() { - let results = vec![ - CheckResult { - check_name: "a".to_string(), - passed: true, - message: None, - }, - CheckResult { - check_name: "b".to_string(), - passed: false, - message: Some("failed".to_string()), - }, - ]; - assert!(!suite_passes(&results)); - } - - #[test] - fn test_load_checks_file_not_found() { - let path = Path::new("/nonexistent/path/checks.toml"); - let result = load_checks(path); - assert!(matches!(result, Err(ChecksError::Io(_)))); - } - - #[test] - fn test_load_checks_invalid_toml() { - let dir = TempDir::new().unwrap(); - let checks_file = dir.path().join("checks.toml"); - std::fs::write(&checks_file, "this is not valid toml [[[[").unwrap(); - let result = load_checks(&checks_file); - assert!(matches!(result, Err(ChecksError::Parse(_)))); - } -} diff --git a/crates/fastskill-evals/src/config.rs b/crates/fastskill-evals/src/config.rs deleted file mode 100644 index 83abd03..0000000 --- a/crates/fastskill-evals/src/config.rs +++ /dev/null @@ -1,194 +0,0 @@ -//! Eval configuration resolution - -use std::path::{Path, PathBuf}; -use thiserror::Error; - -/// Input for eval configuration resolution. -/// Callers populate this from their own config format (TOML, YAML, etc.) -/// and pass it to [`resolve_from_input`]. -#[derive(Debug, Clone)] -pub struct EvalConfigInput { - /// Path to prompts CSV file (may be relative; resolved against project_root) - pub prompts: PathBuf, - /// Optional path to checks TOML file - pub checks: Option, - /// Timeout in seconds for each case - pub timeout_seconds: u64, - /// Trials per case (>= 1) - pub trials_per_case: u32, - /// Optional maximum parallelism for trials within one case - pub parallel: Option, - /// Pass threshold for trial aggregation (0.0-1.0) - pub pass_threshold: f64, - /// Whether to fail fast if agent is not available - pub fail_on_missing_agent: bool, -} - -/// Resolved eval configuration (all paths absolute, values validated) -#[derive(Debug, Clone)] -pub struct EvalConfig { - /// Absolute path to prompts CSV file - pub prompts_path: PathBuf, - /// Absolute path to checks TOML file (optional) - pub checks_path: Option, - /// Timeout in seconds for each case - pub timeout_seconds: u64, - /// Trials per case (>= 1) - pub trials_per_case: u32, - /// Optional maximum parallelism for trials within one case - pub parallel: Option, - /// Pass threshold for trial aggregation (0.0-1.0) - pub pass_threshold: f64, - /// Whether to fail fast if agent is not available - pub fail_on_missing_agent: bool, - /// Skill project root directory - pub project_root: PathBuf, -} - -/// Errors during eval configuration resolution -#[derive(Debug, Error)] -pub enum EvalConfigError { - #[error("EVAL_CONFIG_MISSING: No [tool.fastskill.eval] section found in skill-project.toml")] - ConfigMissing, - #[error("EVAL_PROMPTS_NOT_FOUND: Prompts CSV not found: {0}")] - PromptsNotFound(PathBuf), - #[error("EVAL_INVALID_TRIALS_CONFIG: trials_per_case must be in range [1, 1000], got {0}")] - InvalidTrialsConfig(u32), - #[error("EVAL_INVALID_THRESHOLD: pass_threshold must be in range [0.0, 1.0], got {0}")] - InvalidPassThreshold(f64), - #[error("Failed to read skill-project.toml: {0}")] - Io(#[from] std::io::Error), - #[error("Failed to parse skill-project.toml: {0}")] - Parse(String), -} - -/// Validate and resolve an [`EvalConfigInput`] against a project root directory. -/// -/// Returns [`EvalConfig`] with all paths made absolute and all values validated. -pub fn resolve_from_input( - input: &EvalConfigInput, - project_root: &Path, -) -> Result { - if input.trials_per_case < 1 || input.trials_per_case > 1000 { - return Err(EvalConfigError::InvalidTrialsConfig(input.trials_per_case)); - } - if !(0.0..=1.0).contains(&input.pass_threshold) { - return Err(EvalConfigError::InvalidPassThreshold(input.pass_threshold)); - } - - let prompts_path = if input.prompts.is_absolute() { - input.prompts.clone() - } else { - project_root.join(&input.prompts) - }; - - if !prompts_path.exists() { - return Err(EvalConfigError::PromptsNotFound(prompts_path)); - } - - let checks_path = input.checks.as_ref().map(|p| { - if p.is_absolute() { - p.clone() - } else { - project_root.join(p) - } - }); - - Ok(EvalConfig { - prompts_path, - checks_path, - timeout_seconds: input.timeout_seconds, - trials_per_case: input.trials_per_case, - parallel: input.parallel, - pass_threshold: input.pass_threshold, - fail_on_missing_agent: input.fail_on_missing_agent, - project_root: project_root.to_path_buf(), - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use tempfile::TempDir; - - fn make_input(prompts: PathBuf) -> EvalConfigInput { - EvalConfigInput { - prompts, - checks: None, - timeout_seconds: 600, - trials_per_case: 1, - parallel: None, - pass_threshold: 1.0, - fail_on_missing_agent: false, - } - } - - #[test] - fn test_resolve_eval_config_prompts_not_found() { - let dir = TempDir::new().unwrap(); - let input = make_input(PathBuf::from("evals/prompts.csv")); - let result = resolve_from_input(&input, dir.path()); - assert!(matches!(result, Err(EvalConfigError::PromptsNotFound(_)))); - } - - #[test] - fn test_resolve_eval_config_rejects_invalid_trials_per_case() { - let dir = TempDir::new().unwrap(); - let evals_dir = dir.path().join("evals"); - std::fs::create_dir_all(&evals_dir).unwrap(); - std::fs::write( - evals_dir.join("prompts.csv"), - "id,prompt,should_trigger\ntest-1,hello,true\n", - ) - .unwrap(); - - let mut input = make_input(PathBuf::from("evals/prompts.csv")); - input.trials_per_case = 0; - let result = resolve_from_input(&input, dir.path()); - assert!(matches!( - result, - Err(EvalConfigError::InvalidTrialsConfig(0)) - )); - } - - #[test] - fn test_resolve_eval_config_rejects_invalid_pass_threshold() { - let dir = TempDir::new().unwrap(); - let evals_dir = dir.path().join("evals"); - std::fs::create_dir_all(&evals_dir).unwrap(); - std::fs::write( - evals_dir.join("prompts.csv"), - "id,prompt,should_trigger\ntest-1,hello,true\n", - ) - .unwrap(); - - let mut input = make_input(PathBuf::from("evals/prompts.csv")); - input.pass_threshold = 1.5; - let result = resolve_from_input(&input, dir.path()); - assert!(matches!( - result, - Err(EvalConfigError::InvalidPassThreshold(_)) - )); - } - - #[test] - fn test_resolve_eval_config_success() { - let dir = TempDir::new().unwrap(); - let evals_dir = dir.path().join("evals"); - std::fs::create_dir_all(&evals_dir).unwrap(); - std::fs::write( - evals_dir.join("prompts.csv"), - "id,prompt,should_trigger\ntest-1,hello,true\n", - ) - .unwrap(); - - let input = make_input(PathBuf::from("evals/prompts.csv")); - let result = resolve_from_input(&input, dir.path()); - assert!(result.is_ok()); - let config = result.unwrap(); - assert_eq!(config.timeout_seconds, 600); - assert_eq!(config.trials_per_case, 1); - assert_eq!(config.pass_threshold, 1.0); - assert!(!config.fail_on_missing_agent); - } -} diff --git a/crates/fastskill-evals/src/config_adapter.rs b/crates/fastskill-evals/src/config_adapter.rs index e10007e..29b7d6c 100644 --- a/crates/fastskill-evals/src/config_adapter.rs +++ b/crates/fastskill-evals/src/config_adapter.rs @@ -1,6 +1,6 @@ //! Adapter that reads fastskill project files and delegates to fastskill_evals config resolution -use crate::config::{resolve_from_input, EvalConfig, EvalConfigError, EvalConfigInput}; +use aikit_evals::{resolve_from_input, EvalConfig, EvalConfigError, EvalConfigInput}; use fastskill_core::core::manifest::SkillProjectToml; use std::path::Path; diff --git a/crates/fastskill-evals/src/lib.rs b/crates/fastskill-evals/src/lib.rs index 27a50c1..8f6322a 100644 --- a/crates/fastskill-evals/src/lib.rs +++ b/crates/fastskill-evals/src/lib.rs @@ -1,44 +1,30 @@ -//! Eval runner and trace infrastructure for FastSkill. -//! -//! `fastskill-evals` owns all eval runner and trace conversion code that depends on -//! `aikit-sdk`. It is intentionally separate from `fastskill-core` so that consumers -//! needing only skill resolution do not pay the compile-time cost of the eval runner. +//! Thin adapter: re-exports aikit-evals eval infrastructure and adds +//! fastskill-specific config resolution from skill-project.toml. //! //! # Crate boundaries //! - `fastskill-evals` MAY depend on `fastskill-core` (for `SkillProjectToml`). //! - `fastskill-core` MUST NOT depend on `fastskill-evals`. //! - `fastskill-agent-runtime` MUST NOT depend on `fastskill-evals`. -//! -//! # Re-exported items -//! All public types are available at the crate root: -//! [`EvalCase`], [`EvalSuite`], [`EvalRunner`], [`AikitEvalRunner`], -//! [`CaseRunOptions`], [`CaseRunOutput`], [`CaseResult`], [`CaseStatus`], -//! [`TrialResult`], [`CaseTrialsResult`], [`SummaryResult`], [`CheckDefinition`], -//! [`CheckResult`], [`EvalConfig`], [`EvalConfigInput`], [`TraceEvent`], -//! [`TracePayload`], [`RunnerError`], [`SuiteError`], [`EvalConfigError`], -//! [`ChecksError`], [`ArtifactsError`], [`resolve_eval_config`]. -pub mod artifacts; -pub mod checks; -pub mod config; pub mod config_adapter; -pub mod runner; -pub mod suite; -pub mod trace; -pub use artifacts::{ - allocate_run_dir, read_case_results, read_summary, write_case_artifacts, - write_case_trials_summary, write_summary, write_trial_artifacts, ArtifactsError, CaseResult, - CaseStatus, CaseSummary, CaseTrialsResult, RunArtifacts, SummaryResult, TrialResult, -}; -pub use checks::{ - count_raw_json_events, load_checks, run_checks, suite_passes, CheckDefinition, CheckResult, - ChecksError, ChecksToml, +// Re-export submodules so that path-based callers (fastskill-cli) continue to work unchanged. +pub use aikit_evals::artifacts; +pub use aikit_evals::checks; +pub use aikit_evals::config; +pub use aikit_evals::runner; +pub use aikit_evals::suite; +pub use aikit_evals::trace; + +// Re-export all public items at the crate root for convenience. +pub use aikit_evals::{ + agent_events_to_trace, allocate_run_dir, count_raw_json_events, load_checks, load_suite, + read_case_results, read_summary, resolve_from_input, run_checks, run_eval_case, + stdout_to_trace, suite_passes, trace_to_jsonl, write_case_artifacts, write_case_trials_summary, + write_summary, write_trial_artifacts, AikitEvalRunner, ArtifactsError, CaseResult, + CaseRunOptions, CaseRunOutput, CaseStatus, CaseSummary, CaseTrialsResult, CheckDefinition, + CheckResult, ChecksError, ChecksToml, EvalCase, EvalConfig, EvalConfigError, EvalConfigInput, + EvalRunner, EvalSuite, RunArtifacts, RunnerError, SuiteError, SummaryResult, TraceEvent, + TracePayload, TrialResult, }; -pub use config::{resolve_from_input, EvalConfig, EvalConfigError, EvalConfigInput}; pub use config_adapter::resolve_eval_config; -pub use runner::{ - run_eval_case, AikitEvalRunner, CaseRunOptions, CaseRunOutput, EvalRunner, RunnerError, -}; -pub use suite::{load_suite, EvalCase, EvalSuite, SuiteError}; -pub use trace::{agent_events_to_trace, stdout_to_trace, trace_to_jsonl, TraceEvent, TracePayload}; diff --git a/crates/fastskill-evals/src/runner.rs b/crates/fastskill-evals/src/runner.rs deleted file mode 100644 index 300ba30..0000000 --- a/crates/fastskill-evals/src/runner.rs +++ /dev/null @@ -1,515 +0,0 @@ -//! Eval runner implementation using aikit-sdk - -use crate::artifacts::{CaseResult, CaseStatus, CaseTrialsResult, TrialResult}; -use crate::checks::{count_raw_json_events, run_checks, CheckDefinition}; -use crate::suite::EvalCase; -use crate::trace::{agent_events_to_trace, trace_to_jsonl, TraceEvent, TracePayload}; -use aikit_sdk::{run_agent_events, AgentEvent, RunOptions}; -use async_trait::async_trait; -use std::path::PathBuf; -use std::sync::Arc; -use std::time::Duration; -use thiserror::Error; -use tokio::sync::Semaphore; -use tokio::task::JoinSet; - -/// Options for running a single eval case -#[derive(Debug, Clone)] -pub struct CaseRunOptions { - /// Agent key (e.g. "codex", "claude") - pub agent_key: String, - /// Optional model override - pub model: Option, - /// Skill project root (used as working directory when no workspace_subdir) - pub project_root: PathBuf, - /// Timeout in seconds - pub timeout_seconds: u64, - /// Per-case trial aggregation pass threshold (0.0-1.0) - pub pass_threshold: f64, -} - -/// Raw output from running a case -#[derive(Debug)] -pub struct CaseRunOutput { - pub stdout: Vec, - pub stderr: Vec, - pub exit_code: Option, - pub timed_out: bool, -} - -/// Errors during case execution -#[derive(Debug, Error)] -pub enum RunnerError { - #[error("EVAL_AGENT_UNAVAILABLE: Agent '{0}' is not available")] - AgentUnavailable(String), - #[error("EVAL_CASE_TIMEOUT: Case timed out after {0}s")] - Timeout(u64), - #[error("Execution failed: {0}")] - ExecutionFailed(String), -} - -/// Abstraction over eval case execution (default: aikit-backed). -#[async_trait] -pub trait EvalRunner: Send + Sync { - /// Run one case, produce stdout/stderr capture, scored result, and canonical trace JSONL. - async fn run_case( - &self, - case: &EvalCase, - opts: &CaseRunOptions, - checks: &[CheckDefinition], - ) -> (CaseRunOutput, CaseResult, String); - - /// Run multiple trials for one case, returning the aggregated result. - async fn run_case_trials( - &self, - case: &EvalCase, - opts: &CaseRunOptions, - checks: &[CheckDefinition], - trial_count: u32, - max_parallelism: Option, - ) -> CaseTrialsResult; -} - -/// Default runner: `aikit_sdk::run_agent_events` inside `spawn_blocking` with SDK timeout/cwd. -#[derive(Debug, Clone, Copy, Default)] -pub struct AikitEvalRunner; - -/// Result of agent execution within spawn_blocking -struct AgentExecutionResult { - result: Result, - events: Vec, -} - -#[async_trait] -impl EvalRunner for AikitEvalRunner { - async fn run_case( - &self, - case: &EvalCase, - opts: &CaseRunOptions, - checks: &[CheckDefinition], - ) -> (CaseRunOutput, CaseResult, String) { - self.run_case_inner(case, opts, checks).await - } - - async fn run_case_trials( - &self, - case: &EvalCase, - opts: &CaseRunOptions, - checks: &[CheckDefinition], - trial_count: u32, - max_parallelism: Option, - ) -> CaseTrialsResult { - let max_parallel = max_parallelism - .unwrap_or_else(|| num_cpus::get().max(1) as u32) - .max(1) as usize; - let semaphore = Arc::new(Semaphore::new(max_parallel)); - let mut join_set: JoinSet = JoinSet::new(); - - for trial_id in 1..=trial_count { - let permit = Arc::clone(&semaphore); - let case_clone = case.clone(); - let opts_clone = opts.clone(); - let checks_vec = checks.to_vec(); - let runner = *self; - - join_set.spawn(async move { - let Ok(_permit) = permit.acquire().await else { - return TrialResult { - trial_id, - status: CaseStatus::Error, - command_count: None, - input_tokens: None, - output_tokens: None, - check_results: vec![], - error_message: Some( - "EVAL_PARALLEL_EXHAUSTION: semaphore closed".to_string(), - ), - }; - }; - let (_output, case_result, _trace) = runner - .run_case_inner(&case_clone, &opts_clone, &checks_vec) - .await; - TrialResult { - trial_id, - status: case_result.status, - command_count: case_result.command_count, - input_tokens: case_result.input_tokens, - output_tokens: case_result.output_tokens, - check_results: case_result.check_results, - error_message: case_result.error_message, - } - }); - } - - let mut trials = Vec::with_capacity(trial_count as usize); - while let Some(res) = join_set.join_next().await { - match res { - Ok(trial) => trials.push(trial), - Err(e) => { - // Join errors are treated as failed trials. - let next_id = (trials.len() as u32) + 1; - trials.push(TrialResult { - trial_id: next_id, - status: CaseStatus::Error, - command_count: None, - input_tokens: None, - output_tokens: None, - check_results: vec![], - error_message: Some(format!("EVAL_PARALLEL_EXHAUSTION: {}", e)), - }); - } - } - } - - trials.sort_by_key(|t| t.trial_id); - let pass_count = trials - .iter() - .filter(|t| t.status == CaseStatus::Passed) - .count() as u32; - let total_trials = trial_count.max(1); - let pass_rate = pass_count as f64 / total_trials as f64; - let aggregated_status = if pass_rate >= opts.pass_threshold { - CaseStatus::Passed - } else { - CaseStatus::Failed - }; - - CaseTrialsResult { - id: case.id.clone(), - trials, - aggregated_status, - pass_count, - total_trials, - pass_rate, - } - } -} - -impl AikitEvalRunner { - async fn run_case_inner( - &self, - case: &EvalCase, - opts: &CaseRunOptions, - checks: &[CheckDefinition], - ) -> (CaseRunOutput, CaseResult, String) { - let agent_key = opts.agent_key.clone(); - let model = opts.model.clone(); - let prompt = case.prompt.clone(); - let timeout_secs = opts.timeout_seconds; - - let working_dir = match &case.workspace_subdir { - Some(subdir) => opts.project_root.join(subdir), - None => opts.project_root.clone(), - }; - - let mut run_opts = RunOptions::new() - .with_yolo(true) - .with_stream(true) - .with_timeout(Duration::from_secs(timeout_secs)) - .with_current_dir(working_dir.clone()) - .with_emit_token_usage_events(true); - if let Some(model_name) = model { - if !model_name.trim().is_empty() { - run_opts = run_opts.with_model(model_name); - } - } - - let spawn_result = tokio::task::spawn_blocking(move || { - let mut events: Vec = Vec::new(); - let result = run_agent_events(&agent_key, &prompt, run_opts, |ev| { - events.push(ev.clone()); - }); - AgentExecutionResult { result, events } - }); - - let (run_output, trace_events, token_usage) = match spawn_result.await { - Ok(exec_result) => match exec_result.result { - Ok(run_result) => { - let token_usage = run_result.token_usage.clone(); - let exit_code = run_result.exit_code(); - let output = CaseRunOutput { - stdout: run_result.stdout, - stderr: run_result.stderr, - exit_code, - timed_out: false, - }; - let trace = agent_events_to_trace(&exec_result.events); - (output, trace, token_usage) - } - Err(aikit_sdk::RunError::TimedOut { - timeout, stderr, .. - }) => { - let mut trace = agent_events_to_trace(&exec_result.events); - trace.push(TraceEvent { - seq: trace.len(), - payload: TracePayload::Timeout, - }); - let output = CaseRunOutput { - stdout: vec![], - stderr, - exit_code: None, - timed_out: true, - }; - if output.stderr.is_empty() { - let fallback = format!("Case timed out after {}s", timeout.as_secs()); - let output = CaseRunOutput { - stdout: vec![], - stderr: fallback.into_bytes(), - exit_code: None, - timed_out: true, - }; - (output, trace, None) - } else { - (output, trace, None) - } - } - Err(e) => { - let trace = agent_events_to_trace(&exec_result.events); - let output = CaseRunOutput { - stdout: vec![], - stderr: format!("Agent execution failed: {}", e).into_bytes(), - exit_code: None, - timed_out: false, - }; - (output, trace, None) - } - }, - Err(e) => { - let output = CaseRunOutput { - stdout: vec![], - stderr: format!("spawn_blocking failed: {}", e).into_bytes(), - exit_code: None, - timed_out: false, - }; - (output, vec![], None) - } - }; - - let trace_jsonl = trace_to_jsonl(&trace_events); - let stdout_str = String::from_utf8_lossy(&run_output.stdout).to_string(); - let command_count = count_raw_json_events(&trace_jsonl); - let check_results = run_checks(checks, &stdout_str, &trace_jsonl, &working_dir); - let all_passed = check_results.iter().all(|r| r.passed); - - let status = if run_output.timed_out { - CaseStatus::Error - } else if checks.is_empty() { - if run_output.exit_code == Some(0) { - CaseStatus::Passed - } else { - CaseStatus::Failed - } - } else if all_passed { - CaseStatus::Passed - } else { - CaseStatus::Failed - }; - - let case_result = CaseResult { - id: case.id.clone(), - status, - command_count: Some(command_count), - input_tokens: token_usage.as_ref().map(|u| u.input_tokens), - output_tokens: token_usage.as_ref().map(|u| u.output_tokens), - check_results, - error_message: if run_output.timed_out { - Some(format!( - "EVAL_CASE_TIMEOUT: Case timed out after {}s", - timeout_secs - )) - } else { - None - }, - }; - - (run_output, case_result, trace_jsonl) - } -} - -/// Run a single eval case using the default [`AikitEvalRunner`]. -/// -/// Sole agent execution path per spec (acceptance criterion 29). -pub async fn run_eval_case( - case: &EvalCase, - opts: &CaseRunOptions, - checks: &[CheckDefinition], -) -> (CaseRunOutput, CaseResult, String) { - AikitEvalRunner.run_case(case, opts, checks).await -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::artifacts::CaseStatus; - - /// Stub runner for trait wiring tests (no aikit). - struct StubEvalRunner; - - #[async_trait] - impl EvalRunner for StubEvalRunner { - async fn run_case( - &self, - case: &EvalCase, - _opts: &CaseRunOptions, - _checks: &[CheckDefinition], - ) -> (CaseRunOutput, CaseResult, String) { - let trace_jsonl = - r#"{"seq":0,"payload":{"type":"raw_line","line":"stub"}}"#.to_string(); - let out = CaseRunOutput { - stdout: b"ok".to_vec(), - stderr: vec![], - exit_code: Some(0), - timed_out: false, - }; - let result = CaseResult { - id: case.id.clone(), - status: CaseStatus::Passed, - command_count: Some(0), - input_tokens: Some(100), - output_tokens: Some(50), - check_results: vec![], - error_message: None, - }; - (out, result, trace_jsonl) - } - - async fn run_case_trials( - &self, - case: &EvalCase, - opts: &CaseRunOptions, - checks: &[CheckDefinition], - trial_count: u32, - _max_parallelism: Option, - ) -> CaseTrialsResult { - let mut trials = Vec::new(); - for trial_id in 1..=trial_count { - let (_out, result, _trace) = self.run_case(case, opts, checks).await; - trials.push(TrialResult { - trial_id, - status: result.status, - command_count: result.command_count, - input_tokens: result.input_tokens, - output_tokens: result.output_tokens, - check_results: result.check_results, - error_message: result.error_message, - }); - } - let pass_count = trials - .iter() - .filter(|t| t.status == CaseStatus::Passed) - .count() as u32; - let total_trials = trial_count.max(1); - let pass_rate = pass_count as f64 / total_trials as f64; - let aggregated_status = if pass_rate >= opts.pass_threshold { - CaseStatus::Passed - } else { - CaseStatus::Failed - }; - CaseTrialsResult { - id: case.id.clone(), - trials, - aggregated_status, - pass_count, - total_trials, - pass_rate, - } - } - } - - #[tokio::test] - async fn test_eval_runner_trait_stub_returns_expected_trace() { - let case = EvalCase { - id: "c1".to_string(), - prompt: "p".to_string(), - should_trigger: true, - tags: vec![], - workspace_subdir: None, - }; - let opts = CaseRunOptions { - agent_key: "agent".to_string(), - model: None, - project_root: PathBuf::from("/tmp"), - timeout_seconds: 1, - pass_threshold: 1.0, - }; - let runner = StubEvalRunner; - let (out, res, trace) = runner.run_case(&case, &opts, &[]).await; - assert_eq!(out.exit_code, Some(0)); - assert_eq!(res.id, "c1"); - assert!(trace.contains("raw_line")); - } - - #[test] - fn test_case_run_options_builder() { - let opts = CaseRunOptions { - agent_key: "codex".to_string(), - model: Some("gpt-4".to_string()), - project_root: PathBuf::from("/tmp"), - timeout_seconds: 300, - pass_threshold: 1.0, - }; - assert_eq!(opts.agent_key, "codex"); - assert_eq!(opts.model, Some("gpt-4".to_string())); - } - - #[test] - fn test_runner_error_display() { - let err = RunnerError::AgentUnavailable("codex".to_string()); - assert!(err.to_string().contains("codex")); - assert!(err.to_string().contains("EVAL_AGENT_UNAVAILABLE")); - } - - #[tokio::test] - async fn test_stub_runner_returns_non_null_token_fields() { - let case = EvalCase { - id: "tok-case".to_string(), - prompt: "p".to_string(), - should_trigger: true, - tags: vec![], - workspace_subdir: None, - }; - let opts = CaseRunOptions { - agent_key: "agent".to_string(), - model: None, - project_root: PathBuf::from("/tmp"), - timeout_seconds: 1, - pass_threshold: 1.0, - }; - let runner = StubEvalRunner; - let (_out, res, _trace) = runner.run_case(&case, &opts, &[]).await; - assert_eq!( - res.input_tokens, - Some(100), - "stub must return non-null input_tokens" - ); - assert_eq!( - res.output_tokens, - Some(50), - "stub must return non-null output_tokens" - ); - } - - #[tokio::test] - async fn test_stub_runner_trials_propagate_token_fields() { - let case = EvalCase { - id: "tok-trial".to_string(), - prompt: "p".to_string(), - should_trigger: true, - tags: vec![], - workspace_subdir: None, - }; - let opts = CaseRunOptions { - agent_key: "agent".to_string(), - model: None, - project_root: PathBuf::from("/tmp"), - timeout_seconds: 1, - pass_threshold: 1.0, - }; - let runner = StubEvalRunner; - let trials_result = runner.run_case_trials(&case, &opts, &[], 2, None).await; - for trial in &trials_result.trials { - assert_eq!(trial.input_tokens, Some(100)); - assert_eq!(trial.output_tokens, Some(50)); - } - } -} diff --git a/crates/fastskill-evals/src/suite.rs b/crates/fastskill-evals/src/suite.rs deleted file mode 100644 index 8d4df58..0000000 --- a/crates/fastskill-evals/src/suite.rs +++ /dev/null @@ -1,277 +0,0 @@ -//! Eval suite loading and CSV parsing - -use std::path::{Path, PathBuf}; -use thiserror::Error; - -/// An individual eval case definition -#[derive(Debug, Clone)] -pub struct EvalCase { - /// Unique case identifier (alphanumeric + hyphens) - pub id: String, - /// Prompt to send to the agent - pub prompt: String, - /// Whether the skill should trigger (documentation-only; checks.toml is authoritative for pass/fail) - pub should_trigger: bool, - /// Tags for filtering - pub tags: Vec, - /// Optional workspace subdirectory (relative to skill project root) - pub workspace_subdir: Option, -} - -/// A collection of eval cases -#[derive(Debug, Default)] -pub struct EvalSuite { - pub cases: Vec, -} - -impl EvalSuite { - pub fn new(cases: Vec) -> Self { - Self { cases } - } - - /// Filter cases by ID - pub fn filter_by_id(&self, id: &str) -> EvalSuite { - EvalSuite { - cases: self.cases.iter().filter(|c| c.id == id).cloned().collect(), - } - } - - /// Filter cases by tag - pub fn filter_by_tag(&self, tag: &str) -> EvalSuite { - EvalSuite { - cases: self - .cases - .iter() - .filter(|c| c.tags.iter().any(|t| t == tag)) - .cloned() - .collect(), - } - } -} - -/// Errors that can occur when loading an eval suite -#[derive(Debug, Error)] -pub enum SuiteError { - #[error("EVAL_PROMPTS_NOT_FOUND: Prompts CSV file not found: {0}")] - PromptsNotFound(PathBuf), - #[error("EVAL_INVALID_CSV: {0}")] - InvalidCsv(String), - #[error("EVAL_INVALID_CSV: IO error: {0}")] - Io(#[from] std::io::Error), -} - -/// Load an eval suite from a CSV file -/// -/// Expected CSV columns: id,prompt,should_trigger,tags,workspace_subdir -pub fn load_suite(prompts_path: &Path) -> Result { - if !prompts_path.exists() { - return Err(SuiteError::PromptsNotFound(prompts_path.to_path_buf())); - } - - let content = std::fs::read_to_string(prompts_path)?; - parse_prompts_csv(&content) -} - -/// Parse prompts CSV content -fn parse_prompts_csv(content: &str) -> Result { - let mut lines = content.lines(); - - // Parse header - let header = lines - .next() - .ok_or_else(|| SuiteError::InvalidCsv("CSV is empty".to_string()))?; - let headers: Vec = parse_csv_line(header); - - // Find column indices - let id_idx = find_col(&headers, "id")?; - let prompt_idx = find_col(&headers, "prompt")?; - let should_trigger_idx = find_col(&headers, "should_trigger")?; - let tags_idx = headers.iter().position(|h| h.trim() == "tags"); - let workspace_subdir_idx = headers.iter().position(|h| h.trim() == "workspace_subdir"); - - let mut cases = Vec::new(); - - for (line_num, line) in lines.enumerate() { - let line = line.trim(); - if line.is_empty() { - continue; - } - - let cols = parse_csv_line(line); - - let id = cols - .get(id_idx) - .map(|s| s.trim().to_string()) - .filter(|s| !s.is_empty()) - .ok_or_else(|| { - SuiteError::InvalidCsv(format!("Missing id at line {}", line_num + 2)) - })?; - - let prompt = cols - .get(prompt_idx) - .map(|s| s.trim_matches('"').trim().to_string()) - .ok_or_else(|| { - SuiteError::InvalidCsv(format!("Missing prompt at line {}", line_num + 2)) - })?; - - let should_trigger_str = cols - .get(should_trigger_idx) - .map(|s| s.trim().to_lowercase()) - .unwrap_or_else(|| "false".to_string()); - let should_trigger = should_trigger_str == "true" || should_trigger_str == "1"; - - let tags = if let Some(idx) = tags_idx { - cols.get(idx) - .map(|s| { - s.trim() - .trim_matches('"') - .split(',') - .map(|t| t.trim().to_string()) - .filter(|t| !t.is_empty()) - .collect() - }) - .unwrap_or_default() - } else { - vec![] - }; - - let workspace_subdir = if let Some(idx) = workspace_subdir_idx { - cols.get(idx).and_then(|s| { - let s = s.trim(); - if s.is_empty() { - None - } else { - Some(PathBuf::from(s)) - } - }) - } else { - None - }; - - cases.push(EvalCase { - id, - prompt, - should_trigger, - tags, - workspace_subdir, - }); - } - - Ok(EvalSuite::new(cases)) -} - -fn find_col(headers: &[String], name: &str) -> Result { - headers - .iter() - .position(|h| h.trim() == name) - .ok_or_else(|| SuiteError::InvalidCsv(format!("Missing required column: {}", name))) -} - -/// Simple CSV line parser that handles quoted fields -fn parse_csv_line(line: &str) -> Vec { - let mut fields = Vec::new(); - let mut current = String::new(); - let mut in_quotes = false; - let mut chars = line.chars().peekable(); - - while let Some(ch) = chars.next() { - match ch { - '"' => { - if in_quotes { - // Check for escaped quote "" - if chars.peek() == Some(&'"') { - chars.next(); - current.push('"'); - } else { - in_quotes = false; - } - } else { - in_quotes = true; - } - } - ',' if !in_quotes => { - fields.push(current.clone()); - current.clear(); - } - _ => { - current.push(ch); - } - } - } - fields.push(current); - fields -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_parse_prompts_csv_basic() { - let csv = "id,prompt,should_trigger,tags,workspace_subdir\n\ - test-1,\"Do something\",true,\"basic\",\n\ - test-2,\"Do nothing\",false,\"\",\n"; - let suite = parse_prompts_csv(csv).unwrap(); - assert_eq!(suite.cases.len(), 2); - assert_eq!(suite.cases[0].id, "test-1"); - assert!(suite.cases[0].should_trigger); - assert_eq!(suite.cases[1].id, "test-2"); - assert!(!suite.cases[1].should_trigger); - } - - #[test] - fn test_parse_prompts_csv_missing_required_col() { - let csv = "id,prompt\ntest-1,hello\n"; - let result = parse_prompts_csv(csv); - assert!(result.is_err()); - assert!(result.unwrap_err().to_string().contains("should_trigger")); - } - - #[test] - fn test_filter_by_id() { - let cases = vec![ - EvalCase { - id: "a".to_string(), - prompt: "p1".to_string(), - should_trigger: true, - tags: vec![], - workspace_subdir: None, - }, - EvalCase { - id: "b".to_string(), - prompt: "p2".to_string(), - should_trigger: false, - tags: vec![], - workspace_subdir: None, - }, - ]; - let suite = EvalSuite::new(cases); - let filtered = suite.filter_by_id("a"); - assert_eq!(filtered.cases.len(), 1); - assert_eq!(filtered.cases[0].id, "a"); - } - - #[test] - fn test_filter_by_tag() { - let cases = vec![ - EvalCase { - id: "a".to_string(), - prompt: "p1".to_string(), - should_trigger: true, - tags: vec!["foo".to_string(), "bar".to_string()], - workspace_subdir: None, - }, - EvalCase { - id: "b".to_string(), - prompt: "p2".to_string(), - should_trigger: false, - tags: vec!["baz".to_string()], - workspace_subdir: None, - }, - ]; - let suite = EvalSuite::new(cases); - let filtered = suite.filter_by_tag("foo"); - assert_eq!(filtered.cases.len(), 1); - assert_eq!(filtered.cases[0].id, "a"); - } -} diff --git a/crates/fastskill-evals/src/trace.rs b/crates/fastskill-evals/src/trace.rs deleted file mode 100644 index 88aebd5..0000000 --- a/crates/fastskill-evals/src/trace.rs +++ /dev/null @@ -1,209 +0,0 @@ -//! Trace event types for eval case execution - -use aikit_sdk::{AgentEvent, AgentEventPayload}; -use serde::{Deserialize, Serialize}; - -/// A single line in a trace.jsonl file -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct TraceEvent { - /// Sequence number (0-based) - pub seq: usize, - /// Event payload - pub payload: TracePayload, -} - -/// Payload of a trace event -#[derive(Debug, Clone, Serialize, Deserialize)] -#[serde(tag = "type", rename_all = "snake_case")] -pub enum TracePayload { - /// A raw JSON line from the agent (tool commands, structured output) - RawJson { data: serde_json::Value }, - /// A raw text line from stdout - RawLine { line: String }, - /// A raw bytes chunk (base64-encoded) - RawBytes { b64: String }, - /// Execution error - Error { message: String }, - /// Case timed out - Timeout, - /// Token usage event emitted by the SDK during agent execution - TokenUsageLine { - usage: serde_json::Value, - source: String, - raw_agent_line_seq: u64, - }, -} - -/// Convert aikit-sdk AgentEvent to internal TraceEvent -pub fn agent_events_to_trace(events: &[AgentEvent]) -> Vec { - events - .iter() - .map(|ev| { - let payload = match &ev.payload { - AgentEventPayload::JsonLine(value) => TracePayload::RawJson { - data: value.clone(), - }, - AgentEventPayload::RawLine(line) => TracePayload::RawLine { line: line.clone() }, - AgentEventPayload::RawBytes(bytes) => { - use base64::{engine::general_purpose::STANDARD, Engine as _}; - TracePayload::RawBytes { - b64: STANDARD.encode(bytes), - } - } - AgentEventPayload::TokenUsageLine { - usage, - source, - raw_agent_line_seq, - } => TracePayload::TokenUsageLine { - usage: serde_json::to_value(usage).unwrap_or(serde_json::Value::Null), - source: serde_json::to_value(source) - .ok() - .and_then(|v| v.as_str().map(|s| s.to_lowercase())) - .unwrap_or_else(|| format!("{:?}", source).to_lowercase()), - raw_agent_line_seq: *raw_agent_line_seq, - }, - _ => TracePayload::RawJson { - data: serde_json::json!({ - "type": "unknown_agent_event_payload", - "raw": format!("{:?}", ev.payload) - }), - }, - }; - TraceEvent { - seq: ev.seq as usize, - payload, - } - }) - .collect() -} - -/// Convert raw stdout lines to trace events -pub fn stdout_to_trace(stdout: &[u8]) -> Vec { - let text = String::from_utf8_lossy(stdout); - let mut events = Vec::new(); - - for (seq, line) in text.lines().enumerate() { - let trimmed = line.trim(); - if trimmed.is_empty() { - continue; - } - - // Try to parse as JSON first - let payload = if let Ok(value) = serde_json::from_str::(trimmed) { - TracePayload::RawJson { data: value } - } else { - TracePayload::RawLine { - line: line.to_string(), - } - }; - - events.push(TraceEvent { seq, payload }); - } - - events -} - -/// Serialize trace events to JSONL format -pub fn trace_to_jsonl(events: &[TraceEvent]) -> String { - events - .iter() - .filter_map(|e| serde_json::to_string(e).ok()) - .collect::>() - .join("\n") -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_stdout_to_trace_text_lines() { - let stdout = b"hello world\nfoo bar\n"; - let events = stdout_to_trace(stdout); - assert_eq!(events.len(), 2); - assert_eq!(events[0].seq, 0); - assert!( - matches!(&events[0].payload, TracePayload::RawLine { line } if line == "hello world") - ); - } - - #[test] - fn test_stdout_to_trace_json_lines() { - let stdout = b"{\"key\": \"value\"}\nplain line\n"; - let events = stdout_to_trace(stdout); - assert_eq!(events.len(), 2); - assert!(matches!(&events[0].payload, TracePayload::RawJson { .. })); - assert!(matches!(&events[1].payload, TracePayload::RawLine { .. })); - } - - #[test] - fn test_trace_to_jsonl() { - let events = vec![TraceEvent { - seq: 0, - payload: TracePayload::RawLine { - line: "test".to_string(), - }, - }]; - let jsonl = trace_to_jsonl(&events); - assert!(jsonl.contains("\"seq\":0")); - assert!(jsonl.contains("raw_line")); - } - - #[test] - fn test_token_usage_line_serializes_as_distinct_type() { - let event = TraceEvent { - seq: 7, - payload: TracePayload::TokenUsageLine { - usage: serde_json::json!({"input_tokens": 1234, "output_tokens": 567}), - source: "claude".to_string(), - raw_agent_line_seq: 6, - }, - }; - let jsonl = trace_to_jsonl(&[event.clone()]); - assert!( - jsonl.contains("\"type\":\"token_usage_line\""), - "expected token_usage_line type tag, got: {}", - jsonl - ); - assert!( - !jsonl.contains("\"type\":\"raw_json\""), - "token_usage_line must not serialize as raw_json, got: {}", - jsonl - ); - let deserialized: TraceEvent = serde_json::from_str(&jsonl).unwrap(); - assert!( - matches!(deserialized.payload, TracePayload::TokenUsageLine { .. }), - "deserialized payload must be TokenUsageLine" - ); - } - - #[test] - fn test_count_raw_json_excludes_token_usage_line() { - use crate::checks::count_raw_json_events; - let events = vec![ - TraceEvent { - seq: 0, - payload: TracePayload::TokenUsageLine { - usage: serde_json::json!({"input_tokens": 100, "output_tokens": 50}), - source: "claude".to_string(), - raw_agent_line_seq: 0, - }, - }, - TraceEvent { - seq: 1, - payload: TracePayload::TokenUsageLine { - usage: serde_json::json!({"input_tokens": 200, "output_tokens": 100}), - source: "codex".to_string(), - raw_agent_line_seq: 1, - }, - }, - ]; - let jsonl = trace_to_jsonl(&events); - let count = count_raw_json_events(&jsonl); - assert_eq!( - count, 0, - "count_raw_json_events must return 0 for token_usage_line-only traces, got: {}", - count - ); - } -}