diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 2fa26d9..4c955f7 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "name": "lore", "source": "./", "description": "Shared memory for Claude Code, Codex, and DOXA: capped curated facts, evidence-backed beliefs, FTS5 session search, staged review, learned skills, and signed sync.", - "version": "0.62.23" + "version": "0.62.24" } ] } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 034a889..6d824c2 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "lore", "description": "Memory for Claude Code that reasons about you and improves itself: Honcho-pattern belief store with deriver, dreamer and dialectic (/lore:ask), automatic skillification with a closed judge/update/retire loop, on a Hermes-pattern core \u2014 hard-capped curated memory, FTS5 session search, staged review.", - "version": "0.62.23", + "version": "0.62.24", "author": { "name": "Fabian Wilde", "email": "docwilde@proton.me" diff --git a/CHANGELOG.md b/CHANGELOG.md index 69ff27a..9fb94c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## 0.62.24 — 2026-10-19 + +**Diagnostic traces** + +- Add **`trace capture|show`** for linked Claude/Codex actions and successful observations, with scrubbing, source hashes and explicit omission counts; automatic native review capture is opt-in. +- Keep traces local and separate from the reviewer digest. Provider success flags do not verify task outcomes. + +**Skill experiments** + +- Add **`skills evaluate`** for exact pending coding skills: freeze baseline, candidate, fixtures and verifier; require training improvement, validation non-regression and preservation of each replay success. +- Consume final tests once; retain immutable candidates, task traces and reports. Human approval remains required; the included deterministic fixture does not measure real model improvements. + +**Quality checks** + +- Check a **16 KiB** skill cap, excessive growth and operator-supplied answer markers before execution; label changed proposals/baselines stale and damaged evidence unavailable in pending review. +- Scripts require human terminal authority and run without inherited credentials. They are trusted local code, with process deadlines and output limits; no sandbox or model adapter is supplied. + ## 0.62.23 — 2026-10-19 **Background review history** diff --git a/Cargo.lock b/Cargo.lock index 22207ff..484e549 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -612,7 +612,7 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lore-core" -version = "0.62.23" +version = "0.62.24" dependencies = [ "fancy-regex", "hmac", diff --git a/README.md b/README.md index f8043cf..b26aef0 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ and relations between beliefs do not turn them into approved facts. - **Session search.** Local FTS5 index of Claude Code, DOXA, and Codex transcripts. - **File maps.** A short `path — purpose` guide to important project files. - **Learned skills.** Verified recipes are proposed, evaluated, and updated or retired. +- **Local skill experiments.** [Capture diagnostic traces and compare pending coding skills](docs/skill-evaluation.md) against frozen tests before human approval. - **Cross-machine sync.** Signed hub, peer, or offline transfer; off by default. - **Agent integration.** Claude Code and Codex plugins inject startup context; Claude review and explicit Codex writes retain the proposal and approval gates. diff --git a/docs/manual.md b/docs/manual.md index f41e2d3..9de61de 100644 --- a/docs/manual.md +++ b/docs/manual.md @@ -235,6 +235,7 @@ Every value below is optional and lives in `~/.claude/settings.json` → `"env"` | `LORE_MACHINE_HOST` | `hostname` | this host's machine-memory key; override when the hostname is not the name you use | | `LORE_REVIEW_MODEL` | unset | umbrella override for both headless roles | | `LORE_DERIVER_MODEL` | `haiku` | extraction — the easy role | +| `LORE_DIAGNOSTIC_TRACES` | unset | `1`: capture local action/observation traces for admitted native reviews; [evaluation workflow](skill-evaluation.md) | | `LORE_DREAMER_MODEL` | `sonnet` | belief reconciliation and promotions — the judgment-heavy role | | `LORE_DIALECTIC_MODEL` | session default | model for the `/lore:ask` subagent | | `LORE_REVIEW_MIN_MESSAGES` | 3 | skip review below this many user messages | diff --git a/docs/skill-evaluation.md b/docs/skill-evaluation.md new file mode 100644 index 0000000..8028e41 --- /dev/null +++ b/docs/skill-evaluation.md @@ -0,0 +1,136 @@ +# Diagnostic traces and procedural skill evaluation + +This is an opt-in local experiment workflow for coding skills. Evaluation +results are advisory evidence for a person reviewing a pending proposal. +Nothing installs a candidate, edits a factual memory, or calls a model by +default. The ordinary reviewer digest and its token budget stay separate. + +## 1. Capture diagnostic evidence + +```sh +lore trace capture --transcript /absolute/session.jsonl --session-id SESSION --engine claude +lore trace show SHA256 +``` + +Use `--engine codex` for Codex rollout JSONL. Both formats retain textual +user/assistant messages, full tool arguments and successful as well as failed +tool observations, with source line numbers and call IDs. Provider error +flags remain observations; a tool saying “tests passed” does not verify a task. +The supplied session label is descriptive. It grants no review authority. + +Set `lore config set LORE_DIAGNOSTIC_TRACES 1` to capture automatically when +a native reviewer admits a session. Its learned skill proposals include the +trace hash. Dry runs, internal workers and skipped sessions do not capture. +This setting increases local disk use; turn it off with `lore config unset +LORE_DIAGNOSTIC_TRACES`. + +Traces are private, local, content-addressed JSON in +`$LORE_ROOT/diagnostic-traces/`. They are not sent to the reviewer, indexed, +injected or synced. Credentials are scrubbed before any truncation. Each +event allows 32 KiB of scrubbed content; an artifact allows 512 KiB. Clipped +events retain the scrubbed full-content hash and length. Omitted events and +unsupported blocks are counted, and `complete` becomes false. Nontext and +reasoning blocks are not a complete execution trace. Invalid or changed +source files refuse capture, as do fields larger than the scrubber's 1 MiB +input bound or source lines larger than 8 MiB. Existing artifacts are verified before reuse. +Remove unneeded trace files locally to reclaim space; there is no automatic +retention policy yet. + +## 2. Evaluate an exact pending skill + +From a human terminal, against an operator-authored suite: + +```sh +lore skills evaluate PROPOSAL_ID --suite /absolute/corpus/suite.json +lore skills evaluation REPORT_SHA256 +lore skills artifact --kind candidate --sha256 CANDIDATE_OBJECT_SHA256 +lore skills artifact --kind trace --sha256 TRACE_SHA256 +lore skills artifact --kind suite --sha256 SUITE_OBJECT_SHA256 +lore pending show PROPOSAL_ID +``` + +Evaluation requires the native carrier's human-review authority. Model and +detached worker requests cannot execute a suite. The installed baseline and +the candidate's final wrapped `SKILL.md` bytes are frozen before running. +Add proposals use an empty baseline; retire proposals are unsupported. +The exact proposal hash binds the results; a changed proposal or installed +baseline makes attached evidence stale. Corrupt artifacts display unavailable. +Human approval and the existing exact-byte approval check still apply. + +The suite is a directory containing one JSON manifest, two standalone scripts +and task fixture directories. See [the deterministic contract fixture](../examples/skill-evaluation/suite.json). +Its scores test this mechanism; they do not measure a model or prove a learned +skill improves real coding. Copy it to a dedicated corpus directory before +customizing it. The manifest specifies: + +| Field | Meaning | +| --- | --- | +| `schema`, `family` | `lore.skill-suite.v1`, `coding` | +| `model`, `runtime` | Frozen operator-supplied identities; not independently attested | +| `interpreter` | Absolute executable path; resolved and hashed before every invocation | +| `runner`, `verifier` | Relative paths to standalone scripts in the suite | +| `timeout_seconds` | Per-process deadline, 1–120 seconds | +| `cases` | Unique IDs, split and relative fixture-directory path; at most 64 | +| `answer_markers` | Optional known task-answer snippets, 24–4096 bytes after whitespace/case normalization | + +Each split (`train`, `replay`, `validation`, `test`) must contain a case. +Identical fixture contents across splits refuse evaluation. A corpus allows +512 files, 16 MiB total and directory depth 32; symlinks and special files +are refused. Keep only corpus files in that directory. Files are copied +from a frozen in-memory snapshot into a fresh workspace for every arm/case. +The suite snapshot records every filename, size and hash, not retained raw +fixture bytes. Keep the original corpus under version control for replay. + +The runner receives these environment variables: + +- `LORE_EVAL_SKILL`: absolute path to the frozen baseline or candidate text. +- `LORE_EVAL_TASK_ID`, `LORE_EVAL_MODEL`, `LORE_EVAL_RUNTIME`: case and run identity. +- `HOME` and `TMPDIR`: a private per-run directory; `PATH`: `/usr/bin:/bin`. + +Its working directory contains that case's fixture. Caller environment, +credentials and HOME are not inherited. The interpreter executes the runner +script, then a separately materialized verifier script in the same workspace. +Both must exit zero for a task to pass. The verifier must independently test +the task rather than trust the runner's own result text. Each process has a +deadline and 1 MiB per output stream; process groups are killed on completion +or timeout. Reports keep hashes and 1500-character scrubbed log excerpts. + +Scripts are trusted executable code. These directories and process groups +are not a filesystem, network or memory sandbox. Use a pinned external +container/runtime in the runner for experiments requiring that isolation. +The model/runtime strings identify the chosen experiment configuration; +LORE cannot attest a remote model's weights or the interpreter's libraries. +LORE provides no model adapter or API spending policy here. + +Train, replay and validation run first. A candidate must improve training +passes, match or beat the baseline's validation count, and preserve **each** +baseline replay success. These are integer gates; no statistical confidence +or generalization claim follows from a tiny suite. Only a candidate passing +those gates reaches the final test, where every candidate case must pass. + +Final tests are consumed before execution, including on interruption. The +consumption key binds test fixtures and verifier, so changing the candidate, +runner, model or manifest cannot silently reopen them. Later candidates can +receive `validation_passed` with `final_test: already_consumed`; that is not +a final-test pass. Use new untouched tasks for the next final evaluation. + +## 3. Check generalization and growth + +Before execution, the candidate is checked for copied operator-supplied +answer markers, a 16 KiB size cap, and growth exceeding both twice the +baseline and 4096 added bytes. A blocked candidate and its quality result +are still retained for review. Without answer markers, answer copying is +not checked. These exact-string/size checks do not detect paraphrased leaks, +semantic overfitting or unnecessary rules; a person must assess those. + +Immutable candidate, suite-manifest, verifier-trace and report objects live +under `$LORE_ROOT/skill-evaluations/`. A per-proposal pointer attaches the +latest report without changing the proposal bytes. Prior objects remain +available by hash. Reports include exact baseline/candidate hashes, fixture +hashes, executable hash, configured model/runtime, per-split counts, quality +checks and the final-test state. They are local and excluded from sync. +There is no automatic best-snapshot selector or performance-based promotion. + +This workflow covers procedural skill experiments. It does not establish +the truth of beliefs or factual memories, generate gold reference runs, or +infer causal errors from a trajectory's first differing action. diff --git a/examples/skill-evaluation/runner.py b/examples/skill-evaluation/runner.py new file mode 100644 index 0000000..7edeb59 --- /dev/null +++ b/examples/skill-evaluation/runner.py @@ -0,0 +1,9 @@ +"""Deterministic contract fixture, not a model or a skill-effectiveness benchmark.""" +import os +from pathlib import Path + +skill = Path(os.environ["LORE_EVAL_SKILL"]).read_text() +expression = "sorted(set(values))" if "Deduplicate before sorting" in skill else "sorted(values)" +if "Break replay" in skill and os.environ["LORE_EVAL_TASK_ID"] == "replay": + expression = "[]" +Path("solution.py").write_text(f"def solve(values):\n return {expression}\n") diff --git a/examples/skill-evaluation/suite.json b/examples/skill-evaluation/suite.json new file mode 100644 index 0000000..7bc9145 --- /dev/null +++ b/examples/skill-evaluation/suite.json @@ -0,0 +1,17 @@ +{ + "schema":"lore.skill-suite.v1", + "family":"coding", + "model":"deterministic-contract-fixture", + "runtime":"python3-stdlib", + "interpreter":"/usr/bin/python3", + "runner":"runner.py", + "verifier":"verifier.py", + "timeout_seconds":5, + "answer_markers":["memorized task-specific answer must never appear in a reusable skill"], + "cases":[ + {"id":"train","split":"train","fixture":"tasks/train"}, + {"id":"replay","split":"replay","fixture":"tasks/replay"}, + {"id":"validation","split":"validation","fixture":"tasks/validation"}, + {"id":"test","split":"test","fixture":"tasks/test"} + ] +} diff --git a/examples/skill-evaluation/tasks/replay/case.json b/examples/skill-evaluation/tasks/replay/case.json new file mode 100644 index 0000000..8991418 --- /dev/null +++ b/examples/skill-evaluation/tasks/replay/case.json @@ -0,0 +1 @@ +{"values":[1,2,3]} diff --git a/examples/skill-evaluation/tasks/test/case.json b/examples/skill-evaluation/tasks/test/case.json new file mode 100644 index 0000000..b878158 --- /dev/null +++ b/examples/skill-evaluation/tasks/test/case.json @@ -0,0 +1 @@ +{"values":[8,2,1,2,8]} diff --git a/examples/skill-evaluation/tasks/train/case.json b/examples/skill-evaluation/tasks/train/case.json new file mode 100644 index 0000000..4f74ba0 --- /dev/null +++ b/examples/skill-evaluation/tasks/train/case.json @@ -0,0 +1 @@ +{"values":[3,1,3]} diff --git a/examples/skill-evaluation/tasks/validation/case.json b/examples/skill-evaluation/tasks/validation/case.json new file mode 100644 index 0000000..d4cecf1 --- /dev/null +++ b/examples/skill-evaluation/tasks/validation/case.json @@ -0,0 +1 @@ +{"values":[-2,7,-2,0]} diff --git a/examples/skill-evaluation/verifier.py b/examples/skill-evaluation/verifier.py new file mode 100644 index 0000000..413bf61 --- /dev/null +++ b/examples/skill-evaluation/verifier.py @@ -0,0 +1,8 @@ +"""Frozen task check; the runner's own success message does not score the task.""" +import json +from pathlib import Path + +namespace = {} +exec(Path("solution.py").read_text(), namespace) +values = json.loads(Path("case.json").read_text())["values"] +assert namespace["solve"](values) == sorted(set(values)) diff --git a/plugin.json b/plugin.json index db2c606..8f16dbb 100644 --- a/plugin.json +++ b/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "lore", - "version": "0.62.23", + "version": "0.62.24", "description": "Unified user, repo, and session memory for Claude Code, Codex, and DOXA", "author": { "name": "Dr. Fabian Wilde", diff --git a/rust/lore-core/Cargo.toml b/rust/lore-core/Cargo.toml index 010765c..12be6c4 100644 --- a/rust/lore-core/Cargo.toml +++ b/rust/lore-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lore-core" -version = "0.62.23" +version = "0.62.24" edition = "2021" license = "AGPL-3.0-only" description = "Native LORE memory, beliefs, review and session index" diff --git a/rust/lore-core/src/core.rs b/rust/lore-core/src/core.rs index 070f5aa..10c47f6 100644 --- a/rust/lore-core/src/core.rs +++ b/rust/lore-core/src/core.rs @@ -28,6 +28,11 @@ impl Core { "filemap", "codegraph_snapshot_read_v1", "codegraph_snapshot_store_v1", + "diagnostic_trace_capture_v1", + "diagnostic_trace_read_v1", + "skill_evaluate_v1", + "skill_evaluation_read_v1", + "skill_evaluation_artifact_v1", "refresh_interval", "transcript_identity", "consult", @@ -70,6 +75,11 @@ impl Core { let config = &self.config; let auth = &self.authority; let value = match req["op"].as_str().ok_or(Error::InvalidRequest)? { + "diagnostic_trace_capture_v1" => crate::diagnostics::capture(config, req)?, + "diagnostic_trace_read_v1" => crate::diagnostics::show(config, req)?, + "skill_evaluate_v1" => crate::skill_evaluation::evaluate(config, req, auth)?, + "skill_evaluation_read_v1" => crate::skill_evaluation::show(config, req)?, + "skill_evaluation_artifact_v1" => crate::skill_evaluation::artifact(config, req)?, "runtime_config_v1" => crate::config::runtime(config), "store_status_v1" => { let conn = crate::store::read_only(config)?; diff --git a/rust/lore-core/src/diagnostics.rs b/rust/lore-core/src/diagnostics.rs new file mode 100644 index 0000000..a6a9c52 --- /dev/null +++ b/rust/lore-core/src/diagnostics.rs @@ -0,0 +1,200 @@ +//! Local diagnostic evidence, deliberately separate from the lossy review digest. +//! Provider flags describe observations; only a host-run verifier can check a task. +use crate::{config, files, Error, Result}; +use serde_json::{json, Value}; +use std::{fs::File, io::{BufRead, BufReader, Seek}, path::Path}; + +// Leave room for the standalone carrier's pretty JSON within its 1 MiB frame. +pub const ARTIFACT_CAP: usize = 512 * 1024; +const EVENT_CAP: usize = 32 * 1024; +const LINE_CAP: u64 = 8 * 1024 * 1024; + +pub fn valid_hash(s: &str) -> bool { + s.len() == 64 && s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +/// Content-addressed local objects are never replaced by a different byte sequence. +pub(crate) fn put(cfg: &config::Config, area: &str, value: &Value) -> Result { + let bytes = serde_json::to_vec(value).map_err(|_| Error::InvalidRequest)?; + if bytes.len() > ARTIFACT_CAP { return Err(Error::TooLarge); } + let hash = crate::digest(&bytes); + let path = cfg.root.join(area).join(format!("{hash}.json")); + let _lock = files::Locks::acquire(&cfg.root, std::slice::from_ref(&path), cfg.timeout)?; + if path.try_exists()? { + if files::read_regular(&path, ARTIFACT_CAP)? != bytes { return Err(Error::Changed); } + } else { + files::atomic_write(&path, &bytes)?; + } + Ok(hash) +} + +pub(crate) fn get(cfg: &config::Config, area: &str, hash: &str) -> Result { + if !valid_hash(hash) { return Err(Error::InvalidRequest); } + let bytes = files::read_regular(&cfg.root.join(area).join(format!("{hash}.json")), ARTIFACT_CAP)?; + if crate::digest(&bytes) != hash { return Err(Error::Changed); } + serde_json::from_slice(&bytes).map_err(|_| Error::InvalidRequest) +} + +struct Trace { + events: Vec, bytes: usize, clipped: usize, dropped: usize, unsupported: usize, +} +impl Trace { + fn event(&mut self, line: usize, kind: &str, id: &Value, name: &Value, + content: &Value, error: &Value) -> Result<()> { + // Scrub the entire field before clipping: credentials spanning the boundary + // must not leave a usable prefix. JSON preserves multiline observations. + let text = content.as_str().map(str::to_owned).unwrap_or_else(|| content.to_string()); + let text = crate::scrub::scrub(&text)?; + let clipped = text.len() > EVENT_CAP; + let end = if clipped { + let mut end = EVENT_CAP; + while !text.is_char_boundary(end) { end -= 1; } + end + } else { text.len() }; + let event = json!({"sequence":self.events.len(),"source_line":line,"kind":kind, + "call_id":crate::scrub::scrub(id.as_str().unwrap_or(""))?, + "tool":crate::scrub::scrub(name.as_str().unwrap_or(""))?, + "content":&text[..end],"content_sha256":crate::digest(text.as_bytes()), + "content_bytes":text.len(),"truncated":clipped, + "provider_is_error":error.as_bool(),"verification":"unverified"}); + let n = serde_json::to_vec(&event).map_err(|_| Error::InvalidRequest)?.len(); + if self.bytes + n > ARTIFACT_CAP - 8192 { self.dropped += 1; } + else { self.bytes += n; self.clipped += usize::from(clipped); self.events.push(event); } + Ok(()) + } +} + +pub fn parse(file: &File, codex: bool) -> Result { + let mut copy = file.try_clone()?; + copy.rewind()?; + let mut reader = BufReader::new(copy); + let mut trace = Trace { events:vec![], bytes:0, clipped:0, dropped:0, unsupported:0 }; + let mut line = Vec::new(); + let mut lines = 0; + loop { + use std::io::Read; + line.clear(); + let n = reader.by_ref().take(LINE_CAP + 1).read_until(b'\n', &mut line)?; + if n == 0 { break; } + if n as u64 > LINE_CAP { return Err(Error::TooLarge); } + lines += 1; + let v: Value = serde_json::from_slice(&line).map_err(|_| Error::InvalidRequest)?; + if !v.is_object() { return Err(Error::InvalidRequest); } + if codex { + if v["type"] != "response_item" { continue; } + let p = &v["payload"]; + match p["type"].as_str().unwrap_or("") { + "message" => { + let role = p["role"].as_str().unwrap_or(""); + if !matches!(role, "user" | "assistant") { continue; } + for b in p["content"].as_array().ok_or(Error::InvalidRequest)? { + if matches!(b["type"].as_str(), Some("input_text" | "output_text")) { + trace.event(lines, role, &Value::Null, &Value::Null, &b["text"], &Value::Null)?; + } else { trace.unsupported += 1; } + } + } + "function_call" | "custom_tool_call" | "local_shell_call" | "tool_search_call" | "web_search_call" => { + let content = p.get("arguments").or_else(||p.get("input")).or_else(||p.get("action")).unwrap_or(&Value::Null); + trace.event(lines, "action", p.get("call_id").unwrap_or(&p["id"]), p.get("name").unwrap_or(&p["type"]), content, &Value::Null)?; + } + "function_call_output" | "custom_tool_call_output" | "tool_search_output" => { + trace.event(lines, "observation", &p["call_id"], &Value::Null, + p.get("output").or_else(||p.get("tools")).unwrap_or(&Value::Null), &p["is_error"])?; + } + // Reasoning and encrypted/image payloads are not text evidence. + _ => trace.unsupported += 1, + } + } else { + let role = v["type"].as_str().unwrap_or(""); + if !matches!(role, "user" | "assistant") || v["isMeta"] == true { continue; } + let content = &v["message"]["content"]; + if content.is_string() { + trace.event(lines, role, &Value::Null, &Value::Null, content, &Value::Null)?; + } else { + for b in content.as_array().ok_or(Error::InvalidRequest)? { + match b["type"].as_str().unwrap_or("") { + "text" => trace.event(lines, role, &Value::Null, &Value::Null, &b["text"], &Value::Null)?, + "tool_use" => trace.event(lines, "action", &b["id"], &b["name"], &b["input"], &Value::Null)?, + "tool_result" => trace.event(lines, "observation", &b["tool_use_id"], &Value::Null, &b["content"], &b["is_error"])?, + _ => trace.unsupported += 1, + } + } + } + } + } + Ok(json!({"events":trace.events,"source_lines":lines,"truncated_events":trace.clipped, + "dropped_events":trace.dropped,"unsupported_blocks":trace.unsupported, + "complete":trace.clipped == 0 && trace.dropped == 0 && trace.unsupported == 0})) +} + +pub fn capture(cfg: &config::Config, req: &Value) -> Result { + let path = Path::new(req["transcript"].as_str().ok_or(Error::InvalidRequest)?); + let sid = req["session_id"].as_str().filter(|s|config::valid_id(s)).ok_or(Error::InvalidRequest)?; + let engine = req["engine"].as_str().filter(|s|matches!(*s,"claude"|"codex")).ok_or(Error::InvalidRequest)?; + let (file, proof) = crate::review::snapshot_source(path)?; + let mut trace = parse(&file, engine == "codex")?; + let (_, after) = crate::review::snapshot_source(path)?; + if proof != after { return Err(Error::Changed); } + trace["schema"] = json!("lore.diagnostic-trace.v1"); + trace["session_id"] = json!(sid); + trace["engine"] = json!(engine); + trace["source_sha256"] = proof["sha256"].clone(); + trace["source_bytes"] = proof["size"].clone(); + trace["task_outcome"] = json!("unverified"); + let hash = put(cfg, "diagnostic-traces", &trace)?; + Ok(json!({"sha256":hash,"source_sha256":proof["sha256"],"events":trace["events"].as_array().map(Vec::len), + "complete":trace["complete"],"task_outcome":"unverified"})) +} + +pub fn show(cfg: &config::Config, req: &Value) -> Result { + get(cfg, "diagnostic-traces", req["sha256"].as_str().ok_or(Error::InvalidRequest)?) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + fn parsed(rows: &[Value], codex: bool) -> Value { + let mut f = tempfile::tempfile().unwrap(); + for row in rows { writeln!(f,"{row}").unwrap(); } + parse(&f,codex).unwrap() + } + #[test] + fn successful_observations_are_linked_and_never_task_verification() { + for codex in [false,true] { + let rows = if codex { vec![ + json!({"type":"response_item","payload":{"type":"function_call","name":"exec_command","call_id":"a","arguments":"{\"cmd\":\"cargo test\"}"}}), + json!({"type":"response_item","payload":{"type":"function_call_output","call_id":"a","output":"all tests passed\n42 cases"}}) + ] } else { vec![ + json!({"type":"assistant","message":{"content":[{"type":"tool_use","id":"a","name":"Bash","input":{"command":"cargo test"}}]}}), + json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"a","content":"all tests passed\n42 cases","is_error":false}]}}) + ] }; + let trace = parsed(&rows,codex); + assert_eq!(trace["events"][1]["call_id"],"a"); + assert_eq!(trace["events"][1]["kind"],"observation"); + assert_eq!(trace["events"][1]["content"],"all tests passed\n42 cases"); + assert_eq!(trace["events"][1]["verification"],"unverified"); + assert_eq!(trace["complete"],true); + } + } + #[test] + fn clipping_and_unsupported_content_are_explicit() { + let t = parsed(&[json!({"type":"assistant","message":{"content":[ + {"type":"text","text":"trace output ".repeat(EVENT_CAP/10)}, {"type":"image"}]}})],false); + assert_eq!(t["complete"],false); + assert_eq!(t["truncated_events"],1); + assert_eq!(t["unsupported_blocks"],1); + } + #[test] + fn source_is_scrubbed_before_any_cut_and_tampered_objects_refuse() { + let secret=format!("sk-{}","a".repeat(80)); + let text=format!("{}{}", "ordinary output ".repeat(EVENT_CAP/16),secret); + let t=parsed(&[json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"a","content":text,"is_error":true}]}})],false); + assert!(!t.to_string().contains("sk-")); + assert_eq!(t["events"][0]["provider_is_error"],true); + let dir=tempfile::tempdir().unwrap(); let cfg=config::Config::for_root(dir.path().join("store")); + let hash=put(&cfg,"diagnostic-traces",&t).unwrap(); + files::atomic_write(&cfg.root.join("diagnostic-traces").join(format!("{hash}.json")),b"{}").unwrap(); + assert_eq!(get(&cfg,"diagnostic-traces",&hash),Err(Error::Changed)); + } +} diff --git a/rust/lore-core/src/lib.rs b/rust/lore-core/src/lib.rs index 7fa4fcc..93ccb29 100644 --- a/rust/lore-core/src/lib.rs +++ b/rust/lore-core/src/lib.rs @@ -7,6 +7,7 @@ pub mod codegraph_snapshot; pub mod context; mod core; pub mod dream; +pub mod diagnostics; pub mod filemap; pub mod files; pub mod gate; @@ -18,6 +19,7 @@ pub mod pending; pub mod review; pub mod scrub; pub mod skills; +pub mod skill_evaluation; pub mod store; pub mod sync; pub mod sync_apply; diff --git a/rust/lore-core/src/pending.rs b/rust/lore-core/src/pending.rs index 1daaaa0..26d0c93 100644 --- a/rust/lore-core/src/pending.rs +++ b/rust/lore-core/src/pending.rs @@ -305,6 +305,14 @@ fn content_pairs(value: &Value, depth: usize) -> Result<()> { } fn reviewable(snapshot: &Snapshot) -> Result<()> { let mut checked = snapshot.item.clone(); + if checked["kind"] == "skill" { + if let Some(hash) = checked.get("diagnostic_trace_sha256").filter(|v| !v.is_null()) { + if !hash.as_str().is_some_and(crate::diagnostics::valid_hash) { return Err(Error::InvalidRequest); } + // Public, typed content address only; it grants no task score or + // authority. Preserve the exact raw bytes used for approval. + checked["diagnostic_trace_sha256"] = Value::Null; + } + } if checked["kind"] == "sync" { let op = checked .get_mut("op") @@ -502,7 +510,11 @@ pub fn review(cfg: &Config, req: &Value) -> Result { if req.get("expected").is_some() { check_expected(req, &snap)?; } - Ok(json!({"pid":id,"raw":snap.raw,"sha256":snap.sha256,"inode":snap.inode,"complete":true})) + let mut value = json!({"pid":id,"raw":snap.raw,"sha256":snap.sha256,"inode":snap.inode,"complete":true}); + if snap.item["kind"] == "skill" { + value["skill_evaluation"] = crate::skill_evaluation::summary(cfg, id, &snap.sha256, &snap.item); + } + Ok(value) } fn listings(cfg: &Config) -> Result { let path = cfg.root.join("pending/.listed"); diff --git a/rust/lore-core/src/review.rs b/rust/lore-core/src/review.rs index 7c19af9..6156da6 100644 --- a/rust/lore-core/src/review.rs +++ b/rust/lore-core/src/review.rs @@ -83,6 +83,7 @@ pub struct ReviewJob { cwd: PathBuf, authority: Authority, source: SourceProof, + diagnostic_trace: Option, } impl ReviewJob { pub fn prompt(&self) -> &str { @@ -882,6 +883,11 @@ pub fn build_review_job( if session_id.len() > 128 { return Err(Error::InvalidRequest); } + let diagnostic_trace = if record_usage && cfg.var("LORE_DIAGNOSTIC_TRACES").is_ok_and(|v| v == "1") { + let trace = crate::diagnostics::capture(cfg, &json!({"transcript":path,"session_id":sid,"engine":if codex {"codex"} else {"claude"}}))?; + if trace["source_sha256"] != source.hash { return Err(Error::Changed); } + Some(trace["sha256"].as_str().ok_or(Error::InvalidRequest)?.to_owned()) + } else { None }; let job = ReviewJob { prompt, project: slug, @@ -889,6 +895,7 @@ pub fn build_review_job( cwd: cwd.into(), authority: authority.clone(), source, + diagnostic_trace, }; job.validate_source()?; if record_usage { @@ -921,6 +928,12 @@ pub fn stage_proposals( slug: &str, sid: &str, authority: &Authority, +) -> Result { + stage_proposals_with_trace(cfg, data, slug, sid, authority, None) +} +fn stage_proposals_with_trace( + cfg: &Config, data: &Value, slug: &str, sid: &str, + authority: &Authority, diagnostic_trace: Option<&str>, ) -> Result { require_derived(authority)?; validate_channels(data)?; @@ -1066,7 +1079,7 @@ pub fn stage_proposals( } gate::stage( cfg, - &json!({"kind":"skill","project":slug,"source_project":slug,"session_id":sid,"name":name,"action":action,"description":safe_line(&proposal["description"],300)?,"body":body}), + &json!({"kind":"skill","project":slug,"source_project":slug,"session_id":sid,"name":name,"action":action,"description":safe_line(&proposal["description"],300)?,"body":body,"diagnostic_trace_sha256":diagnostic_trace}), authority, )?; staged += 1; @@ -1332,14 +1345,14 @@ pub fn process_result(cfg: &Config, job: &ReviewJob, output: &str) -> Result Result> { + use std::io::Read; + let mut options = std::fs::OpenOptions::new(); options.read(true); + #[cfg(unix)] { use std::os::unix::fs::OpenOptionsExt; options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); } + let mut file = options.open(path)?; + let meta = file.metadata()?; + if !meta.is_file() || meta.len()>128*1024*1024 { return Err(Error::UnsafePath); } + #[cfg(unix)] { + use std::os::unix::fs::MetadataExt; + if ![0,unsafe {libc::geteuid()}].contains(&meta.uid()) || meta.mode() & 0o022 != 0 { return Err(Error::UnsafePath); } + } + let mut bytes=Vec::new(); file.by_ref().take(128*1024*1024+1).read_to_end(&mut bytes)?; + if bytes.len()>128*1024*1024 { return Err(Error::TooLarge); } + Ok(bytes) +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Suite { + schema: String, + family: String, + model: String, + runtime: String, + interpreter: String, + runner: String, + verifier: String, + timeout_seconds: u64, + cases: Vec, + #[serde(default)] answer_markers: Vec, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Case { id: String, split: String, fixture: String } + +fn relative(s: &str) -> bool { + !s.is_empty() && s.len() <= 240 && Path::new(s).components().all(|c|matches!(c,std::path::Component::Normal(_))) +} +fn collect(root: &Path, path: &Path, out: &mut BTreeMap>, total: &mut usize) -> Result<()> { + if path.components().count().saturating_sub(root.components().count()) > 32 { return Err(Error::TooLarge); } + let mut names = files::directory_names(path, 512)?; + names.sort(); + for name in names { + let p = path.join(name); + let meta = std::fs::symlink_metadata(&p)?; + if meta.file_type().is_symlink() { return Err(Error::UnsafePath); } + if meta.is_dir() { collect(root,&p,out,total)?; } + else if meta.is_file() { + if out.len() >= 512 { return Err(Error::TooLarge); } + let bytes = files::read_regular(&p,TREE_CAP.saturating_sub(*total))?; + *total += bytes.len(); + if *total > TREE_CAP { return Err(Error::TooLarge); } + let name = p.strip_prefix(root).map_err(|_|Error::UnsafePath)?.to_str().ok_or(Error::UnsafePath)?.to_owned(); + out.insert(name,bytes); + } else { return Err(Error::UnsafePath); } + } + Ok(()) +} +fn tree_hash(tree: &BTreeMap>) -> Result { + let hashes: BTreeMap<_,_> = tree.iter().map(|(p,b)|(p,crate::digest(b))).collect(); + Ok(crate::digest(&serde_json::to_vec(&hashes).map_err(|_|Error::InvalidRequest)?)) +} +fn fixture(tree: &BTreeMap>, prefix: &str) -> Result>> { + let prefix = format!("{prefix}/"); + let part: BTreeMap<_,_> = tree.iter().filter_map(|(k,v)|k.strip_prefix(&prefix).map(|k|(k.to_owned(),v.clone()))).collect(); + if part.is_empty() { return Err(Error::InvalidRequest); } + Ok(part) +} +fn materialize(root: &Path, tree: &BTreeMap>) -> Result<()> { + for (name,bytes) in tree { files::atomic_write(&root.join(name),bytes)?; } + Ok(()) +} +fn normalize(s: &str) -> String { s.split_whitespace().collect::>().join(" ").to_lowercase() } + +fn quality(baseline: &str, candidate: &str, markers: &[String]) -> Value { + // The absolute cap also bounds newly created skills (empty baseline). + let oversized = candidate.len() > 16*1024; + let excessive_growth = !baseline.is_empty() && candidate.len() > baseline.len().saturating_mul(2) + && candidate.len().saturating_sub(baseline.len()) > 4096; + let normalized = normalize(candidate); + let copied_answer = markers.iter().any(|m|normalized.contains(&normalize(m))); + json!({"passed":!oversized && !excessive_growth && !copied_answer, + "baseline_bytes":baseline.len(),"candidate_bytes":candidate.len(), + "max_bytes":16*1024,"growth_allowance_bytes":4096,"growth_ratio":2, + "oversized":oversized,"excessive_growth":excessive_growth,"answer_marker_match":copied_answer, + "coverage":"size and operator-supplied exact answer markers; semantic generalization is unverified"}) +} + +/// No shell, inherited credentials or caller HOME. The selected suite is trusted +/// executable code, not an adversarial sandbox. Rehash the interpreter per run. +fn execute(interpreter: &Path, interpreter_hash: &str, script: &Path, workspace: &Path, + skill: &Path, home: &Path, case: &Case, suite: &Suite, log: &Path) -> Result { + if crate::digest(&interpreter_bytes(interpreter)?) != interpreter_hash { return Err(Error::Changed); } + let stdout = log.with_extension("stdout"); + let stderr = log.with_extension("stderr"); + let out = std::fs::OpenOptions::new().write(true).create_new(true).open(&stdout)?; + let err = std::fs::OpenOptions::new().write(true).create_new(true).open(&stderr)?; + let mut command = Command::new(interpreter); + command.arg(script).current_dir(workspace).env_clear() + .env("PATH","/usr/bin:/bin").env("HOME",home).env("TMPDIR",home) + .env("LORE_EVAL_SKILL",skill).env("LORE_EVAL_TASK_ID",&case.id) + .env("LORE_EVAL_MODEL",&suite.model).env("LORE_EVAL_RUNTIME",&suite.runtime) + .stdin(Stdio::null()).stdout(Stdio::from(out)).stderr(Stdio::from(err)); + #[cfg(unix)] { + use std::os::unix::process::CommandExt; + unsafe { command.pre_exec(|| { if libc::setsid() < 0 { return Err(std::io::Error::last_os_error()); } Ok(()) }); } + } + let start = Instant::now(); + let mut child = command.spawn()?; + let mut reason = "exit"; + let status = loop { + if stdout.metadata()?.len() > OUTPUT_CAP || stderr.metadata()?.len() > OUTPUT_CAP { + reason = "output_limit"; break None; + } + if let Some(status) = child.try_wait()? { break Some(status); } + if start.elapsed() >= Duration::from_secs(suite.timeout_seconds) { reason = "timeout"; break None; } + std::thread::sleep(Duration::from_millis(20)); + }; + // Kill descendants even when the direct child exits successfully. + #[cfg(unix)] unsafe { libc::kill(-(child.id() as i32),libc::SIGKILL); } + if status.is_none() { let _ = child.kill(); } + let _ = child.wait(); + let read_log = |p:&Path| -> Result { + use std::io::Read; + let mut b = Vec::new(); + let mut options = std::fs::OpenOptions::new(); options.read(true); + #[cfg(unix)] { use std::os::unix::fs::OpenOptionsExt; options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); } + let file=options.open(p)?; + if !file.metadata()?.is_file() { return Err(Error::UnsafePath); } + file.take(OUTPUT_CAP).read_to_end(&mut b)?; + crate::scrub::scrub(&String::from_utf8_lossy(&b)) + }; + // Store hashes and short scrubbed excerpts; raw logs die with the workspace. + let out = read_log(&stdout)?; let err = read_log(&stderr)?; + Ok(json!({"passed":reason=="exit" && status.is_some_and(|s|s.success()), + "exit_code":status.and_then(|s|s.code()),"termination":reason, + "stdout_sha256":crate::digest(out.as_bytes()),"stderr_sha256":crate::digest(err.as_bytes()), + "stdout_excerpt":out.chars().take(1500).collect::(), + "stderr_excerpt":err.chars().take(1500).collect::()})) +} + +fn run_case(cfg: &Config, base: &Path, suite: &Suite, tree: &BTreeMap>, + interpreter: &Path, interpreter_hash: &str, case: &Case, arm: &str, skill_text: &str) -> Result { + let dir = base.join(format!("{}-{arm}",case.id)); + let workspace = dir.join("workspace"); + materialize(&workspace,&fixture(tree,&case.fixture)?)?; + let skill = dir.join("skill.md"); + files::atomic_write(&skill,skill_text.as_bytes())?; + let home = dir.join("home"); files::private_dir(&home)?; + let runner = dir.join("runner"); files::atomic_write(&runner,&tree[&suite.runner])?; + let run = execute(interpreter,interpreter_hash,&runner,&workspace,&skill,&home,case,suite,&dir.join("run"))?; + // Materialize verifier from frozen bytes after the candidate has executed. + let verifier = dir.join("verifier"); files::atomic_write(&verifier,&tree[&suite.verifier])?; + let verify = execute(interpreter,interpreter_hash,&verifier,&workspace,&skill,&home,case,suite,&dir.join("verify"))?; + let transcript = json!({"schema":"lore.evaluation-trace.v1","case":case.id,"arm":arm, + "skill_sha256":crate::digest(skill_text.as_bytes()),"fixture_sha256":tree_hash(&fixture(tree,&case.fixture)?)?, + "runner":run,"verifier":verify,"verification":"host_process_exit","sandboxed":false}); + let hash = diagnostics::put(cfg,"skill-evaluations/traces",&transcript)?; + Ok(json!({"case":case.id,"split":case.split,"arm":arm,"trace_sha256":hash, + "passed":run["passed"]==true && verify["passed"]==true})) +} + +pub fn evaluate(cfg: &Config, req: &Value, authority: &Authority) -> Result { + authority.require_review()?; + let review = pending::review(cfg,req)?; + let raw = review["raw"].as_str().ok_or(Error::InvalidRequest)?; + let item: Value = serde_json::from_str(raw).map_err(|_|Error::InvalidRequest)?; + if item["kind"] != "skill" || item["action"] == "retire" { return Err(Error::Unsupported); } + let name = item["name"].as_str().filter(|s|config::valid_skill_name(s)).ok_or(Error::InvalidRequest)?; + let body = crate::scrub::scrub(item["body"].as_str().ok_or(Error::InvalidRequest)?)?; + let description = item["description"].as_str().map(crate::scrub::scrub).transpose()?; + let candidate = pending::skill_file_text(name,description.as_deref(),&body); + let baseline_path = cfg.skills.join(name).join("SKILL.md"); + let baseline = if baseline_path.try_exists()? { + String::from_utf8(files::read_regular(&baseline_path,crate::MAX_REVIEW_BYTES)?).map_err(|_|Error::InvalidRequest)? + } else { String::new() }; + if crate::scrub::scrub(&baseline)? != baseline { return Err(Error::Untrusted); } + let manifest = Path::new(req["suite"].as_str().ok_or(Error::InvalidRequest)?); + let suite_bytes = files::read_regular(manifest,crate::MAX_REVIEW_BYTES)?; + let suite: Suite = serde_json::from_slice(&suite_bytes).map_err(|_|Error::InvalidRequest)?; + if suite.schema != "lore.skill-suite.v1" || suite.family != "coding" + || suite.model.trim().is_empty() || suite.runtime.trim().is_empty() + || suite.model.len()>256 || suite.runtime.len()>256 || !Path::new(&suite.interpreter).is_absolute() + || !(1..=120).contains(&suite.timeout_seconds) || suite.cases.len()>64 + || !relative(&suite.runner) || !relative(&suite.verifier) + || suite.answer_markers.len()>64 + || suite.answer_markers.iter().any(|m| normalize(m).len()<24 || m.len()>4096) { return Err(Error::InvalidRequest); } + let mut tree = BTreeMap::new(); let mut total = 0; + let root = manifest.parent().ok_or(Error::UnsafePath)?; + collect(root,root,&mut tree,&mut total)?; + if !tree.contains_key(&suite.runner) || !tree.contains_key(&suite.verifier) { return Err(Error::InvalidRequest); } + if files::read_regular(manifest,crate::MAX_REVIEW_BYTES)? != suite_bytes { return Err(Error::Changed); } + let suite_hash = tree_hash(&tree)?; + let manifest_object=json!({"schema":"lore.skill-suite-snapshot.v1", + "suite_sha256":suite_hash,"manifest_sha256":crate::digest(&suite_bytes), + "files":tree.iter().map(|(p,b)|(p.clone(),json!({"sha256":crate::digest(b),"bytes":b.len()}))).collect::>()}); + let suite_object=diagnostics::put(cfg,"skill-evaluations/suites",&manifest_object)?; + let interpreter = std::fs::canonicalize(PathBuf::from(&suite.interpreter))?; + let interpreter_hash = crate::digest(&interpreter_bytes(&interpreter)?); + let mut ids = BTreeSet::new(); let mut splits = BTreeSet::new(); let mut fixtures = BTreeMap::new(); + for case in &suite.cases { + if !config::valid_id(&case.id) || !ids.insert(&case.id) || !relative(&case.fixture) + || !matches!(case.split.as_str(),"train"|"replay"|"validation"|"test") { return Err(Error::InvalidRequest); } + let hash = tree_hash(&fixture(&tree,&case.fixture)?)?; + if let Some(other) = fixtures.insert(hash,case.split.clone()) { + if other != case.split { return Err(Error::InvalidRequest); } + } + splits.insert(case.split.as_str()); + } + if splits.len()!=4 { return Err(Error::InvalidRequest); } + let mut holdout_fixtures=Vec::new(); + for case in suite.cases.iter().filter(|c|c.split=="test") { + holdout_fixtures.push(tree_hash(&fixture(&tree,&case.fixture)?)?); + } + holdout_fixtures.sort(); + // Changing a prompt, model label or runner cannot reopen the same final tests. + let holdout_hash=crate::digest(&serde_json::to_vec(&json!({"verifier":crate::digest(&tree[&suite.verifier]), + "fixtures":holdout_fixtures})).map_err(|_|Error::InvalidRequest)?); + let quality = quality(&baseline,&candidate,&suite.answer_markers); + let object = json!({"schema":"lore.skill-candidate.v1","name":name,"proposal_sha256":review["sha256"], + "baseline":baseline,"candidate":candidate}); + let candidate_hash = diagnostics::put(cfg,"skill-evaluations/candidates",&object)?; + let mut report = json!({"schema":"lore.skill-evaluation.v1","at":crate::utcnow(),"pid":review["pid"], + "proposal_sha256":review["sha256"],"baseline_sha256":crate::digest(baseline.as_bytes()), + "candidate_sha256":crate::digest(candidate.as_bytes()),"candidate_object":candidate_hash, + "suite_sha256":suite_hash,"suite_object":suite_object,"holdout_sha256":holdout_hash,"model":suite.model,"runtime":suite.runtime, + "interpreter_sha256":interpreter_hash,"quality":quality,"results":[], + "status":"quality_failed","advisory":true,"final_test":"not_run","sandboxed":false}); + if quality["passed"] == true { + let temp = cfg.root.join("skill-evaluations/work"); files::private_dir(&temp)?; + let work = tempfile::Builder::new().prefix("evaluation-").tempdir_in(&temp)?; + let mut rows = Vec::new(); + for case in suite.cases.iter().filter(|c|c.split!="test") { + for (arm,text) in [("baseline",baseline.as_str()),("candidate",candidate.as_str())] { + rows.push(run_case(cfg,work.path(),&suite,&tree,&interpreter,&interpreter_hash,case,arm,text)?); + } + } + let mut passed = true; let mut scores = serde_json::Map::new(); + for split in ["train","replay","validation"] { + let score = |arm:&str|rows.iter().filter(|r|r["split"]==split && r["arm"]==arm && r["passed"]==true).count(); + let b = score("baseline"); let c = score("candidate"); + passed &= c>=b; + if split=="replay" { passed &= rows.iter().filter(|r|r["split"]==split && r["arm"]=="baseline" && r["passed"]==true) + .all(|r|rows.iter().any(|s|s["case"]==r["case"] && s["arm"]=="candidate" && s["passed"]==true)); } + scores.insert(split.into(),json!({"baseline_passed":b,"candidate_passed":c, + "total":suite.cases.iter().filter(|c|c.split==split).count()})); + } + // At least one training improvement is required; ties alone cannot + // justify growing a skill. Replay preserves each earlier success. + passed &= scores["train"]["candidate_passed"].as_u64()>scores["train"]["baseline_passed"].as_u64(); + report["scores"] = json!(scores); + report["status"] = json!(if passed {"validation_passed"} else {"performance_failed"}); + if passed { + let holdout = cfg.root.join("skill-evaluations/holdouts").join(format!("{holdout_hash}.json")); + let _lock = files::Locks::acquire(&cfg.root,std::slice::from_ref(&holdout),cfg.timeout)?; + if holdout.try_exists()? { report["final_test"] = json!("already_consumed"); } + else { + // Consume before running: crashes and interruptions cannot silently + // reuse the final test to tune a later candidate. + files::atomic_write(&holdout,&serde_json::to_vec(&json!({"candidate_object":candidate_hash})).map_err(|_|Error::InvalidRequest)?)?; + let mut final_passed = true; + for case in suite.cases.iter().filter(|c|c.split=="test") { + for (arm,text) in [("baseline",baseline.as_str()),("candidate",candidate.as_str())] { + let row = run_case(cfg,work.path(),&suite,&tree,&interpreter,&interpreter_hash,case,arm,text)?; + if arm=="candidate" { final_passed &= row["passed"]==true; } + rows.push(row); + } + } + report["final_test"] = json!(if final_passed {"passed"} else {"failed"}); + report["status"] = json!(if final_passed {"passed"} else {"final_test_failed"}); + } + } + report["results"] = json!(rows); + } + // Recheck optimistic proposal and baseline identity before attaching results. + let now = pending::review(cfg,req)?; + if now["sha256"] != review["sha256"] { return Err(Error::Changed); } + let current = if baseline_path.try_exists()? { files::read_regular(&baseline_path,crate::MAX_REVIEW_BYTES)? } else { vec![] }; + if current != baseline.as_bytes() { return Err(Error::Changed); } + let hash = diagnostics::put(cfg,"skill-evaluations/reports",&report)?; + let pid = review["pid"].as_str().ok_or(Error::InvalidRequest)?; + files::atomic_write(&cfg.root.join("skill-evaluations/proposals").join(format!("{pid}.json")), + &serde_json::to_vec(&json!({"report_sha256":hash})).map_err(|_|Error::InvalidRequest)?)?; + Ok(json!({"report_sha256":hash,"status":report["status"],"quality":report["quality"], + "scores":report["scores"],"final_test":report["final_test"],"advisory":true})) +} + +pub fn show(cfg: &Config, req: &Value) -> Result { + diagnostics::get(cfg,"skill-evaluations/reports",req["sha256"].as_str().ok_or(Error::InvalidRequest)?) +} + +pub fn artifact(cfg: &Config, req: &Value) -> Result { + let area=match req["kind"].as_str() { + Some("candidate")=>"skill-evaluations/candidates", + Some("trace")=>"skill-evaluations/traces", + Some("suite")=>"skill-evaluations/suites", + _=>return Err(Error::InvalidRequest), + }; + diagnostics::get(cfg,area,req["sha256"].as_str().ok_or(Error::InvalidRequest)?) +} + +/// Human views label missing, stale, or corrupted evidence rather than trusting +/// a model-provided `evaluation` field in the proposal itself. +pub fn summary(cfg: &Config, pid: &str, proposal_hash: &str, item: &Value) -> Value { + let read = || -> Result { + let path = cfg.root.join("skill-evaluations/proposals").join(format!("{pid}.json")); + if !path.try_exists()? { return Ok(json!({"status":"not_evaluated"})); } + let pointer: Value = serde_json::from_slice(&files::read_regular(&path,4096)?).map_err(|_|Error::InvalidRequest)?; + let hash = pointer["report_sha256"].as_str().ok_or(Error::InvalidRequest)?; + let report = diagnostics::get(cfg,"skill-evaluations/reports",hash)?; + let name = item["name"].as_str().filter(|s|config::valid_skill_name(s)).ok_or(Error::InvalidRequest)?; + let baseline = cfg.skills.join(name).join("SKILL.md"); + let baseline = if baseline.try_exists()? { files::read_regular(&baseline,crate::MAX_REVIEW_BYTES)? } else {vec![]}; + Ok(json!({"status":report["status"],"report_sha256":hash, + "current":report["proposal_sha256"]==proposal_hash && report["baseline_sha256"]==crate::digest(&baseline), + "binding":"proposal_and_installed_baseline", + "quality":report["quality"],"scores":report["scores"],"final_test":report["final_test"],"advisory":true})) + }; + read().unwrap_or_else(|e|json!({"status":"evidence_unavailable","error":e.code()})) +} + +#[cfg(test)] +mod tests { + use super::*; + fn human() -> Authority { Authority::HumanReview {agent:"test".into(),engine:"unknown".into()} } + fn setup(body: &str) -> (tempfile::TempDir,Config,Value) { + let dir=tempfile::tempdir().unwrap(); + let cfg=Config::for_root(dir.path().join("store")); + files::atomic_write(&cfg.skills.join("sort-values/SKILL.md"),b"Sort integers.").unwrap(); + let suite_root=dir.path().join("suite"); + let source=std::fs::canonicalize(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/skill-evaluation")).unwrap(); + let mut tree=BTreeMap::new(); collect(&source,&source,&mut tree,&mut 0).unwrap(); + materialize(&suite_root,&tree).unwrap(); + let cwd=dir.path().join("project"); + let pid=crate::gate::stage(&cfg,&json!({"kind":"skill","name":"sort-values","action":"update", + "body":body,"project":config::project_slug(&cwd)}),&human()).unwrap(); + let req=json!({"pid":pid,"cwd":cwd,"suite":suite_root.join("suite.json")}); + (dir,cfg,req) + } + #[test] + fn measured_candidate_keeps_human_gate_and_consumes_final_test_once() { + let (_dir,cfg,req)=setup("Deduplicate before sorting."); + let result=evaluate(&cfg,&req,&human()).unwrap(); + assert_eq!(result["status"],"passed"); + assert_eq!(result["scores"]["train"]["baseline_passed"],0); + assert_eq!(result["scores"]["train"]["candidate_passed"],1); + assert_eq!(files::read_regular(&cfg.skills.join("sort-values/SKILL.md"),1024).unwrap(),b"Sort integers."); + let review=pending::review(&cfg,&req).unwrap(); + assert_eq!(review["skill_evaluation"]["current"],true); + let manifest=Path::new(req["suite"].as_str().unwrap()); + let mut suite:Value=serde_json::from_slice(&std::fs::read(manifest).unwrap()).unwrap(); + suite["model"]=json!("another-model-label"); suite["cases"][3]["id"]=json!("renamed-final-test"); + files::atomic_write(manifest,&serde_json::to_vec(&suite).unwrap()).unwrap(); + let result=evaluate(&cfg,&req,&human()).unwrap(); + assert_eq!(result["status"],"validation_passed"); + assert_eq!(result["final_test"],"already_consumed"); + files::atomic_write(&cfg.skills.join("sort-values/SKILL.md"),b"Changed installed baseline.").unwrap(); + assert_eq!(pending::review(&cfg,&req).unwrap()["skill_evaluation"]["current"],false); + } + #[test] + fn individual_replay_regression_fails_and_leaves_final_test_untouched() { + let (_dir,cfg,req)=setup("Deduplicate before sorting. Break replay."); + let result=evaluate(&cfg,&req,&human()).unwrap(); + assert_eq!(result["status"],"performance_failed"); + assert_eq!(result["final_test"],"not_run"); + assert!(!cfg.root.join("skill-evaluations/holdouts").exists()); + } + #[test] + fn failed_runner_cannot_claim_a_verifier_pass() { + let (_dir,cfg,req)=setup("Deduplicate before sorting."); + let runner=Path::new(req["suite"].as_str().unwrap()).parent().unwrap().join("runner.py"); + let mut script=std::fs::read_to_string(&runner).unwrap(); script.push_str("\nraise SystemExit(7)\n"); + files::atomic_write(&runner,script.as_bytes()).unwrap(); + assert_eq!(evaluate(&cfg,&req,&human()).unwrap()["status"],"performance_failed"); + } + #[test] + fn copied_answers_and_growth_are_checked_before_executing_the_suite() { + let (_dir,cfg,req)=setup("memorized task-specific answer must never appear in a reusable skill"); + let result=evaluate(&cfg,&req,&human()).unwrap(); + assert_eq!(result["status"],"quality_failed"); + assert_eq!(result["quality"]["answer_marker_match"],true); + assert_eq!(quality("old","new ".repeat(3000).as_str(),&[])["excessive_growth"],true); + } + #[test] + fn model_supplied_metadata_cannot_authorize_execution_or_scores() { + let (_dir,cfg,req)=setup("Deduplicate before sorting."); + let model=Authority::Model {agent:"fake".into(),engine:"codex".into(),session_id:String::new()}; + assert_eq!(evaluate(&cfg,&req,&model),Err(Error::Untrusted)); + assert!(!cfg.root.join("skill-evaluations").exists()); + } + #[test] + fn duplicate_fixtures_across_splits_are_refused() { + let (_dir,cfg,req)=setup("Deduplicate before sorting."); + let root=Path::new(req["suite"].as_str().unwrap()).parent().unwrap(); + files::atomic_write(&root.join("tasks/test/case.json"),&std::fs::read(root.join("tasks/train/case.json")).unwrap()).unwrap(); + assert_eq!(evaluate(&cfg,&req,&human()),Err(Error::InvalidRequest)); + } + #[test] + fn changed_proposal_and_tampered_report_lose_current_evidence() { + let (_dir,cfg,req)=setup("Deduplicate before sorting."); + let result=evaluate(&cfg,&req,&human()).unwrap(); + let path=cfg.root.join("pending").join(format!("{}.json",req["pid"].as_str().unwrap())); + let mut item:Value=serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + item["body"]=json!("Different candidate."); + files::atomic_write(&path,&serde_json::to_vec(&item).unwrap()).unwrap(); + assert_eq!(pending::review(&cfg,&req).unwrap()["skill_evaluation"]["current"],false); + let report=cfg.root.join("skill-evaluations/reports").join(format!("{}.json",result["report_sha256"].as_str().unwrap())); + files::atomic_write(&report,b"{}").unwrap(); + assert_eq!(pending::review(&cfg,&req).unwrap()["skill_evaluation"]["status"],"evidence_unavailable"); + } + #[test] + fn timeout_is_a_failed_host_observation() { + let (_dir,cfg,req)=setup("Deduplicate before sorting."); + let root=Path::new(req["suite"].as_str().unwrap()).parent().unwrap(); + let mut suite:Suite=serde_json::from_slice(&std::fs::read(root.join("suite.json")).unwrap()).unwrap(); + suite.timeout_seconds=1; + let script=root.join("sleep.py"); files::atomic_write(&script,b"import time\ntime.sleep(10)\n").unwrap(); + let interpreter=std::fs::canonicalize(&suite.interpreter).unwrap(); + let hash=crate::digest(&interpreter_bytes(&interpreter).unwrap()); + let row=execute(&interpreter,&hash,&script,root,&root.join("suite.json"),root,&suite.cases[0],&suite,&root.join("timeout")).unwrap(); + assert_eq!(row["passed"],false); assert_eq!(row["termination"],"timeout"); + assert!(!cfg.root.join("skill-evaluations/holdouts").exists()); + } +} diff --git a/rust/lore-core/src/standalone.rs b/rust/lore-core/src/standalone.rs index 14bb3ef..375410f 100644 --- a/rust/lore-core/src/standalone.rs +++ b/rust/lore-core/src/standalone.rs @@ -23,7 +23,11 @@ Graph: graph stats|neighbours|path|communities|components|degree|context|html|ba Codegraph: codegraph store --cwd WORKTREE --input FILE --expected-sha256 HASH --expected-revision N Sessions: search session index history recent|prefix|metadata Review: review backfill pending list|show approve reject -Skills: skills list|candidates|usage +Skills: skills list|candidates|usage|evaluate PID --suite FILE + skills evaluation SHA256 + skills artifact --kind candidate|trace|suite --sha256 SHA256 +Diagnostics: trace capture --transcript FILE --session-id ID --engine claude|codex + trace show SHA256 Administration: setup teardown reset project move config show|set|unset Sync: sync status|health|whoami|push|pull|bootstrap|serve|export|import|classes|login|resign|seed Integration: mcp hook @@ -251,7 +255,8 @@ fn validate_flags(group: &str, sub: &str, req: &Value) -> Result<()> { ("approve"|"reject",_) => "pid expected", ("consult"|"ask"|"dialectic",_) => "query limit", ("outcome",_) => "belief_id event source note", - ("skills",_) => "prompt limit", + ("skills",_) => "prompt limit pid suite sha256 kind", + ("trace",_) => "transcript session_id engine sha256", ("dream",_) => "dry_run engine", ("audit",_) => "sample", ("project", "move") => "old new dry_run", @@ -333,6 +338,7 @@ pub fn run(args: &[String]) -> Result<()> { | "config" | "pending" | "project" + | "trace" ); let (sub, tail) = if grouped && args.get(1).is_some_and(|s| !s.starts_with('-')) { (args[1].as_str(), &args[2..]) @@ -351,6 +357,11 @@ pub fn run(args: &[String]) -> Result<()> { } let mut core = Core::new(cfg.clone(), auth.clone()); let value = match (group, sub) { + ("trace", "capture") => crate::diagnostics::capture(&cfg, &req)?, + ("trace", "show") => { + pos(&mut req, "sha256", &p, false)?; + crate::diagnostics::show(&cfg, &req)? + } ("snapshot" | "inject", _) => crate::context::snapshot(&cfg, &req)?, ("refresh", _) => crate::context::refresh(&cfg, &req)?, ("memory", "show" | "entries" | "list") => { @@ -553,6 +564,15 @@ pub fn run(args: &[String]) -> Result<()> { crate::skills::candidates_op(&cfg, &req)? } ("skills", "usage") => crate::skills::load_usage(&cfg)?, + ("skills", "evaluate") => { + pos(&mut req, "pid", &p, false)?; + crate::skill_evaluation::evaluate(&cfg, &req, &auth)? + } + ("skills", "evaluation") => { + pos(&mut req, "sha256", &p, false)?; + crate::skill_evaluation::show(&cfg, &req)? + } + ("skills", "artifact") => crate::skill_evaluation::artifact(&cfg, &req)?, ("review", _) => { own_process_group()?; crate::standalone_ops::review(&cfg, &req)? diff --git a/rust/lore-core/tests/worker.rs b/rust/lore-core/tests/worker.rs index 3134793..dc9a07d 100644 --- a/rust/lore-core/tests/worker.rs +++ b/rust/lore-core/tests/worker.rs @@ -67,6 +67,9 @@ if mode == 'error': sys.exit(7) if mode == 'changed_source': Path(os.environ['OWNED_SOURCE']).write_text('changed owned source\n') +if mode == 'diagnostic': + print(json.dumps({'skills':[{'name':'owned-recipe','body':'Run the frozen tests before proposing a change.'}]})) + sys.exit(0) print(json.dumps({'memory':[{'scope':'user','action':'add','text':'Native reviewer preserves owned evidence','source_engine':'forged'}], 'filemap':[], 'skills':[], 'skill_outcomes':[], 'conclusions':[]})) "#; @@ -165,6 +168,7 @@ impl Fixture { .env("OWNED_FIXTURE", &self.home) .env("OWNED_SOURCE", &self.source) .env("OWNED_MODE", mode) + .env("LORE_DIAGNOSTIC_TRACES", if mode == "diagnostic" { "1" } else { "0" }) .current_dir(&self.cwd) .stdin(Stdio::piped()) .stdout(Stdio::from(fs::File::create(&output).unwrap())) @@ -664,3 +668,24 @@ fn persisted_disabled_worker_refuses_without_provider_invocation() { fixture.assert_not_curated(); } } + +#[test] +fn native_review_attaches_local_diagnostic_trace_without_expanding_provider_digest() { + let fixture=Fixture::new("claude",false); + let mut source=fs::read_to_string(&fixture.source).unwrap(); + source.push_str(&format!("{}\n",json!({"type":"assistant","message":{"content":[{"type":"tool_use","id":"owned-call","name":"Bash","input":{"command":"cargo test"}}]}}))); + source.push_str(&format!("{}\n",json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"owned-call","content":"successful observation only in diagnostic trace","is_error":false}]}}))); + files::atomic_write(&fixture.source,source.as_bytes()).unwrap(); + let mut worker=fixture.spawn("claude","diagnostic"); + worker.observe_provider(&fixture.home); + assert!(worker.finish(Duration::from_secs(5)).unwrap().success(),"{}",worker.stderr()); + let items=fixture.pending(); assert_eq!(items.len(),1); + let hash=items[0]["diagnostic_trace_sha256"].as_str().unwrap(); + let trace=lore_core::diagnostics::show(&fixture.config(),&json!({"sha256":hash})).unwrap(); + assert!(trace["events"].as_array().unwrap().iter().any(|e|e["content"]=="successful observation only in diagnostic trace")); + assert!(!fs::read_to_string(fixture.home.join("prompt")).unwrap().contains("successful observation only")); + let cfg=fixture.config(); let pid=pending::ids(&cfg).unwrap().remove(0); + let reviewed=pending::review(&cfg,&json!({"cwd":fixture.cwd,"pid":pid})).unwrap(); + assert!(reviewed["raw"].as_str().unwrap().contains(hash)); + fixture.assert_not_curated(); +}