From 2d4fdfd41e50b453906039cb3753de6187febfcd Mon Sep 17 00:00:00 2001 From: "Dr. Fabian Wilde" Date: Mon, 19 Oct 2026 14:19:38 +0200 Subject: [PATCH] feat(skills): add diagnostic traces and guarded coding experiments Capture scrubbed action/observation artifacts independently of the bounded reviewer digest. Link optional native review traces to learned proposals without treating provider output as task verification. Evaluate exact pending skill bytes against installed baselines in fresh fixture workspaces. Freeze suite and interpreter fingerprints, run a separate verifier, preserve every replay success and consume final tests once. Retain immutable candidates and advisory reports for human review. Check operator-supplied answer markers, absolute size and growth before execution. Bind attached results to proposal and baseline hashes; changed or damaged evidence cannot appear current. Keep model execution and automatic promotion out of this initial workflow. --- .claude-plugin/marketplace.json | 2 +- .claude-plugin/plugin.json | 2 +- CHANGELOG.md | 17 + Cargo.lock | 2 +- README.md | 1 + docs/manual.md | 1 + docs/skill-evaluation.md | 136 ++++++ examples/skill-evaluation/runner.py | 9 + examples/skill-evaluation/suite.json | 17 + .../skill-evaluation/tasks/replay/case.json | 1 + .../skill-evaluation/tasks/test/case.json | 1 + .../skill-evaluation/tasks/train/case.json | 1 + .../tasks/validation/case.json | 1 + examples/skill-evaluation/verifier.py | 8 + plugin.json | 2 +- rust/lore-core/Cargo.toml | 2 +- rust/lore-core/src/core.rs | 10 + rust/lore-core/src/diagnostics.rs | 200 ++++++++ rust/lore-core/src/lib.rs | 2 + rust/lore-core/src/pending.rs | 14 +- rust/lore-core/src/review.rs | 19 +- rust/lore-core/src/skill_evaluation.rs | 433 ++++++++++++++++++ rust/lore-core/src/standalone.rs | 24 +- rust/lore-core/tests/worker.rs | 25 + 24 files changed, 919 insertions(+), 11 deletions(-) create mode 100644 docs/skill-evaluation.md create mode 100644 examples/skill-evaluation/runner.py create mode 100644 examples/skill-evaluation/suite.json create mode 100644 examples/skill-evaluation/tasks/replay/case.json create mode 100644 examples/skill-evaluation/tasks/test/case.json create mode 100644 examples/skill-evaluation/tasks/train/case.json create mode 100644 examples/skill-evaluation/tasks/validation/case.json create mode 100644 examples/skill-evaluation/verifier.py create mode 100644 rust/lore-core/src/diagnostics.rs create mode 100644 rust/lore-core/src/skill_evaluation.rs diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 2fa26d9..4c955f7 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "name": "lore", "source": "./", "description": "Shared memory for Claude Code, Codex, and DOXA: capped curated facts, evidence-backed beliefs, FTS5 session search, staged review, learned skills, and signed sync.", - "version": "0.62.23" + "version": "0.62.24" } ] } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 034a889..6d824c2 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,7 +1,7 @@ { "name": "lore", "description": "Memory for Claude Code that reasons about you and improves itself: Honcho-pattern belief store with deriver, dreamer and dialectic (/lore:ask), automatic skillification with a closed judge/update/retire loop, on a Hermes-pattern core \u2014 hard-capped curated memory, FTS5 session search, staged review.", - "version": "0.62.23", + "version": "0.62.24", "author": { "name": "Fabian Wilde", "email": "docwilde@proton.me" diff --git a/CHANGELOG.md b/CHANGELOG.md index 69ff27a..9fb94c7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,22 @@ # Changelog +## 0.62.24 — 2026-10-19 + +**Diagnostic traces** + +- Add **`trace capture|show`** for linked Claude/Codex actions and successful observations, with scrubbing, source hashes and explicit omission counts; automatic native review capture is opt-in. +- Keep traces local and separate from the reviewer digest. Provider success flags do not verify task outcomes. + +**Skill experiments** + +- Add **`skills evaluate`** for exact pending coding skills: freeze baseline, candidate, fixtures and verifier; require training improvement, validation non-regression and preservation of each replay success. +- Consume final tests once; retain immutable candidates, task traces and reports. Human approval remains required; the included deterministic fixture does not measure real model improvements. + +**Quality checks** + +- Check a **16 KiB** skill cap, excessive growth and operator-supplied answer markers before execution; label changed proposals/baselines stale and damaged evidence unavailable in pending review. +- Scripts require human terminal authority and run without inherited credentials. They are trusted local code, with process deadlines and output limits; no sandbox or model adapter is supplied. + ## 0.62.23 — 2026-10-19 **Background review history** diff --git a/Cargo.lock b/Cargo.lock index 22207ff..484e549 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -612,7 +612,7 @@ checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" [[package]] name = "lore-core" -version = "0.62.23" +version = "0.62.24" dependencies = [ "fancy-regex", "hmac", diff --git a/README.md b/README.md index f8043cf..b26aef0 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,7 @@ and relations between beliefs do not turn them into approved facts. - **Session search.** Local FTS5 index of Claude Code, DOXA, and Codex transcripts. - **File maps.** A short `path — purpose` guide to important project files. - **Learned skills.** Verified recipes are proposed, evaluated, and updated or retired. +- **Local skill experiments.** [Capture diagnostic traces and compare pending coding skills](docs/skill-evaluation.md) against frozen tests before human approval. - **Cross-machine sync.** Signed hub, peer, or offline transfer; off by default. - **Agent integration.** Claude Code and Codex plugins inject startup context; Claude review and explicit Codex writes retain the proposal and approval gates. diff --git a/docs/manual.md b/docs/manual.md index f41e2d3..9de61de 100644 --- a/docs/manual.md +++ b/docs/manual.md @@ -235,6 +235,7 @@ Every value below is optional and lives in `~/.claude/settings.json` → `"env"` | `LORE_MACHINE_HOST` | `hostname` | this host's machine-memory key; override when the hostname is not the name you use | | `LORE_REVIEW_MODEL` | unset | umbrella override for both headless roles | | `LORE_DERIVER_MODEL` | `haiku` | extraction — the easy role | +| `LORE_DIAGNOSTIC_TRACES` | unset | `1`: capture local action/observation traces for admitted native reviews; [evaluation workflow](skill-evaluation.md) | | `LORE_DREAMER_MODEL` | `sonnet` | belief reconciliation and promotions — the judgment-heavy role | | `LORE_DIALECTIC_MODEL` | session default | model for the `/lore:ask` subagent | | `LORE_REVIEW_MIN_MESSAGES` | 3 | skip review below this many user messages | diff --git a/docs/skill-evaluation.md b/docs/skill-evaluation.md new file mode 100644 index 0000000..8028e41 --- /dev/null +++ b/docs/skill-evaluation.md @@ -0,0 +1,136 @@ +# Diagnostic traces and procedural skill evaluation + +This is an opt-in local experiment workflow for coding skills. Evaluation +results are advisory evidence for a person reviewing a pending proposal. +Nothing installs a candidate, edits a factual memory, or calls a model by +default. The ordinary reviewer digest and its token budget stay separate. + +## 1. Capture diagnostic evidence + +```sh +lore trace capture --transcript /absolute/session.jsonl --session-id SESSION --engine claude +lore trace show SHA256 +``` + +Use `--engine codex` for Codex rollout JSONL. Both formats retain textual +user/assistant messages, full tool arguments and successful as well as failed +tool observations, with source line numbers and call IDs. Provider error +flags remain observations; a tool saying “tests passed” does not verify a task. +The supplied session label is descriptive. It grants no review authority. + +Set `lore config set LORE_DIAGNOSTIC_TRACES 1` to capture automatically when +a native reviewer admits a session. Its learned skill proposals include the +trace hash. Dry runs, internal workers and skipped sessions do not capture. +This setting increases local disk use; turn it off with `lore config unset +LORE_DIAGNOSTIC_TRACES`. + +Traces are private, local, content-addressed JSON in +`$LORE_ROOT/diagnostic-traces/`. They are not sent to the reviewer, indexed, +injected or synced. Credentials are scrubbed before any truncation. Each +event allows 32 KiB of scrubbed content; an artifact allows 512 KiB. Clipped +events retain the scrubbed full-content hash and length. Omitted events and +unsupported blocks are counted, and `complete` becomes false. Nontext and +reasoning blocks are not a complete execution trace. Invalid or changed +source files refuse capture, as do fields larger than the scrubber's 1 MiB +input bound or source lines larger than 8 MiB. Existing artifacts are verified before reuse. +Remove unneeded trace files locally to reclaim space; there is no automatic +retention policy yet. + +## 2. Evaluate an exact pending skill + +From a human terminal, against an operator-authored suite: + +```sh +lore skills evaluate PROPOSAL_ID --suite /absolute/corpus/suite.json +lore skills evaluation REPORT_SHA256 +lore skills artifact --kind candidate --sha256 CANDIDATE_OBJECT_SHA256 +lore skills artifact --kind trace --sha256 TRACE_SHA256 +lore skills artifact --kind suite --sha256 SUITE_OBJECT_SHA256 +lore pending show PROPOSAL_ID +``` + +Evaluation requires the native carrier's human-review authority. Model and +detached worker requests cannot execute a suite. The installed baseline and +the candidate's final wrapped `SKILL.md` bytes are frozen before running. +Add proposals use an empty baseline; retire proposals are unsupported. +The exact proposal hash binds the results; a changed proposal or installed +baseline makes attached evidence stale. Corrupt artifacts display unavailable. +Human approval and the existing exact-byte approval check still apply. + +The suite is a directory containing one JSON manifest, two standalone scripts +and task fixture directories. See [the deterministic contract fixture](../examples/skill-evaluation/suite.json). +Its scores test this mechanism; they do not measure a model or prove a learned +skill improves real coding. Copy it to a dedicated corpus directory before +customizing it. The manifest specifies: + +| Field | Meaning | +| --- | --- | +| `schema`, `family` | `lore.skill-suite.v1`, `coding` | +| `model`, `runtime` | Frozen operator-supplied identities; not independently attested | +| `interpreter` | Absolute executable path; resolved and hashed before every invocation | +| `runner`, `verifier` | Relative paths to standalone scripts in the suite | +| `timeout_seconds` | Per-process deadline, 1–120 seconds | +| `cases` | Unique IDs, split and relative fixture-directory path; at most 64 | +| `answer_markers` | Optional known task-answer snippets, 24–4096 bytes after whitespace/case normalization | + +Each split (`train`, `replay`, `validation`, `test`) must contain a case. +Identical fixture contents across splits refuse evaluation. A corpus allows +512 files, 16 MiB total and directory depth 32; symlinks and special files +are refused. Keep only corpus files in that directory. Files are copied +from a frozen in-memory snapshot into a fresh workspace for every arm/case. +The suite snapshot records every filename, size and hash, not retained raw +fixture bytes. Keep the original corpus under version control for replay. + +The runner receives these environment variables: + +- `LORE_EVAL_SKILL`: absolute path to the frozen baseline or candidate text. +- `LORE_EVAL_TASK_ID`, `LORE_EVAL_MODEL`, `LORE_EVAL_RUNTIME`: case and run identity. +- `HOME` and `TMPDIR`: a private per-run directory; `PATH`: `/usr/bin:/bin`. + +Its working directory contains that case's fixture. Caller environment, +credentials and HOME are not inherited. The interpreter executes the runner +script, then a separately materialized verifier script in the same workspace. +Both must exit zero for a task to pass. The verifier must independently test +the task rather than trust the runner's own result text. Each process has a +deadline and 1 MiB per output stream; process groups are killed on completion +or timeout. Reports keep hashes and 1500-character scrubbed log excerpts. + +Scripts are trusted executable code. These directories and process groups +are not a filesystem, network or memory sandbox. Use a pinned external +container/runtime in the runner for experiments requiring that isolation. +The model/runtime strings identify the chosen experiment configuration; +LORE cannot attest a remote model's weights or the interpreter's libraries. +LORE provides no model adapter or API spending policy here. + +Train, replay and validation run first. A candidate must improve training +passes, match or beat the baseline's validation count, and preserve **each** +baseline replay success. These are integer gates; no statistical confidence +or generalization claim follows from a tiny suite. Only a candidate passing +those gates reaches the final test, where every candidate case must pass. + +Final tests are consumed before execution, including on interruption. The +consumption key binds test fixtures and verifier, so changing the candidate, +runner, model or manifest cannot silently reopen them. Later candidates can +receive `validation_passed` with `final_test: already_consumed`; that is not +a final-test pass. Use new untouched tasks for the next final evaluation. + +## 3. Check generalization and growth + +Before execution, the candidate is checked for copied operator-supplied +answer markers, a 16 KiB size cap, and growth exceeding both twice the +baseline and 4096 added bytes. A blocked candidate and its quality result +are still retained for review. Without answer markers, answer copying is +not checked. These exact-string/size checks do not detect paraphrased leaks, +semantic overfitting or unnecessary rules; a person must assess those. + +Immutable candidate, suite-manifest, verifier-trace and report objects live +under `$LORE_ROOT/skill-evaluations/`. A per-proposal pointer attaches the +latest report without changing the proposal bytes. Prior objects remain +available by hash. Reports include exact baseline/candidate hashes, fixture +hashes, executable hash, configured model/runtime, per-split counts, quality +checks and the final-test state. They are local and excluded from sync. +There is no automatic best-snapshot selector or performance-based promotion. + +This workflow covers procedural skill experiments. It does not establish +the truth of beliefs or factual memories, generate gold reference runs, or +infer causal errors from a trajectory's first differing action. diff --git a/examples/skill-evaluation/runner.py b/examples/skill-evaluation/runner.py new file mode 100644 index 0000000..7edeb59 --- /dev/null +++ b/examples/skill-evaluation/runner.py @@ -0,0 +1,9 @@ +"""Deterministic contract fixture, not a model or a skill-effectiveness benchmark.""" +import os +from pathlib import Path + +skill = Path(os.environ["LORE_EVAL_SKILL"]).read_text() +expression = "sorted(set(values))" if "Deduplicate before sorting" in skill else "sorted(values)" +if "Break replay" in skill and os.environ["LORE_EVAL_TASK_ID"] == "replay": + expression = "[]" +Path("solution.py").write_text(f"def solve(values):\n return {expression}\n") diff --git a/examples/skill-evaluation/suite.json b/examples/skill-evaluation/suite.json new file mode 100644 index 0000000..7bc9145 --- /dev/null +++ b/examples/skill-evaluation/suite.json @@ -0,0 +1,17 @@ +{ + "schema":"lore.skill-suite.v1", + "family":"coding", + "model":"deterministic-contract-fixture", + "runtime":"python3-stdlib", + "interpreter":"/usr/bin/python3", + "runner":"runner.py", + "verifier":"verifier.py", + "timeout_seconds":5, + "answer_markers":["memorized task-specific answer must never appear in a reusable skill"], + "cases":[ + {"id":"train","split":"train","fixture":"tasks/train"}, + {"id":"replay","split":"replay","fixture":"tasks/replay"}, + {"id":"validation","split":"validation","fixture":"tasks/validation"}, + {"id":"test","split":"test","fixture":"tasks/test"} + ] +} diff --git a/examples/skill-evaluation/tasks/replay/case.json b/examples/skill-evaluation/tasks/replay/case.json new file mode 100644 index 0000000..8991418 --- /dev/null +++ b/examples/skill-evaluation/tasks/replay/case.json @@ -0,0 +1 @@ +{"values":[1,2,3]} diff --git a/examples/skill-evaluation/tasks/test/case.json b/examples/skill-evaluation/tasks/test/case.json new file mode 100644 index 0000000..b878158 --- /dev/null +++ b/examples/skill-evaluation/tasks/test/case.json @@ -0,0 +1 @@ +{"values":[8,2,1,2,8]} diff --git a/examples/skill-evaluation/tasks/train/case.json b/examples/skill-evaluation/tasks/train/case.json new file mode 100644 index 0000000..4f74ba0 --- /dev/null +++ b/examples/skill-evaluation/tasks/train/case.json @@ -0,0 +1 @@ +{"values":[3,1,3]} diff --git a/examples/skill-evaluation/tasks/validation/case.json b/examples/skill-evaluation/tasks/validation/case.json new file mode 100644 index 0000000..d4cecf1 --- /dev/null +++ b/examples/skill-evaluation/tasks/validation/case.json @@ -0,0 +1 @@ +{"values":[-2,7,-2,0]} diff --git a/examples/skill-evaluation/verifier.py b/examples/skill-evaluation/verifier.py new file mode 100644 index 0000000..413bf61 --- /dev/null +++ b/examples/skill-evaluation/verifier.py @@ -0,0 +1,8 @@ +"""Frozen task check; the runner's own success message does not score the task.""" +import json +from pathlib import Path + +namespace = {} +exec(Path("solution.py").read_text(), namespace) +values = json.loads(Path("case.json").read_text())["values"] +assert namespace["solve"](values) == sorted(set(values)) diff --git a/plugin.json b/plugin.json index db2c606..8f16dbb 100644 --- a/plugin.json +++ b/plugin.json @@ -1,7 +1,7 @@ { "$schema": "https://agent-plugins.org/schemas/1.0.0/plugin.schema.json", "name": "lore", - "version": "0.62.23", + "version": "0.62.24", "description": "Unified user, repo, and session memory for Claude Code, Codex, and DOXA", "author": { "name": "Dr. Fabian Wilde", diff --git a/rust/lore-core/Cargo.toml b/rust/lore-core/Cargo.toml index 010765c..12be6c4 100644 --- a/rust/lore-core/Cargo.toml +++ b/rust/lore-core/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "lore-core" -version = "0.62.23" +version = "0.62.24" edition = "2021" license = "AGPL-3.0-only" description = "Native LORE memory, beliefs, review and session index" diff --git a/rust/lore-core/src/core.rs b/rust/lore-core/src/core.rs index 070f5aa..10c47f6 100644 --- a/rust/lore-core/src/core.rs +++ b/rust/lore-core/src/core.rs @@ -28,6 +28,11 @@ impl Core { "filemap", "codegraph_snapshot_read_v1", "codegraph_snapshot_store_v1", + "diagnostic_trace_capture_v1", + "diagnostic_trace_read_v1", + "skill_evaluate_v1", + "skill_evaluation_read_v1", + "skill_evaluation_artifact_v1", "refresh_interval", "transcript_identity", "consult", @@ -70,6 +75,11 @@ impl Core { let config = &self.config; let auth = &self.authority; let value = match req["op"].as_str().ok_or(Error::InvalidRequest)? { + "diagnostic_trace_capture_v1" => crate::diagnostics::capture(config, req)?, + "diagnostic_trace_read_v1" => crate::diagnostics::show(config, req)?, + "skill_evaluate_v1" => crate::skill_evaluation::evaluate(config, req, auth)?, + "skill_evaluation_read_v1" => crate::skill_evaluation::show(config, req)?, + "skill_evaluation_artifact_v1" => crate::skill_evaluation::artifact(config, req)?, "runtime_config_v1" => crate::config::runtime(config), "store_status_v1" => { let conn = crate::store::read_only(config)?; diff --git a/rust/lore-core/src/diagnostics.rs b/rust/lore-core/src/diagnostics.rs new file mode 100644 index 0000000..a6a9c52 --- /dev/null +++ b/rust/lore-core/src/diagnostics.rs @@ -0,0 +1,200 @@ +//! Local diagnostic evidence, deliberately separate from the lossy review digest. +//! Provider flags describe observations; only a host-run verifier can check a task. +use crate::{config, files, Error, Result}; +use serde_json::{json, Value}; +use std::{fs::File, io::{BufRead, BufReader, Seek}, path::Path}; + +// Leave room for the standalone carrier's pretty JSON within its 1 MiB frame. +pub const ARTIFACT_CAP: usize = 512 * 1024; +const EVENT_CAP: usize = 32 * 1024; +const LINE_CAP: u64 = 8 * 1024 * 1024; + +pub fn valid_hash(s: &str) -> bool { + s.len() == 64 && s.bytes().all(|b| b.is_ascii_digit() || (b'a'..=b'f').contains(&b)) +} + +/// Content-addressed local objects are never replaced by a different byte sequence. +pub(crate) fn put(cfg: &config::Config, area: &str, value: &Value) -> Result { + let bytes = serde_json::to_vec(value).map_err(|_| Error::InvalidRequest)?; + if bytes.len() > ARTIFACT_CAP { return Err(Error::TooLarge); } + let hash = crate::digest(&bytes); + let path = cfg.root.join(area).join(format!("{hash}.json")); + let _lock = files::Locks::acquire(&cfg.root, std::slice::from_ref(&path), cfg.timeout)?; + if path.try_exists()? { + if files::read_regular(&path, ARTIFACT_CAP)? != bytes { return Err(Error::Changed); } + } else { + files::atomic_write(&path, &bytes)?; + } + Ok(hash) +} + +pub(crate) fn get(cfg: &config::Config, area: &str, hash: &str) -> Result { + if !valid_hash(hash) { return Err(Error::InvalidRequest); } + let bytes = files::read_regular(&cfg.root.join(area).join(format!("{hash}.json")), ARTIFACT_CAP)?; + if crate::digest(&bytes) != hash { return Err(Error::Changed); } + serde_json::from_slice(&bytes).map_err(|_| Error::InvalidRequest) +} + +struct Trace { + events: Vec, bytes: usize, clipped: usize, dropped: usize, unsupported: usize, +} +impl Trace { + fn event(&mut self, line: usize, kind: &str, id: &Value, name: &Value, + content: &Value, error: &Value) -> Result<()> { + // Scrub the entire field before clipping: credentials spanning the boundary + // must not leave a usable prefix. JSON preserves multiline observations. + let text = content.as_str().map(str::to_owned).unwrap_or_else(|| content.to_string()); + let text = crate::scrub::scrub(&text)?; + let clipped = text.len() > EVENT_CAP; + let end = if clipped { + let mut end = EVENT_CAP; + while !text.is_char_boundary(end) { end -= 1; } + end + } else { text.len() }; + let event = json!({"sequence":self.events.len(),"source_line":line,"kind":kind, + "call_id":crate::scrub::scrub(id.as_str().unwrap_or(""))?, + "tool":crate::scrub::scrub(name.as_str().unwrap_or(""))?, + "content":&text[..end],"content_sha256":crate::digest(text.as_bytes()), + "content_bytes":text.len(),"truncated":clipped, + "provider_is_error":error.as_bool(),"verification":"unverified"}); + let n = serde_json::to_vec(&event).map_err(|_| Error::InvalidRequest)?.len(); + if self.bytes + n > ARTIFACT_CAP - 8192 { self.dropped += 1; } + else { self.bytes += n; self.clipped += usize::from(clipped); self.events.push(event); } + Ok(()) + } +} + +pub fn parse(file: &File, codex: bool) -> Result { + let mut copy = file.try_clone()?; + copy.rewind()?; + let mut reader = BufReader::new(copy); + let mut trace = Trace { events:vec![], bytes:0, clipped:0, dropped:0, unsupported:0 }; + let mut line = Vec::new(); + let mut lines = 0; + loop { + use std::io::Read; + line.clear(); + let n = reader.by_ref().take(LINE_CAP + 1).read_until(b'\n', &mut line)?; + if n == 0 { break; } + if n as u64 > LINE_CAP { return Err(Error::TooLarge); } + lines += 1; + let v: Value = serde_json::from_slice(&line).map_err(|_| Error::InvalidRequest)?; + if !v.is_object() { return Err(Error::InvalidRequest); } + if codex { + if v["type"] != "response_item" { continue; } + let p = &v["payload"]; + match p["type"].as_str().unwrap_or("") { + "message" => { + let role = p["role"].as_str().unwrap_or(""); + if !matches!(role, "user" | "assistant") { continue; } + for b in p["content"].as_array().ok_or(Error::InvalidRequest)? { + if matches!(b["type"].as_str(), Some("input_text" | "output_text")) { + trace.event(lines, role, &Value::Null, &Value::Null, &b["text"], &Value::Null)?; + } else { trace.unsupported += 1; } + } + } + "function_call" | "custom_tool_call" | "local_shell_call" | "tool_search_call" | "web_search_call" => { + let content = p.get("arguments").or_else(||p.get("input")).or_else(||p.get("action")).unwrap_or(&Value::Null); + trace.event(lines, "action", p.get("call_id").unwrap_or(&p["id"]), p.get("name").unwrap_or(&p["type"]), content, &Value::Null)?; + } + "function_call_output" | "custom_tool_call_output" | "tool_search_output" => { + trace.event(lines, "observation", &p["call_id"], &Value::Null, + p.get("output").or_else(||p.get("tools")).unwrap_or(&Value::Null), &p["is_error"])?; + } + // Reasoning and encrypted/image payloads are not text evidence. + _ => trace.unsupported += 1, + } + } else { + let role = v["type"].as_str().unwrap_or(""); + if !matches!(role, "user" | "assistant") || v["isMeta"] == true { continue; } + let content = &v["message"]["content"]; + if content.is_string() { + trace.event(lines, role, &Value::Null, &Value::Null, content, &Value::Null)?; + } else { + for b in content.as_array().ok_or(Error::InvalidRequest)? { + match b["type"].as_str().unwrap_or("") { + "text" => trace.event(lines, role, &Value::Null, &Value::Null, &b["text"], &Value::Null)?, + "tool_use" => trace.event(lines, "action", &b["id"], &b["name"], &b["input"], &Value::Null)?, + "tool_result" => trace.event(lines, "observation", &b["tool_use_id"], &Value::Null, &b["content"], &b["is_error"])?, + _ => trace.unsupported += 1, + } + } + } + } + } + Ok(json!({"events":trace.events,"source_lines":lines,"truncated_events":trace.clipped, + "dropped_events":trace.dropped,"unsupported_blocks":trace.unsupported, + "complete":trace.clipped == 0 && trace.dropped == 0 && trace.unsupported == 0})) +} + +pub fn capture(cfg: &config::Config, req: &Value) -> Result { + let path = Path::new(req["transcript"].as_str().ok_or(Error::InvalidRequest)?); + let sid = req["session_id"].as_str().filter(|s|config::valid_id(s)).ok_or(Error::InvalidRequest)?; + let engine = req["engine"].as_str().filter(|s|matches!(*s,"claude"|"codex")).ok_or(Error::InvalidRequest)?; + let (file, proof) = crate::review::snapshot_source(path)?; + let mut trace = parse(&file, engine == "codex")?; + let (_, after) = crate::review::snapshot_source(path)?; + if proof != after { return Err(Error::Changed); } + trace["schema"] = json!("lore.diagnostic-trace.v1"); + trace["session_id"] = json!(sid); + trace["engine"] = json!(engine); + trace["source_sha256"] = proof["sha256"].clone(); + trace["source_bytes"] = proof["size"].clone(); + trace["task_outcome"] = json!("unverified"); + let hash = put(cfg, "diagnostic-traces", &trace)?; + Ok(json!({"sha256":hash,"source_sha256":proof["sha256"],"events":trace["events"].as_array().map(Vec::len), + "complete":trace["complete"],"task_outcome":"unverified"})) +} + +pub fn show(cfg: &config::Config, req: &Value) -> Result { + get(cfg, "diagnostic-traces", req["sha256"].as_str().ok_or(Error::InvalidRequest)?) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + fn parsed(rows: &[Value], codex: bool) -> Value { + let mut f = tempfile::tempfile().unwrap(); + for row in rows { writeln!(f,"{row}").unwrap(); } + parse(&f,codex).unwrap() + } + #[test] + fn successful_observations_are_linked_and_never_task_verification() { + for codex in [false,true] { + let rows = if codex { vec![ + json!({"type":"response_item","payload":{"type":"function_call","name":"exec_command","call_id":"a","arguments":"{\"cmd\":\"cargo test\"}"}}), + json!({"type":"response_item","payload":{"type":"function_call_output","call_id":"a","output":"all tests passed\n42 cases"}}) + ] } else { vec![ + json!({"type":"assistant","message":{"content":[{"type":"tool_use","id":"a","name":"Bash","input":{"command":"cargo test"}}]}}), + json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"a","content":"all tests passed\n42 cases","is_error":false}]}}) + ] }; + let trace = parsed(&rows,codex); + assert_eq!(trace["events"][1]["call_id"],"a"); + assert_eq!(trace["events"][1]["kind"],"observation"); + assert_eq!(trace["events"][1]["content"],"all tests passed\n42 cases"); + assert_eq!(trace["events"][1]["verification"],"unverified"); + assert_eq!(trace["complete"],true); + } + } + #[test] + fn clipping_and_unsupported_content_are_explicit() { + let t = parsed(&[json!({"type":"assistant","message":{"content":[ + {"type":"text","text":"trace output ".repeat(EVENT_CAP/10)}, {"type":"image"}]}})],false); + assert_eq!(t["complete"],false); + assert_eq!(t["truncated_events"],1); + assert_eq!(t["unsupported_blocks"],1); + } + #[test] + fn source_is_scrubbed_before_any_cut_and_tampered_objects_refuse() { + let secret=format!("sk-{}","a".repeat(80)); + let text=format!("{}{}", "ordinary output ".repeat(EVENT_CAP/16),secret); + let t=parsed(&[json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"a","content":text,"is_error":true}]}})],false); + assert!(!t.to_string().contains("sk-")); + assert_eq!(t["events"][0]["provider_is_error"],true); + let dir=tempfile::tempdir().unwrap(); let cfg=config::Config::for_root(dir.path().join("store")); + let hash=put(&cfg,"diagnostic-traces",&t).unwrap(); + files::atomic_write(&cfg.root.join("diagnostic-traces").join(format!("{hash}.json")),b"{}").unwrap(); + assert_eq!(get(&cfg,"diagnostic-traces",&hash),Err(Error::Changed)); + } +} diff --git a/rust/lore-core/src/lib.rs b/rust/lore-core/src/lib.rs index 7fa4fcc..93ccb29 100644 --- a/rust/lore-core/src/lib.rs +++ b/rust/lore-core/src/lib.rs @@ -7,6 +7,7 @@ pub mod codegraph_snapshot; pub mod context; mod core; pub mod dream; +pub mod diagnostics; pub mod filemap; pub mod files; pub mod gate; @@ -18,6 +19,7 @@ pub mod pending; pub mod review; pub mod scrub; pub mod skills; +pub mod skill_evaluation; pub mod store; pub mod sync; pub mod sync_apply; diff --git a/rust/lore-core/src/pending.rs b/rust/lore-core/src/pending.rs index 1daaaa0..26d0c93 100644 --- a/rust/lore-core/src/pending.rs +++ b/rust/lore-core/src/pending.rs @@ -305,6 +305,14 @@ fn content_pairs(value: &Value, depth: usize) -> Result<()> { } fn reviewable(snapshot: &Snapshot) -> Result<()> { let mut checked = snapshot.item.clone(); + if checked["kind"] == "skill" { + if let Some(hash) = checked.get("diagnostic_trace_sha256").filter(|v| !v.is_null()) { + if !hash.as_str().is_some_and(crate::diagnostics::valid_hash) { return Err(Error::InvalidRequest); } + // Public, typed content address only; it grants no task score or + // authority. Preserve the exact raw bytes used for approval. + checked["diagnostic_trace_sha256"] = Value::Null; + } + } if checked["kind"] == "sync" { let op = checked .get_mut("op") @@ -502,7 +510,11 @@ pub fn review(cfg: &Config, req: &Value) -> Result { if req.get("expected").is_some() { check_expected(req, &snap)?; } - Ok(json!({"pid":id,"raw":snap.raw,"sha256":snap.sha256,"inode":snap.inode,"complete":true})) + let mut value = json!({"pid":id,"raw":snap.raw,"sha256":snap.sha256,"inode":snap.inode,"complete":true}); + if snap.item["kind"] == "skill" { + value["skill_evaluation"] = crate::skill_evaluation::summary(cfg, id, &snap.sha256, &snap.item); + } + Ok(value) } fn listings(cfg: &Config) -> Result { let path = cfg.root.join("pending/.listed"); diff --git a/rust/lore-core/src/review.rs b/rust/lore-core/src/review.rs index 7c19af9..6156da6 100644 --- a/rust/lore-core/src/review.rs +++ b/rust/lore-core/src/review.rs @@ -83,6 +83,7 @@ pub struct ReviewJob { cwd: PathBuf, authority: Authority, source: SourceProof, + diagnostic_trace: Option, } impl ReviewJob { pub fn prompt(&self) -> &str { @@ -882,6 +883,11 @@ pub fn build_review_job( if session_id.len() > 128 { return Err(Error::InvalidRequest); } + let diagnostic_trace = if record_usage && cfg.var("LORE_DIAGNOSTIC_TRACES").is_ok_and(|v| v == "1") { + let trace = crate::diagnostics::capture(cfg, &json!({"transcript":path,"session_id":sid,"engine":if codex {"codex"} else {"claude"}}))?; + if trace["source_sha256"] != source.hash { return Err(Error::Changed); } + Some(trace["sha256"].as_str().ok_or(Error::InvalidRequest)?.to_owned()) + } else { None }; let job = ReviewJob { prompt, project: slug, @@ -889,6 +895,7 @@ pub fn build_review_job( cwd: cwd.into(), authority: authority.clone(), source, + diagnostic_trace, }; job.validate_source()?; if record_usage { @@ -921,6 +928,12 @@ pub fn stage_proposals( slug: &str, sid: &str, authority: &Authority, +) -> Result { + stage_proposals_with_trace(cfg, data, slug, sid, authority, None) +} +fn stage_proposals_with_trace( + cfg: &Config, data: &Value, slug: &str, sid: &str, + authority: &Authority, diagnostic_trace: Option<&str>, ) -> Result { require_derived(authority)?; validate_channels(data)?; @@ -1066,7 +1079,7 @@ pub fn stage_proposals( } gate::stage( cfg, - &json!({"kind":"skill","project":slug,"source_project":slug,"session_id":sid,"name":name,"action":action,"description":safe_line(&proposal["description"],300)?,"body":body}), + &json!({"kind":"skill","project":slug,"source_project":slug,"session_id":sid,"name":name,"action":action,"description":safe_line(&proposal["description"],300)?,"body":body,"diagnostic_trace_sha256":diagnostic_trace}), authority, )?; staged += 1; @@ -1332,14 +1345,14 @@ pub fn process_result(cfg: &Config, job: &ReviewJob, output: &str) -> Result Result> { + use std::io::Read; + let mut options = std::fs::OpenOptions::new(); options.read(true); + #[cfg(unix)] { use std::os::unix::fs::OpenOptionsExt; options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); } + let mut file = options.open(path)?; + let meta = file.metadata()?; + if !meta.is_file() || meta.len()>128*1024*1024 { return Err(Error::UnsafePath); } + #[cfg(unix)] { + use std::os::unix::fs::MetadataExt; + if ![0,unsafe {libc::geteuid()}].contains(&meta.uid()) || meta.mode() & 0o022 != 0 { return Err(Error::UnsafePath); } + } + let mut bytes=Vec::new(); file.by_ref().take(128*1024*1024+1).read_to_end(&mut bytes)?; + if bytes.len()>128*1024*1024 { return Err(Error::TooLarge); } + Ok(bytes) +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Suite { + schema: String, + family: String, + model: String, + runtime: String, + interpreter: String, + runner: String, + verifier: String, + timeout_seconds: u64, + cases: Vec, + #[serde(default)] answer_markers: Vec, +} +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct Case { id: String, split: String, fixture: String } + +fn relative(s: &str) -> bool { + !s.is_empty() && s.len() <= 240 && Path::new(s).components().all(|c|matches!(c,std::path::Component::Normal(_))) +} +fn collect(root: &Path, path: &Path, out: &mut BTreeMap>, total: &mut usize) -> Result<()> { + if path.components().count().saturating_sub(root.components().count()) > 32 { return Err(Error::TooLarge); } + let mut names = files::directory_names(path, 512)?; + names.sort(); + for name in names { + let p = path.join(name); + let meta = std::fs::symlink_metadata(&p)?; + if meta.file_type().is_symlink() { return Err(Error::UnsafePath); } + if meta.is_dir() { collect(root,&p,out,total)?; } + else if meta.is_file() { + if out.len() >= 512 { return Err(Error::TooLarge); } + let bytes = files::read_regular(&p,TREE_CAP.saturating_sub(*total))?; + *total += bytes.len(); + if *total > TREE_CAP { return Err(Error::TooLarge); } + let name = p.strip_prefix(root).map_err(|_|Error::UnsafePath)?.to_str().ok_or(Error::UnsafePath)?.to_owned(); + out.insert(name,bytes); + } else { return Err(Error::UnsafePath); } + } + Ok(()) +} +fn tree_hash(tree: &BTreeMap>) -> Result { + let hashes: BTreeMap<_,_> = tree.iter().map(|(p,b)|(p,crate::digest(b))).collect(); + Ok(crate::digest(&serde_json::to_vec(&hashes).map_err(|_|Error::InvalidRequest)?)) +} +fn fixture(tree: &BTreeMap>, prefix: &str) -> Result>> { + let prefix = format!("{prefix}/"); + let part: BTreeMap<_,_> = tree.iter().filter_map(|(k,v)|k.strip_prefix(&prefix).map(|k|(k.to_owned(),v.clone()))).collect(); + if part.is_empty() { return Err(Error::InvalidRequest); } + Ok(part) +} +fn materialize(root: &Path, tree: &BTreeMap>) -> Result<()> { + for (name,bytes) in tree { files::atomic_write(&root.join(name),bytes)?; } + Ok(()) +} +fn normalize(s: &str) -> String { s.split_whitespace().collect::>().join(" ").to_lowercase() } + +fn quality(baseline: &str, candidate: &str, markers: &[String]) -> Value { + // The absolute cap also bounds newly created skills (empty baseline). + let oversized = candidate.len() > 16*1024; + let excessive_growth = !baseline.is_empty() && candidate.len() > baseline.len().saturating_mul(2) + && candidate.len().saturating_sub(baseline.len()) > 4096; + let normalized = normalize(candidate); + let copied_answer = markers.iter().any(|m|normalized.contains(&normalize(m))); + json!({"passed":!oversized && !excessive_growth && !copied_answer, + "baseline_bytes":baseline.len(),"candidate_bytes":candidate.len(), + "max_bytes":16*1024,"growth_allowance_bytes":4096,"growth_ratio":2, + "oversized":oversized,"excessive_growth":excessive_growth,"answer_marker_match":copied_answer, + "coverage":"size and operator-supplied exact answer markers; semantic generalization is unverified"}) +} + +/// No shell, inherited credentials or caller HOME. The selected suite is trusted +/// executable code, not an adversarial sandbox. Rehash the interpreter per run. +fn execute(interpreter: &Path, interpreter_hash: &str, script: &Path, workspace: &Path, + skill: &Path, home: &Path, case: &Case, suite: &Suite, log: &Path) -> Result { + if crate::digest(&interpreter_bytes(interpreter)?) != interpreter_hash { return Err(Error::Changed); } + let stdout = log.with_extension("stdout"); + let stderr = log.with_extension("stderr"); + let out = std::fs::OpenOptions::new().write(true).create_new(true).open(&stdout)?; + let err = std::fs::OpenOptions::new().write(true).create_new(true).open(&stderr)?; + let mut command = Command::new(interpreter); + command.arg(script).current_dir(workspace).env_clear() + .env("PATH","/usr/bin:/bin").env("HOME",home).env("TMPDIR",home) + .env("LORE_EVAL_SKILL",skill).env("LORE_EVAL_TASK_ID",&case.id) + .env("LORE_EVAL_MODEL",&suite.model).env("LORE_EVAL_RUNTIME",&suite.runtime) + .stdin(Stdio::null()).stdout(Stdio::from(out)).stderr(Stdio::from(err)); + #[cfg(unix)] { + use std::os::unix::process::CommandExt; + unsafe { command.pre_exec(|| { if libc::setsid() < 0 { return Err(std::io::Error::last_os_error()); } Ok(()) }); } + } + let start = Instant::now(); + let mut child = command.spawn()?; + let mut reason = "exit"; + let status = loop { + if stdout.metadata()?.len() > OUTPUT_CAP || stderr.metadata()?.len() > OUTPUT_CAP { + reason = "output_limit"; break None; + } + if let Some(status) = child.try_wait()? { break Some(status); } + if start.elapsed() >= Duration::from_secs(suite.timeout_seconds) { reason = "timeout"; break None; } + std::thread::sleep(Duration::from_millis(20)); + }; + // Kill descendants even when the direct child exits successfully. + #[cfg(unix)] unsafe { libc::kill(-(child.id() as i32),libc::SIGKILL); } + if status.is_none() { let _ = child.kill(); } + let _ = child.wait(); + let read_log = |p:&Path| -> Result { + use std::io::Read; + let mut b = Vec::new(); + let mut options = std::fs::OpenOptions::new(); options.read(true); + #[cfg(unix)] { use std::os::unix::fs::OpenOptionsExt; options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK); } + let file=options.open(p)?; + if !file.metadata()?.is_file() { return Err(Error::UnsafePath); } + file.take(OUTPUT_CAP).read_to_end(&mut b)?; + crate::scrub::scrub(&String::from_utf8_lossy(&b)) + }; + // Store hashes and short scrubbed excerpts; raw logs die with the workspace. + let out = read_log(&stdout)?; let err = read_log(&stderr)?; + Ok(json!({"passed":reason=="exit" && status.is_some_and(|s|s.success()), + "exit_code":status.and_then(|s|s.code()),"termination":reason, + "stdout_sha256":crate::digest(out.as_bytes()),"stderr_sha256":crate::digest(err.as_bytes()), + "stdout_excerpt":out.chars().take(1500).collect::(), + "stderr_excerpt":err.chars().take(1500).collect::()})) +} + +fn run_case(cfg: &Config, base: &Path, suite: &Suite, tree: &BTreeMap>, + interpreter: &Path, interpreter_hash: &str, case: &Case, arm: &str, skill_text: &str) -> Result { + let dir = base.join(format!("{}-{arm}",case.id)); + let workspace = dir.join("workspace"); + materialize(&workspace,&fixture(tree,&case.fixture)?)?; + let skill = dir.join("skill.md"); + files::atomic_write(&skill,skill_text.as_bytes())?; + let home = dir.join("home"); files::private_dir(&home)?; + let runner = dir.join("runner"); files::atomic_write(&runner,&tree[&suite.runner])?; + let run = execute(interpreter,interpreter_hash,&runner,&workspace,&skill,&home,case,suite,&dir.join("run"))?; + // Materialize verifier from frozen bytes after the candidate has executed. + let verifier = dir.join("verifier"); files::atomic_write(&verifier,&tree[&suite.verifier])?; + let verify = execute(interpreter,interpreter_hash,&verifier,&workspace,&skill,&home,case,suite,&dir.join("verify"))?; + let transcript = json!({"schema":"lore.evaluation-trace.v1","case":case.id,"arm":arm, + "skill_sha256":crate::digest(skill_text.as_bytes()),"fixture_sha256":tree_hash(&fixture(tree,&case.fixture)?)?, + "runner":run,"verifier":verify,"verification":"host_process_exit","sandboxed":false}); + let hash = diagnostics::put(cfg,"skill-evaluations/traces",&transcript)?; + Ok(json!({"case":case.id,"split":case.split,"arm":arm,"trace_sha256":hash, + "passed":run["passed"]==true && verify["passed"]==true})) +} + +pub fn evaluate(cfg: &Config, req: &Value, authority: &Authority) -> Result { + authority.require_review()?; + let review = pending::review(cfg,req)?; + let raw = review["raw"].as_str().ok_or(Error::InvalidRequest)?; + let item: Value = serde_json::from_str(raw).map_err(|_|Error::InvalidRequest)?; + if item["kind"] != "skill" || item["action"] == "retire" { return Err(Error::Unsupported); } + let name = item["name"].as_str().filter(|s|config::valid_skill_name(s)).ok_or(Error::InvalidRequest)?; + let body = crate::scrub::scrub(item["body"].as_str().ok_or(Error::InvalidRequest)?)?; + let description = item["description"].as_str().map(crate::scrub::scrub).transpose()?; + let candidate = pending::skill_file_text(name,description.as_deref(),&body); + let baseline_path = cfg.skills.join(name).join("SKILL.md"); + let baseline = if baseline_path.try_exists()? { + String::from_utf8(files::read_regular(&baseline_path,crate::MAX_REVIEW_BYTES)?).map_err(|_|Error::InvalidRequest)? + } else { String::new() }; + if crate::scrub::scrub(&baseline)? != baseline { return Err(Error::Untrusted); } + let manifest = Path::new(req["suite"].as_str().ok_or(Error::InvalidRequest)?); + let suite_bytes = files::read_regular(manifest,crate::MAX_REVIEW_BYTES)?; + let suite: Suite = serde_json::from_slice(&suite_bytes).map_err(|_|Error::InvalidRequest)?; + if suite.schema != "lore.skill-suite.v1" || suite.family != "coding" + || suite.model.trim().is_empty() || suite.runtime.trim().is_empty() + || suite.model.len()>256 || suite.runtime.len()>256 || !Path::new(&suite.interpreter).is_absolute() + || !(1..=120).contains(&suite.timeout_seconds) || suite.cases.len()>64 + || !relative(&suite.runner) || !relative(&suite.verifier) + || suite.answer_markers.len()>64 + || suite.answer_markers.iter().any(|m| normalize(m).len()<24 || m.len()>4096) { return Err(Error::InvalidRequest); } + let mut tree = BTreeMap::new(); let mut total = 0; + let root = manifest.parent().ok_or(Error::UnsafePath)?; + collect(root,root,&mut tree,&mut total)?; + if !tree.contains_key(&suite.runner) || !tree.contains_key(&suite.verifier) { return Err(Error::InvalidRequest); } + if files::read_regular(manifest,crate::MAX_REVIEW_BYTES)? != suite_bytes { return Err(Error::Changed); } + let suite_hash = tree_hash(&tree)?; + let manifest_object=json!({"schema":"lore.skill-suite-snapshot.v1", + "suite_sha256":suite_hash,"manifest_sha256":crate::digest(&suite_bytes), + "files":tree.iter().map(|(p,b)|(p.clone(),json!({"sha256":crate::digest(b),"bytes":b.len()}))).collect::>()}); + let suite_object=diagnostics::put(cfg,"skill-evaluations/suites",&manifest_object)?; + let interpreter = std::fs::canonicalize(PathBuf::from(&suite.interpreter))?; + let interpreter_hash = crate::digest(&interpreter_bytes(&interpreter)?); + let mut ids = BTreeSet::new(); let mut splits = BTreeSet::new(); let mut fixtures = BTreeMap::new(); + for case in &suite.cases { + if !config::valid_id(&case.id) || !ids.insert(&case.id) || !relative(&case.fixture) + || !matches!(case.split.as_str(),"train"|"replay"|"validation"|"test") { return Err(Error::InvalidRequest); } + let hash = tree_hash(&fixture(&tree,&case.fixture)?)?; + if let Some(other) = fixtures.insert(hash,case.split.clone()) { + if other != case.split { return Err(Error::InvalidRequest); } + } + splits.insert(case.split.as_str()); + } + if splits.len()!=4 { return Err(Error::InvalidRequest); } + let mut holdout_fixtures=Vec::new(); + for case in suite.cases.iter().filter(|c|c.split=="test") { + holdout_fixtures.push(tree_hash(&fixture(&tree,&case.fixture)?)?); + } + holdout_fixtures.sort(); + // Changing a prompt, model label or runner cannot reopen the same final tests. + let holdout_hash=crate::digest(&serde_json::to_vec(&json!({"verifier":crate::digest(&tree[&suite.verifier]), + "fixtures":holdout_fixtures})).map_err(|_|Error::InvalidRequest)?); + let quality = quality(&baseline,&candidate,&suite.answer_markers); + let object = json!({"schema":"lore.skill-candidate.v1","name":name,"proposal_sha256":review["sha256"], + "baseline":baseline,"candidate":candidate}); + let candidate_hash = diagnostics::put(cfg,"skill-evaluations/candidates",&object)?; + let mut report = json!({"schema":"lore.skill-evaluation.v1","at":crate::utcnow(),"pid":review["pid"], + "proposal_sha256":review["sha256"],"baseline_sha256":crate::digest(baseline.as_bytes()), + "candidate_sha256":crate::digest(candidate.as_bytes()),"candidate_object":candidate_hash, + "suite_sha256":suite_hash,"suite_object":suite_object,"holdout_sha256":holdout_hash,"model":suite.model,"runtime":suite.runtime, + "interpreter_sha256":interpreter_hash,"quality":quality,"results":[], + "status":"quality_failed","advisory":true,"final_test":"not_run","sandboxed":false}); + if quality["passed"] == true { + let temp = cfg.root.join("skill-evaluations/work"); files::private_dir(&temp)?; + let work = tempfile::Builder::new().prefix("evaluation-").tempdir_in(&temp)?; + let mut rows = Vec::new(); + for case in suite.cases.iter().filter(|c|c.split!="test") { + for (arm,text) in [("baseline",baseline.as_str()),("candidate",candidate.as_str())] { + rows.push(run_case(cfg,work.path(),&suite,&tree,&interpreter,&interpreter_hash,case,arm,text)?); + } + } + let mut passed = true; let mut scores = serde_json::Map::new(); + for split in ["train","replay","validation"] { + let score = |arm:&str|rows.iter().filter(|r|r["split"]==split && r["arm"]==arm && r["passed"]==true).count(); + let b = score("baseline"); let c = score("candidate"); + passed &= c>=b; + if split=="replay" { passed &= rows.iter().filter(|r|r["split"]==split && r["arm"]=="baseline" && r["passed"]==true) + .all(|r|rows.iter().any(|s|s["case"]==r["case"] && s["arm"]=="candidate" && s["passed"]==true)); } + scores.insert(split.into(),json!({"baseline_passed":b,"candidate_passed":c, + "total":suite.cases.iter().filter(|c|c.split==split).count()})); + } + // At least one training improvement is required; ties alone cannot + // justify growing a skill. Replay preserves each earlier success. + passed &= scores["train"]["candidate_passed"].as_u64()>scores["train"]["baseline_passed"].as_u64(); + report["scores"] = json!(scores); + report["status"] = json!(if passed {"validation_passed"} else {"performance_failed"}); + if passed { + let holdout = cfg.root.join("skill-evaluations/holdouts").join(format!("{holdout_hash}.json")); + let _lock = files::Locks::acquire(&cfg.root,std::slice::from_ref(&holdout),cfg.timeout)?; + if holdout.try_exists()? { report["final_test"] = json!("already_consumed"); } + else { + // Consume before running: crashes and interruptions cannot silently + // reuse the final test to tune a later candidate. + files::atomic_write(&holdout,&serde_json::to_vec(&json!({"candidate_object":candidate_hash})).map_err(|_|Error::InvalidRequest)?)?; + let mut final_passed = true; + for case in suite.cases.iter().filter(|c|c.split=="test") { + for (arm,text) in [("baseline",baseline.as_str()),("candidate",candidate.as_str())] { + let row = run_case(cfg,work.path(),&suite,&tree,&interpreter,&interpreter_hash,case,arm,text)?; + if arm=="candidate" { final_passed &= row["passed"]==true; } + rows.push(row); + } + } + report["final_test"] = json!(if final_passed {"passed"} else {"failed"}); + report["status"] = json!(if final_passed {"passed"} else {"final_test_failed"}); + } + } + report["results"] = json!(rows); + } + // Recheck optimistic proposal and baseline identity before attaching results. + let now = pending::review(cfg,req)?; + if now["sha256"] != review["sha256"] { return Err(Error::Changed); } + let current = if baseline_path.try_exists()? { files::read_regular(&baseline_path,crate::MAX_REVIEW_BYTES)? } else { vec![] }; + if current != baseline.as_bytes() { return Err(Error::Changed); } + let hash = diagnostics::put(cfg,"skill-evaluations/reports",&report)?; + let pid = review["pid"].as_str().ok_or(Error::InvalidRequest)?; + files::atomic_write(&cfg.root.join("skill-evaluations/proposals").join(format!("{pid}.json")), + &serde_json::to_vec(&json!({"report_sha256":hash})).map_err(|_|Error::InvalidRequest)?)?; + Ok(json!({"report_sha256":hash,"status":report["status"],"quality":report["quality"], + "scores":report["scores"],"final_test":report["final_test"],"advisory":true})) +} + +pub fn show(cfg: &Config, req: &Value) -> Result { + diagnostics::get(cfg,"skill-evaluations/reports",req["sha256"].as_str().ok_or(Error::InvalidRequest)?) +} + +pub fn artifact(cfg: &Config, req: &Value) -> Result { + let area=match req["kind"].as_str() { + Some("candidate")=>"skill-evaluations/candidates", + Some("trace")=>"skill-evaluations/traces", + Some("suite")=>"skill-evaluations/suites", + _=>return Err(Error::InvalidRequest), + }; + diagnostics::get(cfg,area,req["sha256"].as_str().ok_or(Error::InvalidRequest)?) +} + +/// Human views label missing, stale, or corrupted evidence rather than trusting +/// a model-provided `evaluation` field in the proposal itself. +pub fn summary(cfg: &Config, pid: &str, proposal_hash: &str, item: &Value) -> Value { + let read = || -> Result { + let path = cfg.root.join("skill-evaluations/proposals").join(format!("{pid}.json")); + if !path.try_exists()? { return Ok(json!({"status":"not_evaluated"})); } + let pointer: Value = serde_json::from_slice(&files::read_regular(&path,4096)?).map_err(|_|Error::InvalidRequest)?; + let hash = pointer["report_sha256"].as_str().ok_or(Error::InvalidRequest)?; + let report = diagnostics::get(cfg,"skill-evaluations/reports",hash)?; + let name = item["name"].as_str().filter(|s|config::valid_skill_name(s)).ok_or(Error::InvalidRequest)?; + let baseline = cfg.skills.join(name).join("SKILL.md"); + let baseline = if baseline.try_exists()? { files::read_regular(&baseline,crate::MAX_REVIEW_BYTES)? } else {vec![]}; + Ok(json!({"status":report["status"],"report_sha256":hash, + "current":report["proposal_sha256"]==proposal_hash && report["baseline_sha256"]==crate::digest(&baseline), + "binding":"proposal_and_installed_baseline", + "quality":report["quality"],"scores":report["scores"],"final_test":report["final_test"],"advisory":true})) + }; + read().unwrap_or_else(|e|json!({"status":"evidence_unavailable","error":e.code()})) +} + +#[cfg(test)] +mod tests { + use super::*; + fn human() -> Authority { Authority::HumanReview {agent:"test".into(),engine:"unknown".into()} } + fn setup(body: &str) -> (tempfile::TempDir,Config,Value) { + let dir=tempfile::tempdir().unwrap(); + let cfg=Config::for_root(dir.path().join("store")); + files::atomic_write(&cfg.skills.join("sort-values/SKILL.md"),b"Sort integers.").unwrap(); + let suite_root=dir.path().join("suite"); + let source=std::fs::canonicalize(Path::new(env!("CARGO_MANIFEST_DIR")).join("../../examples/skill-evaluation")).unwrap(); + let mut tree=BTreeMap::new(); collect(&source,&source,&mut tree,&mut 0).unwrap(); + materialize(&suite_root,&tree).unwrap(); + let cwd=dir.path().join("project"); + let pid=crate::gate::stage(&cfg,&json!({"kind":"skill","name":"sort-values","action":"update", + "body":body,"project":config::project_slug(&cwd)}),&human()).unwrap(); + let req=json!({"pid":pid,"cwd":cwd,"suite":suite_root.join("suite.json")}); + (dir,cfg,req) + } + #[test] + fn measured_candidate_keeps_human_gate_and_consumes_final_test_once() { + let (_dir,cfg,req)=setup("Deduplicate before sorting."); + let result=evaluate(&cfg,&req,&human()).unwrap(); + assert_eq!(result["status"],"passed"); + assert_eq!(result["scores"]["train"]["baseline_passed"],0); + assert_eq!(result["scores"]["train"]["candidate_passed"],1); + assert_eq!(files::read_regular(&cfg.skills.join("sort-values/SKILL.md"),1024).unwrap(),b"Sort integers."); + let review=pending::review(&cfg,&req).unwrap(); + assert_eq!(review["skill_evaluation"]["current"],true); + let manifest=Path::new(req["suite"].as_str().unwrap()); + let mut suite:Value=serde_json::from_slice(&std::fs::read(manifest).unwrap()).unwrap(); + suite["model"]=json!("another-model-label"); suite["cases"][3]["id"]=json!("renamed-final-test"); + files::atomic_write(manifest,&serde_json::to_vec(&suite).unwrap()).unwrap(); + let result=evaluate(&cfg,&req,&human()).unwrap(); + assert_eq!(result["status"],"validation_passed"); + assert_eq!(result["final_test"],"already_consumed"); + files::atomic_write(&cfg.skills.join("sort-values/SKILL.md"),b"Changed installed baseline.").unwrap(); + assert_eq!(pending::review(&cfg,&req).unwrap()["skill_evaluation"]["current"],false); + } + #[test] + fn individual_replay_regression_fails_and_leaves_final_test_untouched() { + let (_dir,cfg,req)=setup("Deduplicate before sorting. Break replay."); + let result=evaluate(&cfg,&req,&human()).unwrap(); + assert_eq!(result["status"],"performance_failed"); + assert_eq!(result["final_test"],"not_run"); + assert!(!cfg.root.join("skill-evaluations/holdouts").exists()); + } + #[test] + fn failed_runner_cannot_claim_a_verifier_pass() { + let (_dir,cfg,req)=setup("Deduplicate before sorting."); + let runner=Path::new(req["suite"].as_str().unwrap()).parent().unwrap().join("runner.py"); + let mut script=std::fs::read_to_string(&runner).unwrap(); script.push_str("\nraise SystemExit(7)\n"); + files::atomic_write(&runner,script.as_bytes()).unwrap(); + assert_eq!(evaluate(&cfg,&req,&human()).unwrap()["status"],"performance_failed"); + } + #[test] + fn copied_answers_and_growth_are_checked_before_executing_the_suite() { + let (_dir,cfg,req)=setup("memorized task-specific answer must never appear in a reusable skill"); + let result=evaluate(&cfg,&req,&human()).unwrap(); + assert_eq!(result["status"],"quality_failed"); + assert_eq!(result["quality"]["answer_marker_match"],true); + assert_eq!(quality("old","new ".repeat(3000).as_str(),&[])["excessive_growth"],true); + } + #[test] + fn model_supplied_metadata_cannot_authorize_execution_or_scores() { + let (_dir,cfg,req)=setup("Deduplicate before sorting."); + let model=Authority::Model {agent:"fake".into(),engine:"codex".into(),session_id:String::new()}; + assert_eq!(evaluate(&cfg,&req,&model),Err(Error::Untrusted)); + assert!(!cfg.root.join("skill-evaluations").exists()); + } + #[test] + fn duplicate_fixtures_across_splits_are_refused() { + let (_dir,cfg,req)=setup("Deduplicate before sorting."); + let root=Path::new(req["suite"].as_str().unwrap()).parent().unwrap(); + files::atomic_write(&root.join("tasks/test/case.json"),&std::fs::read(root.join("tasks/train/case.json")).unwrap()).unwrap(); + assert_eq!(evaluate(&cfg,&req,&human()),Err(Error::InvalidRequest)); + } + #[test] + fn changed_proposal_and_tampered_report_lose_current_evidence() { + let (_dir,cfg,req)=setup("Deduplicate before sorting."); + let result=evaluate(&cfg,&req,&human()).unwrap(); + let path=cfg.root.join("pending").join(format!("{}.json",req["pid"].as_str().unwrap())); + let mut item:Value=serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + item["body"]=json!("Different candidate."); + files::atomic_write(&path,&serde_json::to_vec(&item).unwrap()).unwrap(); + assert_eq!(pending::review(&cfg,&req).unwrap()["skill_evaluation"]["current"],false); + let report=cfg.root.join("skill-evaluations/reports").join(format!("{}.json",result["report_sha256"].as_str().unwrap())); + files::atomic_write(&report,b"{}").unwrap(); + assert_eq!(pending::review(&cfg,&req).unwrap()["skill_evaluation"]["status"],"evidence_unavailable"); + } + #[test] + fn timeout_is_a_failed_host_observation() { + let (_dir,cfg,req)=setup("Deduplicate before sorting."); + let root=Path::new(req["suite"].as_str().unwrap()).parent().unwrap(); + let mut suite:Suite=serde_json::from_slice(&std::fs::read(root.join("suite.json")).unwrap()).unwrap(); + suite.timeout_seconds=1; + let script=root.join("sleep.py"); files::atomic_write(&script,b"import time\ntime.sleep(10)\n").unwrap(); + let interpreter=std::fs::canonicalize(&suite.interpreter).unwrap(); + let hash=crate::digest(&interpreter_bytes(&interpreter).unwrap()); + let row=execute(&interpreter,&hash,&script,root,&root.join("suite.json"),root,&suite.cases[0],&suite,&root.join("timeout")).unwrap(); + assert_eq!(row["passed"],false); assert_eq!(row["termination"],"timeout"); + assert!(!cfg.root.join("skill-evaluations/holdouts").exists()); + } +} diff --git a/rust/lore-core/src/standalone.rs b/rust/lore-core/src/standalone.rs index 14bb3ef..375410f 100644 --- a/rust/lore-core/src/standalone.rs +++ b/rust/lore-core/src/standalone.rs @@ -23,7 +23,11 @@ Graph: graph stats|neighbours|path|communities|components|degree|context|html|ba Codegraph: codegraph store --cwd WORKTREE --input FILE --expected-sha256 HASH --expected-revision N Sessions: search session index history recent|prefix|metadata Review: review backfill pending list|show approve reject -Skills: skills list|candidates|usage +Skills: skills list|candidates|usage|evaluate PID --suite FILE + skills evaluation SHA256 + skills artifact --kind candidate|trace|suite --sha256 SHA256 +Diagnostics: trace capture --transcript FILE --session-id ID --engine claude|codex + trace show SHA256 Administration: setup teardown reset project move config show|set|unset Sync: sync status|health|whoami|push|pull|bootstrap|serve|export|import|classes|login|resign|seed Integration: mcp hook @@ -251,7 +255,8 @@ fn validate_flags(group: &str, sub: &str, req: &Value) -> Result<()> { ("approve"|"reject",_) => "pid expected", ("consult"|"ask"|"dialectic",_) => "query limit", ("outcome",_) => "belief_id event source note", - ("skills",_) => "prompt limit", + ("skills",_) => "prompt limit pid suite sha256 kind", + ("trace",_) => "transcript session_id engine sha256", ("dream",_) => "dry_run engine", ("audit",_) => "sample", ("project", "move") => "old new dry_run", @@ -333,6 +338,7 @@ pub fn run(args: &[String]) -> Result<()> { | "config" | "pending" | "project" + | "trace" ); let (sub, tail) = if grouped && args.get(1).is_some_and(|s| !s.starts_with('-')) { (args[1].as_str(), &args[2..]) @@ -351,6 +357,11 @@ pub fn run(args: &[String]) -> Result<()> { } let mut core = Core::new(cfg.clone(), auth.clone()); let value = match (group, sub) { + ("trace", "capture") => crate::diagnostics::capture(&cfg, &req)?, + ("trace", "show") => { + pos(&mut req, "sha256", &p, false)?; + crate::diagnostics::show(&cfg, &req)? + } ("snapshot" | "inject", _) => crate::context::snapshot(&cfg, &req)?, ("refresh", _) => crate::context::refresh(&cfg, &req)?, ("memory", "show" | "entries" | "list") => { @@ -553,6 +564,15 @@ pub fn run(args: &[String]) -> Result<()> { crate::skills::candidates_op(&cfg, &req)? } ("skills", "usage") => crate::skills::load_usage(&cfg)?, + ("skills", "evaluate") => { + pos(&mut req, "pid", &p, false)?; + crate::skill_evaluation::evaluate(&cfg, &req, &auth)? + } + ("skills", "evaluation") => { + pos(&mut req, "sha256", &p, false)?; + crate::skill_evaluation::show(&cfg, &req)? + } + ("skills", "artifact") => crate::skill_evaluation::artifact(&cfg, &req)?, ("review", _) => { own_process_group()?; crate::standalone_ops::review(&cfg, &req)? diff --git a/rust/lore-core/tests/worker.rs b/rust/lore-core/tests/worker.rs index 3134793..dc9a07d 100644 --- a/rust/lore-core/tests/worker.rs +++ b/rust/lore-core/tests/worker.rs @@ -67,6 +67,9 @@ if mode == 'error': sys.exit(7) if mode == 'changed_source': Path(os.environ['OWNED_SOURCE']).write_text('changed owned source\n') +if mode == 'diagnostic': + print(json.dumps({'skills':[{'name':'owned-recipe','body':'Run the frozen tests before proposing a change.'}]})) + sys.exit(0) print(json.dumps({'memory':[{'scope':'user','action':'add','text':'Native reviewer preserves owned evidence','source_engine':'forged'}], 'filemap':[], 'skills':[], 'skill_outcomes':[], 'conclusions':[]})) "#; @@ -165,6 +168,7 @@ impl Fixture { .env("OWNED_FIXTURE", &self.home) .env("OWNED_SOURCE", &self.source) .env("OWNED_MODE", mode) + .env("LORE_DIAGNOSTIC_TRACES", if mode == "diagnostic" { "1" } else { "0" }) .current_dir(&self.cwd) .stdin(Stdio::piped()) .stdout(Stdio::from(fs::File::create(&output).unwrap())) @@ -664,3 +668,24 @@ fn persisted_disabled_worker_refuses_without_provider_invocation() { fixture.assert_not_curated(); } } + +#[test] +fn native_review_attaches_local_diagnostic_trace_without_expanding_provider_digest() { + let fixture=Fixture::new("claude",false); + let mut source=fs::read_to_string(&fixture.source).unwrap(); + source.push_str(&format!("{}\n",json!({"type":"assistant","message":{"content":[{"type":"tool_use","id":"owned-call","name":"Bash","input":{"command":"cargo test"}}]}}))); + source.push_str(&format!("{}\n",json!({"type":"user","message":{"content":[{"type":"tool_result","tool_use_id":"owned-call","content":"successful observation only in diagnostic trace","is_error":false}]}}))); + files::atomic_write(&fixture.source,source.as_bytes()).unwrap(); + let mut worker=fixture.spawn("claude","diagnostic"); + worker.observe_provider(&fixture.home); + assert!(worker.finish(Duration::from_secs(5)).unwrap().success(),"{}",worker.stderr()); + let items=fixture.pending(); assert_eq!(items.len(),1); + let hash=items[0]["diagnostic_trace_sha256"].as_str().unwrap(); + let trace=lore_core::diagnostics::show(&fixture.config(),&json!({"sha256":hash})).unwrap(); + assert!(trace["events"].as_array().unwrap().iter().any(|e|e["content"]=="successful observation only in diagnostic trace")); + assert!(!fs::read_to_string(fixture.home.join("prompt")).unwrap().contains("successful observation only")); + let cfg=fixture.config(); let pid=pending::ids(&cfg).unwrap().remove(0); + let reviewed=pending::review(&cfg,&json!({"cwd":fixture.cwd,"pid":pid})).unwrap(); + assert!(reviewed["raw"].as_str().unwrap().contains(hash)); + fixture.assert_not_curated(); +}