From a0148afbe945801411e49e30389c5271b91c0f15 Mon Sep 17 00:00:00 2001 From: David Chen <530634352@qq.com> Date: Mon, 2 Nov 2026 19:40:42 +0800 Subject: [PATCH 01/23] Add offline optimization dashboard Agent-assisted-by: Codex (implementation) Signed-off-by: David Chen <530634352@qq.com> --- docs/dashboard.md | 139 +++++++ docs/operations.md | 2 + pyproject.toml | 2 +- src/inferrsi/arguments.py | 31 ++ src/inferrsi/cli.py | 23 ++ src/inferrsi/data/dashboard/template.html | 212 +++++++++++ src/inferrsi/harness/campaign_records.py | 158 ++++++++ src/inferrsi/harness/campaign_trace.py | 40 ++ src/inferrsi/harness/dashboard.py | 161 ++++++++ src/inferrsi/harness/dashboard_data.py | 414 +++++++++++++++++++++ src/inferrsi/harness/dashboard_timeline.py | 170 +++++++++ src/inferrsi/model/anthropic_agent.py | 21 +- tests/test_campaign_records.py | 71 ++++ tests/test_campaign_trace.py | 51 +++ tests/test_codex_plugin.py | 13 + tests/test_dashboard_data.py | 366 ++++++++++++++++++ tools/build_codex_plugin.py | 1 + 17 files changed, 1873 insertions(+), 2 deletions(-) create mode 100644 docs/dashboard.md create mode 100644 src/inferrsi/data/dashboard/template.html create mode 100644 src/inferrsi/harness/campaign_records.py create mode 100644 src/inferrsi/harness/campaign_trace.py create mode 100644 src/inferrsi/harness/dashboard.py create mode 100644 src/inferrsi/harness/dashboard_data.py create mode 100644 src/inferrsi/harness/dashboard_timeline.py create mode 100644 tests/test_campaign_records.py create mode 100644 tests/test_campaign_trace.py create mode 100644 tests/test_dashboard_data.py diff --git a/docs/dashboard.md b/docs/dashboard.md new file mode 100644 index 000000000..d0fff03c8 --- /dev/null +++ b/docs/dashboard.md @@ -0,0 +1,139 @@ +# Offline optimization dashboard + +The dashboard renders a recorded InferRSI run or an explicitly assembled +campaign as a single HTML file. It needs no web service or external assets. +It is a read-only view of recorded data; generating it does not run an +experiment, capture an agent, validate performance, or accept a campaign. + +## Generate and open + +Install InferRSI as described in [quickstart](quickstart.md), then run: + +```sh +inferrsi dashboard --input --output +``` + +Omit `--output` to write `dashboard.html` in the input directory. Open the file +in a browser. Filters and review export work locally. A generated snapshot does +not update as an experiment runs; regenerate it to show new records. + +The input can be one existing run directory containing `run.lock.json`, or a +campaign containing `campaign.json`. Existing runs need no migration. Missing +events, validation, trace and other evidence are shown explicitly; no missing +metric is replaced with zero. + +## Campaign integration + +The following optional layout associates multiple runs with candidate and round +records. Create the manifest and supply your own campaign ID and optional target: + +```sh +inferrsi campaign init --directory --id \ + --target-pct +``` + +Use `inferrsi run --output-dir /runs` to produce the existing +run directories. After a run ID exists, associate it with a candidate and round: + +```sh +inferrsi campaign record --directory --candidate \ + --run-id --round --title '' +``` + +Optional `--agent-trace` and `--profile` arguments point to existing, +campaign-relative files; the recorder hashes them and does not copy them. The +caller assigns IDs and must serialize campaign record updates. Neither command +starts an agent or validates the experiment. + +Record one decision for a round with `inferrsi campaign decide --directory + --candidate --round +--outcome `. Existing decisions are not overwritten. + +```text +/ + campaign.json + rounds.jsonl + summary-chart.json # optional, explicitly recorded chain + candidates// + candidate.json + agent_trace.jsonl # optional, explicitly referenced and hashed + runs// # existing InferRSI run artifacts +``` + +`campaign.json` may include `campaign_id`, `synthetic`, +`goal.target_improvement_pct`, `noise.band_pct`, and `stages_declared`. The goal is supplied by the +campaign's acceptance issue, not a dashboard default. Mark demonstrative data +with `"synthetic": true` so the page displays its synthetic-data banner. +An undeclared stage is marked `not_applicable` when `stages_declared` is supplied; +otherwise a stage with no recorded events is marked `stage_not_recorded`. + +Each `candidate.json` may provide `candidate_id`, `title`, `round`, `run_id`, +`agent_trace_ref`, and `profile_ref`. An explicit reference is an object with a +campaign-relative `path` and the file's SHA-256 in `sha256`. Missing references +can instead specify `state: "missing"` and `missing_reason`. Paths escaping the +campaign, including through symlinks, are rejected. A missing hash or hash +mismatch is shown as missing, and its file is not read as a trace. + +`rounds.jsonl` contains one JSON object per round with an integer `round` and +may include a `decision` object with `candidate_id` and `outcome` (including +`rolled_back`). +Agent trace JSONL can contain the existing `agent.*` event types and the +optional `stage` field (`diagnosis`, `knowledge`, `search`, `execution`, +`validation`, or `review`). An agent integration can pass an opt-in +`inferrsi.harness.campaign_trace.CampaignTrace` instance as its existing event +callback; it appends the original event with time and caller-supplied IDs. The +caller associates the completed trace with `agent_trace_ref` using its SHA-256. +The callback does not allocate IDs or intercept unrelated agents. Existing runner phases are mapped to execution and +validation. The dashboard reads allowlisted event metadata; it does not include +raw agent requests, arbitrary event messages, environment data, or raw logs. +For recorded execution events it identifies expected logs and their hashes, +without embedding their contents. Trace line numbers and artifact references +can be included in a local review export. Missing agent tool-call events and +corrupt round-record line numbers are also shown explicitly. + +The existing [summary chart](campaign-summary-chart.md) renderer is reused. +An explicitly recorded, valid `summary-chart.json` with a chain layout produces +a cumulative chart. Otherwise the dashboard uses a branch chart of independent +candidate results, so it does not imply that unrelated trials accumulated. +Failed candidates without a numeric measurement remain in the table and trace. +The dashboard also plots recorded terminal events and stage events on a shared +UTC time axis. Results without a terminal timestamp are omitted from that plot; +failed runs without a measurement appear in a separate no-value row. Stage dots +mark events, not inferred durations. A common recorded noise band and target +are shown when available, with distinct quality-failure, failure and rollback +markers. Unrelated candidate measurements are not joined into a cumulative line. + +## Interpretation and limits + +- Best gain includes only completed runs with a PASS or FAIL verdict, a supplied + quality command with a recorded zero exit code, verified source integrity, + and no recorded rollback. Successful validation + covers only that command, not untested quality properties. +- Multiple measured runs are summarized together only when their declared model, + model revision, workload, scenario, backend, device IDs, baseline source + snapshot and command, comparison scope, metric, unit and direction agree. + Otherwise the aggregate and shared chart are withheld. +- Noise uses the campaign's declared band, or the larger recorded trial range + when no band is supplied. It is descriptive; statistical significance is not + established. +- Diagnostics currently cover recorded execution failures, rollbacks, noisy or + regressed measurements, and a quality failure accompanying a positive + measurement. Missing stages are not diagnosed as + successful. Profiling interpretation, automatic root-cause attribution, + automatic capture of unrelated agents and framework write-back are not implemented. +- The stage matrix describes recorded coverage. The separate time chart does not + infer durations or result timestamps when they are missing. Noise bands that + differ across candidates are not represented as one shared band. +- The review button creates a local Markdown download and displays its content + for copying if a browser blocks downloads. It does not send or publish it. + +## Privacy + +The exporter does not traverse and link the entire run tree. It reads known +records and explicit references, escapes data before embedding it in HTML, and +withholds raw logs and environment values. Common addresses, URLs, paths and +credential-shaped strings in displayed labels are replaced, but this is not a +general anonymizer. IDs, relative artifact names and review text can still +identify a person or contain secrets. Inspect and sanitize the actual HTML, +screenshots and exported Markdown before sharing them; follow +[privacy and security](privacy-security.md). diff --git a/docs/operations.md b/docs/operations.md index 9614db4f2..8c3860439 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -26,6 +26,8 @@ Effect classes used below: | Amend repository-owned knowledge | Scoped repository PR | — | Repository review and integration authority apply; this is not a runtime store Update | | Inspect/prepare experiments | CLI `doctor`; `plan` or `--dry-run` | Read-only inspection / preview | `doctor` may invoke bounded probes and does not certify target compatibility; `plan` previews a specification without persisting a campaign or authorizing execution | | Execute/inspect experiments | CLI `run`; `status`; `report` | Experiment-executing / read-only / report-generating | `run` executes caller-supplied commands and records evidence; `status` reads recorded progress; `report` generates HTML from recorded evidence | +| Visualize a recorded run or campaign | CLI `dashboard`; [offline dashboard](dashboard.md) | Report-generating | Writes a self-contained read-only HTML snapshot from recorded evidence; does not run an agent or validate a claim | +| Associate runs with a local campaign | CLI `campaign init` / `campaign record` / `campaign decide`; [offline dashboard](dashboard.md) | Local evidence-writing | Creates caller-named campaign, round and candidate records; references existing artifacts and never starts an agent | | Adapt existing Flare delivery evidence | [Offline qualification adapter](flare-qualification.md), `python -m inferrsi.model.validators.flare` | Read-only | Existing raw packets and a separate receipt; one HTTP delivery metric, no model/server/device execution | | Evaluate selected skill revisions | [Bounded skill-evaluation driver](skill-evaluation.md) | Experiment-executing | Explicitly authorized real Codex calls, independent task verdicts and private evidence; separate from numeric engine-performance scoring | | Compare skill additions or updates | [Configured skill comparison](skill-eval-comparison.md), `python -m tools.skill_eval_compare` | Read-only preparation / experiment-executing / report-generating | Fixed Git snapshots and one shared exam; explicit plan authorization for execution; JSON + Markdown regeneration makes no model calls | diff --git a/pyproject.toml b/pyproject.toml index e44a7a806..50a9c792d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,7 +23,7 @@ where = ["src"] namespaces = false [tool.setuptools.package-data] -inferrsi = ["data/**/*.json", "data/**/*.md", "data/**/LICENSE", "data/**/THIRD_PARTY_NOTICES", "data/**/scripts/*.py", "data/**/references/licenses/*.txt", "data/**/references/upstream-NOTICE"] +inferrsi = ["data/**/*.json", "data/**/*.md", "data/**/LICENSE", "data/**/THIRD_PARTY_NOTICES", "data/**/scripts/*.py", "data/**/references/licenses/*.txt", "data/**/references/upstream-NOTICE", "data/dashboard/*.html"] [tool.setuptools_scm] version_file = "src/inferrsi/_version.py" diff --git a/src/inferrsi/arguments.py b/src/inferrsi/arguments.py index 46b5e9336..f09eca283 100644 --- a/src/inferrsi/arguments.py +++ b/src/inferrsi/arguments.py @@ -111,6 +111,37 @@ def parser() -> argparse.ArgumentParser: child.add_argument("--run-id", required=True) child.add_argument("--output-dir", default="runs") + dashboard = commands.add_parser( + "dashboard", help="Export an offline dashboard for one run or recorded campaign" + ) + dashboard.add_argument("--input", type=Path, required=True, help="Run or campaign directory") + dashboard.add_argument( + "--output", type=Path, help="HTML output (default: input/dashboard.html)" + ) + + campaign = commands.add_parser("campaign", help="Create local campaign correlation records") + campaign_actions = campaign.add_subparsers(dest="campaign_action", required=True) + create = campaign_actions.add_parser("init", help="Create a new campaign manifest") + create.add_argument("--directory", type=Path, required=True) + create.add_argument("--id", required=True) + create.add_argument("--target-pct", type=float) + create.add_argument("--noise-pct", type=float) + create.add_argument("--synthetic", action="store_true") + record = campaign_actions.add_parser("record", help="Associate a candidate with one run") + record.add_argument("--directory", type=Path, required=True) + record.add_argument("--candidate", required=True) + record.add_argument("--run-id", required=True) + record.add_argument("--round", type=int, required=True) + record.add_argument("--title", required=True) + record.add_argument("--agent-trace", help="Existing campaign-relative JSONL trace") + record.add_argument("--profile", help="Existing campaign-relative profile artifact") + decision = campaign_actions.add_parser("decide", help="Record one round decision") + decision.add_argument("--directory", type=Path, required=True) + decision.add_argument("--candidate", required=True) + decision.add_argument("--round", type=int, required=True) + decision.add_argument("--outcome", required=True, + choices=("adopted", "rejected", "rolled_back", "no_gain", "noise")) + skills = commands.add_parser( "skills", help="Read explicitly selected skills; never install or execute them" ) diff --git a/src/inferrsi/cli.py b/src/inferrsi/cli.py index 349e3df34..4741ea90b 100644 --- a/src/inferrsi/cli.py +++ b/src/inferrsi/cli.py @@ -280,6 +280,29 @@ def main(argv: list[str] | None = None) -> int: _print({"report": str(write_report(directory))}) return 0 + if args.command == "dashboard": + from .harness.dashboard import write_dashboard + + _print({"dashboard": str(write_dashboard(args.input, args.output))}) + return 0 + + if args.command == "campaign": + from .harness.campaign_records import create_campaign, record_candidate, record_decision + + if args.campaign_action == "init": + path = create_campaign(args.directory, args.id, target=args.target_pct, + noise=args.noise_pct, synthetic=args.synthetic) + elif args.campaign_action == "record": + path = record_candidate(args.directory, candidate_id=args.candidate, + run_id=args.run_id, round_number=args.round, + title=args.title, agent_trace=args.agent_trace, + profile=args.profile) + else: + path = record_decision(args.directory, candidate_id=args.candidate, + round_number=args.round, outcome=args.outcome) + _print({"record": str(path)}) + return 0 + environment = inspect_environment( args.repo, args.repo_type, diff --git a/src/inferrsi/data/dashboard/template.html b/src/inferrsi/data/dashboard/template.html new file mode 100644 index 000000000..0fe2b8b9e --- /dev/null +++ b/src/inferrsi/data/dashboard/template.html @@ -0,0 +1,212 @@ + + + + + + +InferRSI · Optimization dashboard + + + +
+ + Dashboards › InferRSI › + + READ ONLY · OFFLINE +
+
+
campaign
+
+
+
+
metric
+
+
+
Overview
+
+
Workflow Select a stage to filter candidates and trace
+
+
Performance and evidence Recorded values, including unsuccessful attempts
+
+
+

Performance trend

+
__DASHBOARD_CHART__
+

Results and stages over time

+
__DASHBOARD_TEMPORAL__
+

+

Stage coverage

Recorded events; not a time axis
+
+

Candidate details

Select a row to inspect its trace
+
RoundCandidateRunGainQualityOutcomeEvidence
+
+
+
+

Diagnosis

Only rules backed by referenced evidence
+

Trace

+

Evidence completeness

Missing records are explicit
+

Human review

Local Markdown export
+
+ + + +

Review the exported text before sharing. Common private values are redacted; free text can still contain sensitive information.

+
+
+

A successful quality command covers only that command. This view does not establish statistical significance. Raw logs and environment values are withheld; references and SHA-256 identify recorded artifacts.

+
+ + + + diff --git a/src/inferrsi/harness/campaign_records.py b/src/inferrsi/harness/campaign_records.py new file mode 100644 index 000000000..e3836b827 --- /dev/null +++ b/src/inferrsi/harness/campaign_records.py @@ -0,0 +1,158 @@ +"""Create a local campaign manifest and associate completed or live runs.""" + +from __future__ import annotations + +import json +import math +import os +import re +import uuid +from pathlib import Path + +from .dashboard_data import _file_ref, _inside, _object + + +def _identifier(value: str, name: str) -> str: + if (not isinstance(value, str) or not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}", value) + or value in {".", ".."}): + raise ValueError(f"{name} must be a safe single identifier") + return value + + +def create_campaign(root: Path, campaign_id: str, *, target: float | None = None, + noise: float | None = None, synthetic: bool = False) -> Path: + _identifier(campaign_id, "campaign_id") + for name, value in (("target", target), ("noise", noise)): + if value is not None and (isinstance(value, bool) or not isinstance(value, (int, float)) + or not math.isfinite(value) or value < 0): + raise ValueError(f"{name} must be finite and nonnegative") + root = Path(root).resolve() + root.mkdir(parents=True, exist_ok=True) + manifest = {"campaign_id": campaign_id, "synthetic": synthetic} + if target is not None: + manifest["goal"] = {"target_improvement_pct": target} + if noise is not None: + manifest["noise"] = {"band_pct": noise} + path = root / "campaign.json" + with path.open("x", encoding="utf-8") as stream: + stream.write(json.dumps(manifest, ensure_ascii=False, indent=2, allow_nan=False) + "\n") + return path + + +def _round_records(path: Path) -> list[dict]: + try: + lines = path.read_text(encoding="utf-8").splitlines() + except FileNotFoundError: + return [] + records = [] + for line in lines: + try: + item = json.loads(line) + except json.JSONDecodeError as error: + raise ValueError("existing rounds.jsonl contains invalid JSON") from error + if (not isinstance(item, dict) or isinstance(item.get("round"), bool) + or not isinstance(item.get("round"), int) + or not isinstance(item.get("candidates"), list)): + raise ValueError("existing rounds.jsonl contains an invalid round") + records.append(item) + if len({item["round"] for item in records}) != len(records): + raise ValueError("existing rounds.jsonl repeats a round") + return records + + +def _reference(root: Path, raw: str | None, kind: str) -> dict | None: + if raw is None: + return None + path = Path(raw) + if path.is_absolute() or ".." in path.parts: + raise ValueError(f"{kind} must be a campaign-relative path") + ref = _file_ref(root, root / path, kind) + if ref["state"] != "present": + raise ValueError(f"{kind} is missing or outside the campaign") + return ref + + +def _write_rounds(root: Path, path: Path, records: list[dict]) -> None: + temporary = root / f".rounds-{os.getpid()}-{uuid.uuid4().hex}.tmp" + try: + temporary.write_text("".join(json.dumps(item, ensure_ascii=False, allow_nan=False) + "\n" + for item in records), encoding="utf-8") + temporary.replace(path) + finally: + if temporary.exists(): + temporary.unlink() + + +def record_candidate(root: Path, *, candidate_id: str, run_id: str, round_number: int, + title: str, agent_trace: str | None = None, + profile: str | None = None) -> Path: + """Associate a run and optional explicit evidence without copying raw data.""" + root = Path(root).resolve() + _identifier(candidate_id, "candidate_id") + _identifier(run_id, "run_id") + if isinstance(round_number, bool) or not isinstance(round_number, int) or round_number < 1: + raise ValueError("round must be a positive integer") + if not isinstance(title, str) or not title.strip(): + raise ValueError("title is required") + _, state = _object(root / "campaign.json", root) + if state != "present": + raise ValueError("campaign.json is missing or invalid") + run = root / "runs" / run_id + if not _inside(root, run / "run.lock.json") or not (run / "run.lock.json").is_file(): + raise ValueError("run must be an existing InferRSI run inside the campaign") + candidate_dir = root / "candidates" / candidate_id + if not _inside(root, candidate_dir): + raise ValueError("candidate directory escapes the campaign") + metadata = {"candidate_id": candidate_id, "run_id": run_id, + "round": round_number, "title": title} + for key, value in (("agent_trace_ref", agent_trace), ("profile_ref", profile)): + ref = _reference(root, value, key) + if ref is not None: + metadata[key] = ref + rounds_path = root / "rounds.jsonl" + if not _inside(root, rounds_path): + raise ValueError("rounds file escapes the campaign") + records = _round_records(rounds_path) + matching = next((item for item in records if item["round"] == round_number), None) + if matching is None: + matching = {"round": round_number, "candidates": []} + records.append(matching) + if candidate_id in matching["candidates"]: + raise ValueError("candidate is already recorded in this round") + matching["candidates"].append(candidate_id) + candidate_dir.mkdir(parents=True, exist_ok=True) + target = candidate_dir / "candidate.json" + payload = json.dumps(metadata, ensure_ascii=False, indent=2, allow_nan=False) + "\n" + with target.open("x", encoding="utf-8") as stream: + stream.write(payload) + try: + _write_rounds(root, rounds_path, records) + except (OSError, ValueError): + if target.exists() and target.read_text(encoding="utf-8") == payload: + target.unlink() + raise + return target + + +def record_decision(root: Path, *, candidate_id: str, round_number: int, outcome: str) -> Path: + """Record one round decision without overwriting an earlier one.""" + root = Path(root).resolve() + _identifier(candidate_id, "candidate_id") + if isinstance(round_number, bool) or not isinstance(round_number, int) or round_number < 1: + raise ValueError("round must be a positive integer") + if not isinstance(outcome, str) or outcome not in { + "adopted", "rejected", "rolled_back", "no_gain", "noise" + }: + raise ValueError("unknown decision outcome") + path = root / "rounds.jsonl" + if not _inside(root, path): + raise ValueError("rounds file escapes the campaign") + records = _round_records(path) + matching = next((item for item in records if item["round"] == round_number), None) + if matching is None or candidate_id not in matching["candidates"]: + raise ValueError("candidate is not recorded in this round") + if "decision" in matching: + raise ValueError("round already has a decision") + matching["decision"] = {"candidate_id": candidate_id, "outcome": outcome} + _write_rounds(root, path, records) + return path diff --git a/src/inferrsi/harness/campaign_trace.py b/src/inferrsi/harness/campaign_trace.py new file mode 100644 index 000000000..72ec836ff --- /dev/null +++ b/src/inferrsi/harness/campaign_trace.py @@ -0,0 +1,40 @@ +"""Opt-in correlation envelope for existing agent event callbacks.""" + +from __future__ import annotations + +import json +from datetime import datetime, timezone +from pathlib import Path + +from .dashboard_data import STAGES + + +class CampaignTrace: + """Append original events with correlation fields; callers own their IDs.""" + + def __init__(self, path: Path, *, campaign_id: str, candidate_id: str, stage: str, + run_id: str | None = None, round_number: int | None = None): + if (not isinstance(campaign_id, str) or not campaign_id + or not isinstance(candidate_id, str) or not candidate_id or stage not in STAGES): + raise ValueError("campaign, candidate and a known stage are required") + if run_id is not None and (not isinstance(run_id, str) or not run_id): + raise ValueError("run_id must be a nonempty string when supplied") + if round_number is not None and ( + isinstance(round_number, bool) or not isinstance(round_number, int) or round_number < 1 + ): + raise ValueError("round_number must be a positive integer") + self.path = Path(path) + self.context = {"schema_version": 1, "campaign_id": campaign_id, + "candidate_id": candidate_id, "stage": stage, + "run_id": run_id, "round": round_number} + + def __call__(self, event: dict) -> None: + if not isinstance(event, dict) or not isinstance(event.get("type"), str): + raise ValueError("an event object with a type is required") + if set(event) & (set(self.context) | {"time"}): + raise ValueError("event already contains correlation fields") + record = {**event, **self.context, "time": datetime.now(timezone.utc).isoformat()} + line = json.dumps(record, ensure_ascii=False, allow_nan=False) + "\n" + self.path.parent.mkdir(parents=True, exist_ok=True) + with self.path.open("a", encoding="utf-8") as stream: + stream.write(line) diff --git a/src/inferrsi/harness/dashboard.py b/src/inferrsi/harness/dashboard.py new file mode 100644 index 000000000..7db499209 --- /dev/null +++ b/src/inferrsi/harness/dashboard.py @@ -0,0 +1,161 @@ +"""Export a self-contained, read-only dashboard from recorded evidence.""" + +from __future__ import annotations + +import json +import math +import re +from importlib.resources import files +from pathlib import Path + +from ..summary_chart import SpecError, build_svg +from .dashboard_data import _inside, _number, _text, load_dashboard +from .dashboard_timeline import temporal_chart + + +def _sanitize_spec(value: object, depth: int = 0) -> object: + if depth > 12: + raise ValueError("chart specification is too deeply nested") + if isinstance(value, str): + return _text(value, "") + if isinstance(value, list): + return [_sanitize_spec(item, depth + 1) for item in value[:100]] + if isinstance(value, dict): + return {key: _sanitize_spec(item, depth + 1) for key, item in value.items() + if isinstance(key, str)} + return value + + +def _bounded_chain(spec: dict) -> bool: + """Keep malformed or adversarial chart axes from expanding without bound.""" + metrics, points = spec.get("metrics"), spec.get("points") + baseline = spec.get("baseline") + if not isinstance(metrics, list) or not 1 <= len(metrics) <= 7: + return False + if not isinstance(points, list) or not 1 <= len(points) <= 100: + return False + if not isinstance(baseline, dict) or not isinstance(baseline.get("metrics"), dict): + return False + keys = [item.get("key") if isinstance(item, dict) else None for item in metrics] + if any(not isinstance(key, str) or not key for key in keys): + return False + bases = {key: _number(baseline["metrics"].get(key)) for key in keys} + if any(value is None or value <= 0 for value in bases.values()): + return False + ratios = [1.0] + for point in points: + values = point.get("values") if isinstance(point, dict) else None + if not isinstance(values, dict): + return False + for key in keys: + number = _number(values.get(key)) + if number is None or number <= 0: + return False + ratio = number / bases[key] + if not 0 < ratio <= 1_000_000: + return False + ratios.append(ratio) + references = spec.get("references", []) + if not isinstance(references, list): + return False + for ref in references: + if (not isinstance(ref, dict) or not isinstance(ref.get("metric"), str) + or ref["metric"] not in bases): + return False + number = _number(ref.get("value")) + if number is None or not 0 < number / bases[ref["metric"]] <= 1_000_000: + return False + ratios.append(number / bases[ref["metric"]]) + axis = spec.get("axis", {}) + if not isinstance(axis, dict): + return False + step = _number(axis.get("step", 0.5)) + return step is not None and step > 0 and (max(ratios) - min(ratios)) / step <= 150 + + +def _chart(source: Path, view: dict) -> tuple[str, str]: + chart_file = source / "summary-chart.json" + spec = None + message = "Independent candidate results relative to their paired baselines." + if _inside(source, chart_file) and chart_file.is_file(): + try: + if chart_file.stat().st_size <= 256 * 1024: + recorded = json.loads(chart_file.read_text(encoding="utf-8")) + if (isinstance(recorded, dict) and recorded.get("layout", "chain") == "chain" + and _bounded_chain(recorded)): + spec = _sanitize_spec(recorded) + message = "Recorded cumulative campaign steps." + except (OSError, UnicodeError, json.JSONDecodeError, ValueError, RecursionError): + pass + if spec is None: + if view["overview"]["comparison_state"] != "comparable": + return "", "Candidate workloads or metrics cannot be compared." + trials = [] + for index, row in enumerate(view["candidates"], 1): + gain = row["improvement_pct"] + if gain is None or 1 + gain / 100 <= 0: + continue + status = {"improved": "gain", "noise": "flat", "regressed": "regressed"}.get( + row["outcome"], "open" + ) + trials.append({"group": "Candidates", "label": f'{index}. {row["title"]}', + "values": {"gain": 1 + gain / 100}, "status": status, + "note": row["outcome"]}) + if not trials: + return "", "No measured candidate results recorded." + target = view["overview"]["target_improvement_pct"] + gains = [row["improvement_pct"] for row in view["candidates"] + if row["improvement_pct"] is not None and 1 + row["improvement_pct"] / 100 > 0] + if target is not None and 1 + target / 100 > 0: + gains.append(target) + spread = max([0.0, *gains]) - min([0.0, *gains]) + if not math.isfinite(spread): + return "", "Recorded values exceed the supported chart scale." + tick = 0.01 if spread < 0.05 else 10 ** math.floor(math.log10(spread / 5)) + spec = { + "title": "Candidate gain versus paired baseline", "layout": "branch", "sort": "given", + "baseline": {"label": "paired baseline", "metrics": {"gain": 1}}, + "metrics": [{"key": "gain", "label": "gain"}], "trials": trials, + "axis": {"tick": tick}, + "references": [{"label": "Target", "metric": "gain", "value": 1 + target / 100}] + if target is not None and 1 + target / 100 > 0 else [], + } + try: + svg = build_svg(spec) + except (SpecError, TypeError, ValueError, OverflowError): + return "", "The recorded chart specification is invalid or incomplete." + # The shared renderer is unchanged. Adapt its fixed light palette for this panel. + for light, dark in { + "#fcfcfb": "#181b1f", "#0b0b0b": "#e7e9ed", "#52514e": "#aab2bf", + "#8a8984": "#8e98a6", "#f1f1ee": "#20242c", "#e4e3df": "#343a45", + }.items(): + svg = svg.replace(light, dark) + return svg[svg.find(" Path: + """Write one offline HTML document, excluding raw logs and environment data.""" + source = Path(source).resolve() + view = load_dashboard(source) + chart, chart_note = _chart(source, view) + temporal, temporal_note = temporal_chart(view) + view["chart_note"] = chart_note + view["temporal_note"] = temporal_note + payload = json.dumps(view, ensure_ascii=False, allow_nan=False) + # The JSON data block cannot terminate its script element or create markup. + payload = (payload.replace("&", "\\u0026").replace("<", "\\u003c") + .replace(">", "\\u003e").replace("\u2028", "\\u2028") + .replace("\u2029", "\\u2029")) + template = files("inferrsi").joinpath("data", "dashboard", "template.html").read_text( + encoding="utf-8" + ) + replacements = {"__DASHBOARD_DATA__": payload, "__DASHBOARD_CHART__": chart, + "__DASHBOARD_TEMPORAL__": temporal} + document = re.sub( + r"__DASHBOARD_(?:DATA|CHART|TEMPORAL)__", lambda match: replacements[match.group()], + template, + ) + output = Path(destination) if destination is not None else source / "dashboard.html" + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text(document, encoding="utf-8") + return output.resolve() diff --git a/src/inferrsi/harness/dashboard_data.py b/src/inferrsi/harness/dashboard_data.py new file mode 100644 index 000000000..5cde2a5d3 --- /dev/null +++ b/src/inferrsi/harness/dashboard_data.py @@ -0,0 +1,414 @@ +"""Read recorded run and campaign evidence into a dashboard view model. + +Only known fields and explicitly referenced artifacts are exposed. Raw command +lines, environments, error messages and log contents are deliberately omitted. +""" + +from __future__ import annotations + +import hashlib +import json +import math +import re +from pathlib import Path, PurePosixPath, PureWindowsPath + +STAGES = ("diagnosis", "knowledge", "search", "execution", "validation", "review") +PHASE_STAGES = { + "source_snapshot": "execution", + "measurement": "execution", + "quality_validation": "validation", +} +EVENT_TYPES = { + "agent.context", "agent.request", "agent.response", "agent.tool", + "agent.completed", "agent.failed", "agent.protocol", "agent.reservation", +} + + +def _number(value: object) -> float | None: + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + try: + return float(value) if math.isfinite(value) else None + except OverflowError: + return None + + +def _text(value: object, default: str = "not_recorded") -> str: + if not isinstance(value, str) or not value: + return default + value = re.sub(r"https?://\S+", "[redacted-url]", value, flags=re.I) + value = re.sub(r"\b[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}\b", "[redacted-email]", value) + value = re.sub(r"\b(?:\d{1,3}\.){3}\d{1,3}\b", "[redacted-address]", value) + value = re.sub(r"(?\"']+", "[redacted-path]", value) + value = re.sub(r"(?\"']+/)*[^\s/<>\"']+", "[redacted-path]", value) + value = re.sub( + r"(?i)\b(api[_-]?key|token|password|secret)\s*[:=]\s*\S+", + r"\1=[redacted]", value, + ) + return value[:240] + + +def _inside(root: Path, path: Path) -> bool: + try: + path.resolve().relative_to(root.resolve()) + return True + except (OSError, ValueError): + return False + + +def _object(path: Path, root: Path) -> tuple[dict, str]: + if not _inside(root, path): + return {}, "unsafe" + try: + data = json.loads(path.read_text(encoding="utf-8")) + except FileNotFoundError: + return {}, "missing" + except (OSError, UnicodeError, json.JSONDecodeError): + return {}, "corrupt" + return (data, "present") if isinstance(data, dict) else ({}, "corrupt") + + +def _file_ref(root: Path, path: Path, kind: str) -> dict: + try: + relative = path.resolve().relative_to(root.resolve()).as_posix() + except (OSError, ValueError): + return {"kind": kind, "state": "missing", "missing_reason": "unsafe_path"} + try: + hasher = hashlib.sha256() + with path.open("rb") as stream: + for block in iter(lambda: stream.read(1024 * 1024), b""): + hasher.update(block) + digest = hasher.hexdigest() + except FileNotFoundError: + return {"kind": kind, "path": relative, "state": "missing", "missing_reason": "file_absent"} + except OSError: + return {"kind": kind, "path": relative, "state": "missing", "missing_reason": "unreadable"} + return {"kind": kind, "path": relative, "state": "present", "sha256": digest} + + +def _explicit_ref(root: Path, value: object, kind: str) -> dict: + if not isinstance(value, dict): + return {"kind": kind, "state": "missing", "missing_reason": "not_collected"} + if value.get("state") == "missing": + reason = value.get("missing_reason") + allowed = {"not_collected", "file_absent", "redacted", "not_applicable"} + result = {"kind": kind, "state": "missing", "missing_reason": reason + if isinstance(reason, str) and reason in allowed else "not_collected"} + expected = value.get("sha256") + if isinstance(expected, str) and re.fullmatch(r"[a-fA-F0-9]{64}", expected): + result["sha256"] = expected.lower() + return result + raw_path = value.get("path") + if not isinstance(raw_path, str) or not raw_path: + return {"kind": kind, "state": "missing", "missing_reason": "not_collected"} + posix, windows = PurePosixPath(raw_path), PureWindowsPath(raw_path) + if (posix.is_absolute() or windows.is_absolute() or windows.drive + or ".." in posix.parts or ".." in windows.parts): + return {"kind": kind, "state": "missing", "missing_reason": "unsafe_path"} + ref = _file_ref(root, root.joinpath(*posix.parts), kind) + if ref["state"] == "present": + expected = value.get("sha256") + if not isinstance(expected, str) or not re.fullmatch(r"[a-fA-F0-9]{64}", expected): + return {"kind": kind, "state": "missing", "missing_reason": "hash_not_recorded"} + if expected.lower() != ref["sha256"]: + return {"kind": kind, "state": "missing", "missing_reason": "hash_mismatch"} + return ref + + +def _events(path: Path, root: Path) -> tuple[list[dict], list[int], str]: + if not _inside(root, path): + return [], [], "unsafe" + try: + lines = path.read_text(encoding="utf-8").splitlines() + except FileNotFoundError: + return [], [], "missing" + except (OSError, UnicodeError): + return [], [], "corrupt" + events, corrupt = [], [] + for line_number, line in enumerate(lines, 1): + try: + record = json.loads(line) + if not isinstance(record, dict): + raise ValueError("event is not an object") + except (json.JSONDecodeError, ValueError): + corrupt.append(line_number) + continue + phase = record.get("phase") + if phase not in (*PHASE_STAGES, "complete", "crashed"): + phase = None + kind = record.get("type") + event_type = kind if isinstance(kind, str) and kind in EVENT_TYPES else None + stage = record.get("stage") if record.get("stage") in STAGES else PHASE_STAGES.get(phase) + time = record.get("time") + if not isinstance(time, str) or not re.fullmatch(r"[0-9TtZz:+.\-]{10,40}", time): + time = None + trial = record.get("trial") + events.append({ + "line": line_number, "time": time, "phase": phase, "type": event_type, + "stage": stage, "side": record.get("side") + if record.get("side") in ("baseline", "candidate") else None, + "trial": trial if isinstance(trial, int) and not isinstance(trial, bool) + and 0 < trial <= 1_000_000 else None, + "tool": record.get("name") if record.get("name") in ("read_file", "submit_answer") + else None, + }) + return events, corrupt, "present" + + +def _run(root: Path, path: Path, candidate: dict, run_id: str | None, noise: float | None) -> dict: + lock, _ = _object(path / "run.lock.json", root) + repositories, _ = _object(path / "repositories.lock.json", root) + evaluation, evaluation_state = _object(path / "evaluation.json", root) + status, status_state = _object(path / "status.json", root) + validation, validation_state = _object(path / "validation" / "execution.json", root) + events, corrupt, event_state = _events(path / "events.jsonl", root) + refs = { + "events": _file_ref(root, path / "events.jsonl", "events"), + "evaluation": _file_ref(root, path / "evaluation.json", "evaluation"), + "status": _file_ref(root, path / "status.json", "status"), + "quality": _file_ref(root, path / "validation" / "execution.json", "quality"), + "agent_trace": _explicit_ref(root, candidate.get("agent_trace_ref"), "agent_trace"), + "profile": _explicit_ref(root, candidate.get("profile_ref"), "profile"), + } + for key, parse_state in (("evaluation", evaluation_state), ("status", status_state), + ("quality", validation_state), ("events", event_state)): + if parse_state in {"corrupt", "unsafe"}: + refs[key] = {**refs[key], "state": "missing", "missing_reason": + "unsafe_path" if parse_state == "unsafe" else "corrupt"} + for event in events: + event["source"] = "events" + event["refs"] = [refs["events"]] + if event["phase"] == "measurement" and event["side"] and event["trial"]: + trial = path / event["side"] / f'trial-{event["trial"]:03d}' + event["refs"].extend( + _file_ref(root, trial / name, kind) + for name, kind in (("execution.json", "execution"), ("stdout.log", "stdout"), + ("stderr.log", "stderr")) + ) + elif event["phase"] == "quality_validation": + event["refs"].append(refs["quality"]) + if refs["agent_trace"]["state"] == "present": + agent_events, agent_corrupt, agent_state = _events(root / refs["agent_trace"]["path"], root) + if agent_state in {"corrupt", "unsafe"}: + refs["agent_trace"] = {**refs["agent_trace"], "state": "missing", + "missing_reason": "unsafe_path" if agent_state == "unsafe" + else "corrupt"} + for event in agent_events: + event["source"] = "agent_trace" + event["refs"] = [refs["agent_trace"]] + events.extend(agent_events) + corrupt.extend(agent_corrupt) + tool_events = [event for event in events if event["source"] == "agent_trace" + and event["type"] == "agent.tool"] + refs["tool_calls"] = ({**refs["agent_trace"], "kind": "tool_calls"} if tool_events else { + "kind": "tool_calls", "state": "missing", "missing_reason": "not_collected" + }) + validation_status = validation.get("status") + returncode = validation.get("returncode") + if validation_status == "completed" and type(returncode) is int and returncode == 0: + quality = "passed" + elif validation_status == "failed" and type(returncode) is int and returncode != 0: + quality = "failed" + else: + quality = "not_recorded" + integrity = evaluation.get("source_integrity") + verified_value = integrity.get("verified") if isinstance(integrity, dict) else None + verified = verified_value if isinstance(verified_value, bool) else None + baseline = evaluation.get("baseline") if isinstance(evaluation.get("baseline"), dict) else {} + measured = evaluation.get("candidate") if isinstance(evaluation.get("candidate"), dict) else {} + gain = _number(evaluation.get("improvement_pct")) + if noise is None: + ranges = [_number(item.get("range_pct_of_median")) for item in (baseline, measured)] + ranges = [item for item in ranges if item is not None and item >= 0] + noise = max(ranges) if ranges else None + state_value = status.get("state") + state = state_value if isinstance(state_value, str) and state_value in { + "running", "completed", "failed", "crashed", "timeout", "cancelled", "termination_failed" + } else "not_recorded" + if quality == "failed": + outcome = "quality_failed" + elif state in {"failed", "crashed", "timeout", "cancelled", "termination_failed"}: + outcome = "failed" + elif gain is None: + outcome = "not_recorded" + elif quality != "passed" or not verified: + outcome = "unverified" + elif noise is None: + outcome = "noise_not_recorded" + elif abs(gain) <= noise: + outcome = "noise" + else: + outcome = "improved" if gain > 0 else "regressed" + candidate_id = candidate.get("candidate_id") + signature_values = [lock.get(key) for key in ( + "model", "model_revision", "workload_id", "scenario", "backend", "comparison_scope" + )] + [evaluation.get(key) for key in ("metric_key", "metric_unit", "direction")] + repository = repositories.get("baseline") + snapshot = repository.get("snapshot_hash") if isinstance(repository, dict) else None + command, devices = lock.get("baseline_command"), lock.get("device_ids") + valid_sequences = all( + isinstance(seq, list) and seq and all(isinstance(item, str) and item for item in seq) + for seq in (command, devices) + ) + signature = (tuple(signature_values) + (snapshot, tuple(command), tuple(devices))) if ( + all(isinstance(value, str) and value for value in signature_values) + and isinstance(snapshot, str) and snapshot and valid_sequences + ) else None + return { + "candidate_id": _text(candidate_id) if isinstance(candidate_id, str) else None, + "title": _text(candidate.get("title"), "Recorded candidate"), + "run_id": _text(run_id) if run_id else None, + "round": candidate.get("round") if isinstance(candidate.get("round"), int) + and not isinstance(candidate.get("round"), bool) and candidate["round"] > 0 else None, + "state": state, "status_state": status_state, "evaluation_state": evaluation_state, + "validation_state": validation_state, "event_state": event_state, + "quality": quality, "source_verified": verified, "improvement_pct": gain, + "minimum_improvement_pct": _number(evaluation.get("min_improvement_pct")), + "baseline_median": _number(baseline.get("median")), + "candidate_median": _number(measured.get("median")), "noise_band_pct": noise, + "metric_key": _text(evaluation.get("metric_key")), + "metric_unit": _text(evaluation.get("metric_unit")), + "direction": evaluation.get("direction") + if evaluation.get("direction") in ("minimize", "maximize") else None, + "verdict": evaluation.get("verdict") + if evaluation.get("verdict") in ("PASS", "FAIL", "INCONCLUSIVE") else "not_recorded", + "outcome": outcome, "events": events, "corrupt_event_lines": corrupt, + "artifacts": refs, "_comparison_signature": signature, + } + + +def _rounds(path: Path, root: Path) -> tuple[list[dict] | None, str, list[int]]: + if not _inside(root, path): + return None, "unsafe", [] + try: + lines = path.read_text(encoding="utf-8").splitlines() + except FileNotFoundError: + return None, "missing", [] + except (OSError, UnicodeError): + return None, "corrupt", [] + result, corrupt = [], [] + for line_number, line in enumerate(lines, 1): + try: + row = json.loads(line) + except json.JSONDecodeError: + corrupt.append(line_number) + continue + if (isinstance(row, dict) and isinstance(row.get("round"), int) + and not isinstance(row.get("round"), bool) and row["round"] > 0): + record = {"round": row["round"]} + decision = row.get("decision") + if (isinstance(decision, dict) and isinstance(decision.get("candidate_id"), str) + and decision.get("outcome") in ( + "adopted", "rejected", "rolled_back", "no_gain", "noise" + )): + record["decision"] = {"candidate_id": _text(decision["candidate_id"]), + "outcome": decision["outcome"]} + result.append(record) + else: + corrupt.append(line_number) + return result, "corrupt" if corrupt else "present", corrupt + + +def load_dashboard(source: Path) -> dict: + """Aggregate a single legacy run or a campaign without inventing records.""" + root = Path(source).resolve() + if not root.is_dir(): + raise ValueError("dashboard input must be an existing directory") + single = (root / "run.lock.json").is_file() + campaign, campaign_state = _object(root / "campaign.json", root) + if not single and campaign_state != "present": + raise ValueError("dashboard input must contain run.lock.json or campaign.json") + noise_data = campaign.get("noise") if isinstance(campaign.get("noise"), dict) else {} + noise = _number(noise_data.get("band_pct")) + if noise is not None and noise < 0: + noise = None + rows = [] + if single: + rows.append(_run(root, root, {}, root.name, noise)) + else: + candidate_root = root / "candidates" + if _inside(root, candidate_root) and candidate_root.is_dir(): + for directory in sorted(candidate_root.iterdir()): + if not directory.is_dir() or directory.is_symlink(): + continue + metadata, state = _object(directory / "candidate.json", root) + if state != "present": + continue + run_id = metadata.get("run_id") + if (not isinstance(run_id, str) or Path(run_id).name != run_id + or run_id in {".", ".."}): + run_id = None + run_path = root / "runs" / run_id if run_id else directory / "missing-run" + rows.append(_run(root, run_path, metadata, run_id, noise)) + if not rows and _inside(root, root / "runs") and (root / "runs").is_dir(): + for directory in sorted((root / "runs").iterdir()): + if directory.is_dir() and not directory.is_symlink(): + rows.append(_run(root, directory, {}, directory.name, noise)) + rounds, rounds_state, corrupt_round_lines = ( + (None, "missing", []) if single else _rounds(root / "rounds.jsonl", root) + ) + decisions = {(item["round"], item["decision"]["candidate_id"]): item["decision"]["outcome"] + for item in (rounds or []) if "decision" in item} + for row in rows: + row["decision"] = decisions.get((row["round"], row["candidate_id"]), "not_recorded") + signatures = [] + for row in rows: + signature = row.pop("_comparison_signature") + if row["improvement_pct"] is not None: + signatures.append(signature) + comparable = len(signatures) < 2 or ( + all(signature is not None for signature in signatures) + and len(set(signatures)) == 1 + ) + valid_gains = [row["improvement_pct"] for row in rows if row["improvement_pct"] is not None + and row["quality"] == "passed" and row["source_verified"] is True + and row["state"] == "completed" and row["verdict"] in ("PASS", "FAIL") + and row["decision"] != "rolled_back"] + if not comparable: + valid_gains = [] + best = max(valid_gains) if valid_gains else None + goal = campaign.get("goal") if isinstance(campaign.get("goal"), dict) else {} + target = _number(goal.get("target_improvement_pct")) + if target is None and single and rows: + target = rows[0]["minimum_improvement_pct"] + diagnostics = [] + for row in rows: + if row["outcome"] == "failed" and row["artifacts"]["status"]["state"] == "present": + diagnostics.append({"kind": "execution_failed", "candidate_id": row["candidate_id"], + "refs": [row["artifacts"]["status"]]}) + elif row["outcome"] == "quality_failed" and row["improvement_pct"] is not None \ + and row["improvement_pct"] > 0: + refs = [row["artifacts"][key] for key in ("evaluation", "quality")] + if all(ref["state"] == "present" for ref in refs): + diagnostics.append({"kind": "quality_failed_with_gain", + "candidate_id": row["candidate_id"], "refs": refs}) + if row["outcome"] in {"noise", "regressed"}: + ref = row["artifacts"]["evaluation"] + if ref["state"] == "present": + diagnostics.append({"kind": "within_noise" if row["outcome"] == "noise" + else "measured_regression", "candidate_id": row["candidate_id"], + "refs": [ref]}) + if row["decision"] == "rolled_back": + ref = _file_ref(root, root / "rounds.jsonl", "rounds") + if ref["state"] == "present": + diagnostics.append({"kind": "rolled_back", "candidate_id": row["candidate_id"], + "refs": [ref]}) + recorded = {event["stage"] for row in rows for event in row["events"] if event["stage"]} + declared = campaign.get("stages_declared") + if not isinstance(declared, list): + declared = None + return { + "campaign_id": _text(campaign.get("campaign_id"), "Single run" if single else "Campaign"), + "synthetic": campaign.get("synthetic") is True, + "rounds": rounds, "rounds_state": rounds_state, + "corrupt_round_lines": corrupt_round_lines, + "candidates": rows, + "overview": {"comparison_state": "comparable" if comparable else "incompatible", + "best_improvement_pct": best, "target_improvement_pct": target, + "target_gap_pct": target - best if target is not None and best is not None + else None}, + "stages": [{"name": stage, "state": "recorded" if stage in recorded else + "not_applicable" if declared is not None and stage not in declared else + "stage_not_recorded"} for stage in STAGES], + "diagnostics": diagnostics, + } diff --git a/src/inferrsi/harness/dashboard_timeline.py b/src/inferrsi/harness/dashboard_timeline.py new file mode 100644 index 000000000..f7c570ba4 --- /dev/null +++ b/src/inferrsi/harness/dashboard_timeline.py @@ -0,0 +1,170 @@ +"""Render measured candidate results and recorded stages on one time axis.""" + +from __future__ import annotations + +import math +from datetime import datetime, timezone +from html import escape + +from .dashboard_data import STAGES + +_NAMES = { + "diagnosis": "Diagnosis", "knowledge": "Knowledge", "search": "Candidate search", + "execution": "Execution", "validation": "Validation", "review": "Review", +} +_COLORS = { + "diagnosis": "#9d8bff", "knowledge": "#5794f2", "search": "#56c5ce", + "execution": "#73bf69", "validation": "#ff9830", "review": "#e5a8e5", +} + + +def _instant(value: object) -> float | None: + if not isinstance(value, str): + return None + try: + parsed = datetime.fromisoformat(value.replace("Z", "+00:00").replace("z", "+00:00")) + if parsed.tzinfo is None: + return None + return parsed.timestamp() + except (ValueError, OverflowError, OSError): + return None + + +def temporal_chart(view: dict) -> tuple[str, str]: + """Use only timestamped events; a result point requires a terminal event.""" + rows = view["candidates"] + events = [] + terminals = [] + untimed_results = 0 + for row in rows: + timed = [(event, _instant(event.get("time"))) for event in row["events"]] + events.extend((row, event, stamp) for event, stamp in timed + if stamp is not None and event.get("stage") in STAGES) + terminal = [(event, stamp) for event, stamp in timed + if stamp is not None and event.get("phase") in ("complete", "crashed")] + if terminal: + terminals.append((row, max(stamp for _, stamp in terminal))) + elif row["improvement_pct"] is not None or row["outcome"] == "failed": + untimed_results += 1 + stamps = [stamp for _, _, stamp in events] + [stamp for _, stamp in terminals] + if not stamps: + return "", "No timestamped events recorded; a time axis cannot be drawn." + first, last = min(stamps), max(stamps) + if first == last: + first, last = first - 1, last + 1 + + comparable = view["overview"]["comparison_state"] == "comparable" + measured = [(row, stamp) for row, stamp in terminals + if comparable and row["improvement_pct"] is not None] + values = [row["improvement_pct"] for row, _ in measured] + target = view["overview"]["target_improvement_pct"] + if target is not None: + values.append(target) + bands = [row["noise_band_pct"] for row, _ in measured] + noise = bands[0] if bands and bands[0] is not None and all( + band is not None and math.isclose(band, bands[0], rel_tol=1e-9, abs_tol=1e-9) + for band in bands + ) else None + if noise is not None: + values.extend((-noise, noise)) + values.append(0.0) + if any(not math.isfinite(value) or abs(value) > 1_000_000 for value in values): + return "", "Recorded values exceed the supported time-chart scale." + low, high = min(values), max(values) + pad = max((high - low) * 0.12, 1.0) + low, high = low - pad, high + pad + left, right, top, bottom = 140, 925, 40, 220 + def x(stamp: float) -> float: + return left + (stamp - first) / (last - first) * (right - left) + + def y(value: float) -> float: + return bottom - (value - low) / (high - low) * (bottom - top) + + svg = [ + '', + 'Candidate results and stage events over time', + 'Results use terminal event timestamps. Stage dots are ' + 'events, not durations. Crosses in the no-value row have no numeric measurement.', + '', + ] + + def text(px: float, py: float, label: str, color: str = "#aab2bf", size: int = 10) -> None: + svg.append(f'{escape(label)}') + + if noise is not None: + svg.append(f'') + text(left + 5, y(noise) + 12, f"Noise ±{noise:.1f}%", "#ffb76b") + for index in range(5): + tick = low + (high - low) * index / 4 + py = y(tick) + svg.append(f'') + text(76, py + 4, f"{tick:+.1f}%") + if target is not None: + py = y(target) + svg.append(f'') + text(left + 5, py - 5, f"Target {target:+.1f}%", "#f2c96d") + text(12, 26, "Gain vs baseline") + text(12, 246, "No numeric result") + text(12, 280, "Stage events") + for index, stage in enumerate(STAGES): + py = 294 + index * 18 + text(12, py + 4, _NAMES[stage]) + svg.append(f'') + for row, event, stamp in events: + py = 294 + STAGES.index(event["stage"]) * 18 + label = f'{row["title"]}: {_NAMES[event["stage"]]} · {event["time"]}' + svg.append(f'{escape(label)}') + for row, stamp in terminals: + gain, outcome = row["improvement_pct"], row["outcome"] + if gain is not None and not comparable: + continue + if gain is None and outcome != "failed": + continue + px, py = x(stamp), y(gain) if gain is not None else 242 + label = f'{row["title"]}: {outcome.replace("_", " ")}' + label += f' {gain:+.1f}%' if gain is not None else ' · no numeric result' + svg.append(f'' + f'{escape(label)}') + if outcome == "failed": + svg.append(f'') + elif outcome == "quality_failed": + svg.append(f'') + elif outcome == "noise": + svg.append(f'') + else: + color = {"improved": "#73bf69", "regressed": "#f2495c"}.get(outcome, "#8e98a6") + svg.append(f'') + if row["decision"] == "rolled_back": + svg.append(f'') + svg.append('') + text(min(px + 8, right - 110), max(py - 8, top + 8), row["title"][:24], "#d8dce3") + for index in range(3): + stamp = first + (last - first) * index / 2 + px = x(stamp) + svg.append(f'') + label = datetime.fromtimestamp(stamp, timezone.utc).strftime("%Y-%m-%d %H:%M:%SZ") + text(max(left, min(px - 60, right - 120)), 416, label) + svg.append('') + note = "Terminal events place results; stage dots are events, not durations." + if untimed_results: + note += f" {untimed_results} result(s) have no recorded terminal timestamp." + if not comparable: + note += " Incompatible candidate measurements are not plotted together." + elif measured and noise is None: + note += " No common noise band recorded." + return "".join(svg), note diff --git a/src/inferrsi/model/anthropic_agent.py b/src/inferrsi/model/anthropic_agent.py index b8d4ff092..2a47e499e 100644 --- a/src/inferrsi/model/anthropic_agent.py +++ b/src/inferrsi/model/anthropic_agent.py @@ -591,7 +591,26 @@ def trace_metrics(path): and re.fullmatch(r"[0-9a-f]{64}", f["sha256"]) for f in manifest)) paths = [f["path"] for f in manifest] if valid_manifest else [] - valid = (protocols == [{"type": "agent.protocol", "client_kind": FULL_CONTEXT_KIND}] + protocol = protocols[0] if len(protocols) == 1 else {} + original_fields = {"type", "client_kind"} + envelope_fields = {"schema_version", "time", "stage", "campaign_id", "candidate_id", + "run_id", "round"} + run_id, round_number = protocol.get("run_id"), protocol.get("round") + valid_envelope = (original_fields | envelope_fields <= set(protocol) + and set(protocol) <= original_fields | envelope_fields | {"refs"} + and type(protocol.get("schema_version")) is int + and protocol["schema_version"] == 1 + and all(isinstance(protocol.get(key), str) and protocol[key] + for key in ("time", "campaign_id", "candidate_id")) + and protocol.get("stage") in ("diagnosis", "knowledge", "search", + "execution", "validation", "review") + and (run_id is None or isinstance(run_id, str) and bool(run_id)) + and (round_number is None or ( + type(round_number) is int and round_number > 0)) + and ("refs" not in protocol or isinstance(protocol["refs"], list))) + valid_protocol = (len(protocols) == 1 and protocol.get("client_kind") == FULL_CONTEXT_KIND + and (set(protocol) == original_fields or valid_envelope)) + valid = (valid_protocol and len(contexts) == 1 and context.get("client_kind") == FULL_CONTEXT_KIND and valid_manifest and paths == sorted(set(paths)) and "task.json" in paths and context.get("manifest_sha256") == hashlib.sha256( diff --git a/tests/test_campaign_records.py b/tests/test_campaign_records.py new file mode 100644 index 000000000..1bd2f6550 --- /dev/null +++ b/tests/test_campaign_records.py @@ -0,0 +1,71 @@ +"""Local campaign records connect runs without copying private run artifacts.""" + +import json + +import pytest + +from inferrsi.cli import main +from inferrsi.harness.campaign_records import create_campaign, record_candidate, record_decision +from inferrsi.harness.dashboard_data import load_dashboard + + +def test_record_candidates_in_one_round_and_reference_trace(tmp_path): + create_campaign(tmp_path, "campaign", target=20, synthetic=True) + for name in ("a", "b"): + run = tmp_path / "runs" / name + run.mkdir(parents=True) + (run / "run.lock.json").write_text("{}", encoding="utf-8") + trace = tmp_path / "candidates/a/trace.jsonl" + trace.parent.mkdir(parents=True) + trace.write_text('{"type":"agent.tool","stage":"search"}\n', encoding="utf-8") + record_candidate(tmp_path, candidate_id="a", run_id="a", round_number=1, title="First", + agent_trace="candidates/a/trace.jsonl") + record_candidate(tmp_path, candidate_id="b", run_id="b", round_number=1, title="Second") + + rounds = [json.loads(line) for line in (tmp_path / "rounds.jsonl").read_text().splitlines()] + view = load_dashboard(tmp_path) + + assert rounds == [{"round": 1, "candidates": ["a", "b"]}] + assert view["synthetic"] is True + assert view["candidates"][0]["artifacts"]["agent_trace"]["state"] == "present" + assert any(event["stage"] == "search" for event in view["candidates"][0]["events"]) + + +def test_campaign_records_refuse_overwrite_and_escaping_refs(tmp_path): + create_campaign(tmp_path, "campaign") + with pytest.raises(FileExistsError): + create_campaign(tmp_path, "campaign") + run = tmp_path / "runs/a" + run.mkdir(parents=True) + (run / "run.lock.json").write_text("{}", encoding="utf-8") + with pytest.raises(ValueError, match="relative path"): + record_candidate(tmp_path, candidate_id="a", run_id="a", round_number=1, title="First", + agent_trace="../outside.jsonl") + assert not (tmp_path / "rounds.jsonl").exists() + assert not (tmp_path / "candidates/a/candidate.json").exists() + + +def test_campaign_and_dashboard_cli_round_trip(tmp_path, capsys): + root = tmp_path / "campaign" + assert main(["campaign", "init", "--directory", str(root), "--id", "example"]) == 0 + run = root / "runs/run-a" + run.mkdir(parents=True) + (run / "run.lock.json").write_text("{}", encoding="utf-8") + assert main(["campaign", "record", "--directory", str(root), "--candidate", "a", + "--run-id", "run-a", "--round", "1", "--title", "First"]) == 0 + assert main(["dashboard", "--input", str(root)]) == 0 + assert (root / "dashboard.html").is_file() + assert "dashboard" in capsys.readouterr().out + + +def test_round_decision_is_recorded_once_and_visible(tmp_path): + create_campaign(tmp_path, "example") + run = tmp_path / "runs/a" + run.mkdir(parents=True) + (run / "run.lock.json").write_text("{}", encoding="utf-8") + record_candidate(tmp_path, candidate_id="a", run_id="a", round_number=1, title="First") + record_decision(tmp_path, candidate_id="a", round_number=1, outcome="rolled_back") + + assert load_dashboard(tmp_path)["candidates"][0]["decision"] == "rolled_back" + with pytest.raises(ValueError, match="already has a decision"): + record_decision(tmp_path, candidate_id="a", round_number=1, outcome="adopted") diff --git a/tests/test_campaign_trace.py b/tests/test_campaign_trace.py new file mode 100644 index 000000000..8839e9e88 --- /dev/null +++ b/tests/test_campaign_trace.py @@ -0,0 +1,51 @@ +"""Correlation preserves original agent fields and existing trace metrics.""" + +import hashlib +import json + +import pytest + +from inferrsi.harness.campaign_trace import CampaignTrace +from inferrsi.model import anthropic_agent as agent + + +def test_correlated_full_context_trace_keeps_existing_metrics(tmp_path): + manifest = [{"path": "task.json", "bytes": 2, "sha256": "a" * 64}] + digest = hashlib.sha256(agent.canonical(manifest).encode()).hexdigest() + usage = {"input_tokens": 10, "output_tokens": 8, "cached_input_tokens": 0} + events = [ + {"type": "agent.protocol", "client_kind": agent.FULL_CONTEXT_KIND}, + {"type": "agent.context", "client_kind": agent.FULL_CONTEXT_KIND, + "files": manifest, "manifest_sha256": digest}, + {"type": "agent.request", "request": 1}, + {"type": "agent.response", "request": 1, "usage": usage, "provider_model": "fixture"}, + {"type": "agent.tool", "request": 1, "name": "submit_answer"}, + {"type": "agent.completed", "requests": 1, "usage": usage}, + ] + original = tmp_path / "original.jsonl" + original.write_text("".join(json.dumps(event) + "\n" for event in events), encoding="utf-8") + path = tmp_path / "correlated.jsonl" + writer = CampaignTrace(path, campaign_id="campaign", candidate_id="candidate", stage="search", + run_id="run", round_number=1) + for event in events: + writer(event) + correlated = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines()] + assert all(all(row[key] == value for key, value in original_event.items()) + for row, original_event in zip(correlated, events)) + before, after = agent.trace_metrics(original), agent.trace_metrics(path) + for key in ("usage", "observed_usage", "usage_complete", "completed_turns", "api_requests", + "client_kind", "provider_model"): + assert after[key] == before[key] + assert after["usage_complete"] is True + + correlated[0]["stage"] = "invalid-stage" + path.write_text("".join(json.dumps(event) + "\n" for event in correlated), encoding="utf-8") + assert agent.trace_metrics(path)["usage_complete"] is False + + +def test_trace_rejects_envelope_collisions(tmp_path): + writer = CampaignTrace(tmp_path / "trace.jsonl", campaign_id="c", candidate_id="a", + stage="search") + with pytest.raises(ValueError, match="correlation fields"): + writer({"type": "agent.tool", "stage": "execution"}) + assert not (tmp_path / "trace.jsonl").exists() diff --git a/tests/test_codex_plugin.py b/tests/test_codex_plugin.py index 3b9b61e22..57ab19a65 100644 --- a/tests/test_codex_plugin.py +++ b/tests/test_codex_plugin.py @@ -70,6 +70,19 @@ def test_bundle_contains_all_declared_package_data(distribution): assert {f"inferrsi/{name}" for name in expected} <= manifest["files"].keys() +def test_isolated_runtime_can_export_dashboard(distribution, tmp_path): + run = tmp_path / "run" + run.mkdir() + (run / "run.lock.json").write_text("{}", encoding="utf-8") + + result = invoke(distribution, tmp_path, "dashboard", "--input", str(run)) + + assert result.returncode == 0, result.stderr + assert "InferRSI · Optimization dashboard" in (run / "dashboard.html").read_text( + encoding="utf-8" + ) + + def test_isolated_runtime_and_exact_knowledge_reads(distribution, tmp_path): # Neither editable installs, PYTHONPATH nor the target's Python module may # supply the core. This also exercises resource reads from an extracted ZIP. diff --git a/tests/test_dashboard_data.py b/tests/test_dashboard_data.py new file mode 100644 index 000000000..de1c27b27 --- /dev/null +++ b/tests/test_dashboard_data.py @@ -0,0 +1,366 @@ +"""Dashboard aggregation uses recorded evidence and keeps missing data explicit.""" + +import json +from pathlib import Path + +import pytest + +from inferrsi.harness.dashboard import write_dashboard +from inferrsi.harness.dashboard_data import load_dashboard +from inferrsi.harness.dashboard_timeline import temporal_chart + + +def write_json(path: Path, value: dict) -> None: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(value), encoding="utf-8") + + +def run(root: Path, name: str, gain: float, *, quality: str, verified: bool) -> None: + directory = root / "runs" / name + write_json(directory / "run.lock.json", { + "model": "example-model", "model_revision": "example-revision", + "workload_id": "example-workload", "scenario": "http", "backend": "cuda", + "comparison_scope": "http_e2e", "baseline_command": ["python", "bench.py"], + "device_ids": ["0"], + }) + write_json(directory / "repositories.lock.json", {"baseline": {"snapshot_hash": "a" * 64}}) + write_json(directory / "status.json", {"state": "completed", "run_dir": "/private/run"}) + write_json(directory / "evaluation.json", { + "verdict": "PASS", "metric_key": "throughput", "metric_unit": "tokens/s", + "direction": "maximize", "improvement_pct": gain, + "baseline": {"median": 100, "range_pct_of_median": 1}, + "candidate": {"median": 100 + gain, "range_pct_of_median": 2}, + "source_integrity": {"verified": verified}, + "statistical_significance": "not established", + }) + write_json(directory / "validation" / "execution.json", { + "status": quality, "returncode": 0 if quality == "completed" else 1, + }) + + +def candidate(root: Path, name: str, run_id: str, **extra: object) -> None: + write_json(root / "candidates" / name / "candidate.json", { + "candidate_id": name, "title": name, "run_id": run_id, **extra, + }) + + +def test_old_run_preserves_missing_files_and_corrupt_event_lines(tmp_path): + write_json(tmp_path / "run.lock.json", {"repo": "/private/repo"}) + (tmp_path / "events.jsonl").write_text( + '{"phase":"measurement","side":"baseline","trial":1}\n' + 'not-json\n' + '{"phase":"quality_validation"}\n', encoding="utf-8" + ) + + view = load_dashboard(tmp_path) + + assert view["rounds"] is None + assert view["candidates"][0]["candidate_id"] is None + assert view["candidates"][0]["evaluation_state"] == "missing" + assert view["candidates"][0]["quality"] == "not_recorded" + assert view["candidates"][0]["corrupt_event_lines"] == [2] + assert [event["stage"] for event in view["candidates"][0]["events"]] == [ + "execution", "validation" + ] + assert "/private/" not in json.dumps(view) + + +def test_single_run_exposes_its_recorded_target_without_inventing_a_gain(tmp_path): + write_json(tmp_path / "run.lock.json", {}) + write_json(tmp_path / "evaluation.json", {"min_improvement_pct": 20}) + + overview = load_dashboard(tmp_path)["overview"] + + assert overview["target_improvement_pct"] == 20 + assert overview["best_improvement_pct"] is None + assert overview["target_gap_pct"] is None + + +def test_best_gain_requires_quality_and_source_integrity_and_tracks_noise(tmp_path): + write_json(tmp_path / "campaign.json", { + "campaign_id": "example", "goal": {"target_improvement_pct": 20}, + "noise": {"band_pct": 2}, + }) + for name, gain, quality, verified in [ + ("a", 5, "completed", True), + ("b", 30, "failed", True), + ("c", 40, "completed", False), + ("d", 1.5, "completed", True), + ("e", -3, "completed", True), + ]: + run(tmp_path, name, gain, quality=quality, verified=verified) + candidate(tmp_path, name, name) + + view = load_dashboard(tmp_path) + rows = {row["candidate_id"]: row for row in view["candidates"]} + + assert view["overview"]["best_improvement_pct"] == 5 + assert view["overview"]["target_gap_pct"] == 15 + assert rows["b"]["quality"] == "failed" + assert rows["d"]["outcome"] == "noise" + assert rows["e"]["outcome"] == "regressed" + assert all(item["refs"] for item in view["diagnostics"]) + + +def test_campaign_does_not_compare_different_workloads(tmp_path): + write_json(tmp_path / "campaign.json", {"campaign_id": "example"}) + for name, gain in (("a", 5), ("b", 30)): + run(tmp_path, name, gain, quality="completed", verified=True) + candidate(tmp_path, name, name) + lock = tmp_path / "runs/b/run.lock.json" + data = json.loads(lock.read_text(encoding="utf-8")) + data["workload_id"] = "different-workload" + write_json(lock, data) + + overview = load_dashboard(tmp_path)["overview"] + + assert overview["comparison_state"] == "incompatible" + assert overview["best_improvement_pct"] is None + + +def test_best_gain_requires_a_recorded_zero_exit_code_and_completed_run(tmp_path): + write_json(tmp_path / "campaign.json", {"campaign_id": "example"}) + run(tmp_path, "a", 5, quality="completed", verified=True) + run(tmp_path, "b", 9, quality="completed", verified=True) + candidate(tmp_path, "a", "a") + candidate(tmp_path, "b", "b") + write_json(tmp_path / "runs/a/validation/execution.json", {"status": "completed"}) + write_json(tmp_path / "runs/b/status.json", {"state": "failed"}) + + view = load_dashboard(tmp_path) + + assert view["candidates"][0]["quality"] == "not_recorded" + assert view["overview"]["best_improvement_pct"] is None + + +def test_campaign_does_not_compare_different_baseline_snapshots(tmp_path): + write_json(tmp_path / "campaign.json", {"campaign_id": "example"}) + for name in ("a", "b"): + run(tmp_path, name, 5, quality="completed", verified=True) + candidate(tmp_path, name, name) + write_json(tmp_path / "runs/b/repositories.lock.json", { + "baseline": {"snapshot_hash": "b" * 64}, + }) + + assert load_dashboard(tmp_path)["overview"]["comparison_state"] == "incompatible" + + +def test_unmeasured_failed_candidate_does_not_hide_other_recorded_gains(tmp_path): + write_json(tmp_path / "campaign.json", {"campaign_id": "example"}) + run(tmp_path, "a", 5, quality="completed", verified=True) + candidate(tmp_path, "a", "a") + write_json(tmp_path / "runs/b/status.json", {"state": "failed"}) + candidate(tmp_path, "b", "b") + + overview = load_dashboard(tmp_path)["overview"] + + assert overview["comparison_state"] == "comparable" + assert overview["best_improvement_pct"] == 5 + + +def test_rollback_is_visible_and_does_not_count_as_best_gain(tmp_path): + write_json(tmp_path / "campaign.json", {"campaign_id": "example"}) + for name, gain in (("a", 5), ("b", 3)): + run(tmp_path, name, gain, quality="completed", verified=True) + candidate(tmp_path, name, name, round=1) + (tmp_path / "rounds.jsonl").write_text(json.dumps({ + "round": 1, "decision": {"candidate_id": "a", "outcome": "rolled_back"}, + }) + "\n", encoding="utf-8") + + view = load_dashboard(tmp_path) + + assert view["overview"]["best_improvement_pct"] == 3 + assert view["candidates"][0]["decision"] == "rolled_back" + assert any(item["kind"] == "rolled_back" and item["refs"] for item in view["diagnostics"]) + + +def test_unsafe_and_mismatched_artifact_refs_are_not_followed(tmp_path): + write_json(tmp_path / "campaign.json", {"campaign_id": "example"}) + run(tmp_path, "run-a", 1, quality="completed", verified=True) + candidate(tmp_path, "a", "run-a", agent_trace_ref={ + "path": "../outside-secret", "sha256": "0" * 64, + }) + (tmp_path.parent / "outside-secret").write_text("private contents", encoding="utf-8") + candidate(tmp_path, "b", "run-a", agent_trace_ref={ + "path": "candidates/b/agent_trace.jsonl", "sha256": "0" * 64, + }) + (tmp_path / "candidates/b/agent_trace.jsonl").write_text('{}\n', encoding="utf-8") + + view = load_dashboard(tmp_path) + refs = {row["candidate_id"]: row["artifacts"]["agent_trace"] for row in view["candidates"]} + + assert refs["a"]["state"] == "missing" + assert refs["a"]["missing_reason"] == "unsafe_path" + assert refs["b"]["missing_reason"] == "hash_mismatch" + assert "private contents" not in json.dumps(view) + + +def test_run_symlink_outside_campaign_is_not_read(tmp_path): + root = tmp_path / "campaign" + outside = tmp_path / "outside" + write_json(root / "campaign.json", {"campaign_id": "example"}) + candidate(root, "a", "run-a") + outside.mkdir() + (outside / "events.jsonl").write_text('{"phase":"measurement"}\n', encoding="utf-8") + (root / "runs").mkdir() + try: + (root / "runs/run-a").symlink_to(outside, target_is_directory=True) + except OSError: + pytest.skip("directory symlinks are unavailable") + + row = load_dashboard(root)["candidates"][0] + + assert row["event_state"] == "unsafe" + assert row["events"] == [] + assert row["artifacts"]["events"]["missing_reason"] == "unsafe_path" + + +def test_malformed_record_fields_do_not_crash_dashboard(tmp_path): + write_json(tmp_path / "run.lock.json", {}) + write_json(tmp_path / "status.json", {"state": ["bad"]}) + write_json(tmp_path / "validation/execution.json", {"status": {"bad": True}}) + (tmp_path / "events.jsonl").write_text('{"type":{},"phase":[]}\n', encoding="utf-8") + + row = load_dashboard(tmp_path)["candidates"][0] + + assert row["state"] == "not_recorded" + assert row["quality"] == "not_recorded" + assert row["events"][0]["type"] is None + + +def test_corrupt_evaluation_is_not_reported_as_present(tmp_path): + write_json(tmp_path / "run.lock.json", {}) + (tmp_path / "evaluation.json").write_text('{"verdict":', encoding="utf-8") + + row = load_dashboard(tmp_path)["candidates"][0] + + assert row["evaluation_state"] == "corrupt" + assert row["artifacts"]["evaluation"]["state"] == "missing" + assert row["artifacts"]["evaluation"]["missing_reason"] == "corrupt" + + +def test_corrupt_rounds_and_missing_tool_calls_remain_visible(tmp_path): + write_json(tmp_path / "campaign.json", {"campaign_id": "example"}) + run(tmp_path, "a", 5, quality="completed", verified=True) + candidate(tmp_path, "a", "a") + (tmp_path / "rounds.jsonl").write_text('{"round":1}\nnot-json\n', encoding="utf-8") + + view = load_dashboard(tmp_path) + + assert view["rounds_state"] == "corrupt" + assert view["corrupt_round_lines"] == [2] + assert view["candidates"][0]["artifacts"]["tool_calls"]["missing_reason"] == "not_collected" + + +def test_undeclared_stages_are_distinct_from_missing_stages(tmp_path): + write_json(tmp_path / "campaign.json", { + "campaign_id": "example", "stages_declared": ["execution", "validation"], + }) + + stages = {item["name"]: item["state"] for item in load_dashboard(tmp_path)["stages"]} + + assert stages["diagnosis"] == "not_applicable" + assert stages["execution"] == "stage_not_recorded" + + +def test_export_escapes_untrusted_candidate_data_and_omits_raw_paths(tmp_path): + write_json(tmp_path / "campaign.json", {"campaign_id": "example"}) + run(tmp_path, "run-a", 5, quality="completed", verified=True) + candidate(tmp_path, "a", "run-a", title='__DASHBOARD_CHART__') + (tmp_path / "runs/run-a/status.json").write_text( + '{"state":"completed","run_dir":"/private/person/run"}', encoding="utf-8" + ) + + output = write_dashboard(tmp_path) + html = output.read_text(encoding="utf-8") + + assert '/private/person/run' not in html + assert '' not in html + assert r'\u003cimg' in html + assert '__DASHBOARD_CHART__' in html + assert '' not in svg + assert '<script>' in svg diff --git a/tools/build_codex_plugin.py b/tools/build_codex_plugin.py index bfdcb6614..75c5769a6 100644 --- a/tools/build_codex_plugin.py +++ b/tools/build_codex_plugin.py @@ -85,6 +85,7 @@ def build_plugin( path.suffix == ".txt" and relative.parts[-3:-1] == ("references", "licenses") ) + or (path.suffix == ".html" and relative.parent.parts == ("data", "dashboard")) or (path.name == "upstream-NOTICE" and relative.parent.name == "references") ) if path.suffix != ".py" and not package_data: From 0b621010ddeca0362389860837d5c40bbad94851 Mon Sep 17 00:00:00 2001 From: David Chen <530634352@qq.com> Date: Mon, 2 Nov 2026 19:59:32 +0800 Subject: [PATCH 02/23] Harden dashboard evidence and navigation Agent-assisted-by: Codex (implementation) Signed-off-by: David Chen <530634352@qq.com> --- docs/dashboard.md | 13 +- src/inferrsi/data/dashboard/template.html | 24 +- src/inferrsi/harness/dashboard.py | 39 +++- src/inferrsi/harness/dashboard_data.py | 257 +++++++++++++++------ src/inferrsi/harness/dashboard_timeline.py | 10 +- tests/test_campaign_records.py | 5 +- tests/test_dashboard_data.py | 246 +++++++++++++++++++- 7 files changed, 501 insertions(+), 93 deletions(-) diff --git a/docs/dashboard.md b/docs/dashboard.md index d0fff03c8..4c1848024 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -83,6 +83,9 @@ optional `stage` field (`diagnosis`, `knowledge`, `search`, `execution`, `inferrsi.harness.campaign_trace.CampaignTrace` instance as its existing event callback; it appends the original event with time and caller-supplied IDs. The caller associates the completed trace with `agent_trace_ref` using its SHA-256. +The dashboard links an event to a run only when its campaign, candidate, run, +round and stage envelope is complete and matches. Legacy or mismatched events +remain visibly unlinked and do not count as stage coverage or tool calls. The callback does not allocate IDs or intercept unrelated agents. Existing runner phases are mapped to execution and validation. The dashboard reads allowlisted event metadata; it does not include raw agent requests, arbitrary event messages, environment data, or raw logs. @@ -110,9 +113,12 @@ markers. Unrelated candidate measurements are not joined into a cumulative line. and no recorded rollback. Successful validation covers only that command, not untested quality properties. - Multiple measured runs are summarized together only when their declared model, - model revision, workload, scenario, backend, device IDs, baseline source - snapshot and command, comparison scope, metric, unit and direction agree. - Otherwise the aggregate and shared chart are withheld. + model revision, workload, scenario, backend, verified physical CUDA device, + driver, host platform, target Python/packages, baseline source snapshot, + baseline and validation commands, comparison scope, metric, unit and direction + agree. Otherwise the aggregate and shared chart are withheld. Private device + identity is used only for the comparison and is not included in the export. + Cross-run comparison for other device backends is currently withheld. - Noise uses the campaign's declared band, or the larger recorded trial range when no band is supplied. It is descriptive; statistical significance is not established. @@ -126,6 +132,7 @@ markers. Unrelated candidate measurements are not joined into a cumulative line. differ across candidates are not represented as one shared band. - The review button creates a local Markdown download and displays its content for copying if a browser blocks downloads. It does not send or publish it. +- Existing non-dashboard files and symbolic-link outputs are never overwritten. ## Privacy diff --git a/src/inferrsi/data/dashboard/template.html b/src/inferrsi/data/dashboard/template.html index 0fe2b8b9e..ca961335f 100644 --- a/src/inferrsi/data/dashboard/template.html +++ b/src/inferrsi/data/dashboard/template.html @@ -2,6 +2,7 @@ + InferRSI · Optimization dashboard @@ -49,6 +50,7 @@ th, td { border-bottom:1px solid #292e36; padding:7px 10px; white-space:nowrap; } th { color:var(--muted); font-size:11px; font-weight:500; background:#1d2026; } tbody tr.selected { background:#233047; } td button { background:none; border:0; padding:0; color:#a9caff; text-align:left; } +.diagnostic-button { background:none; border:0; padding:0; color:#a9caff; font-weight:600; text-align:left; } .pill { display:inline-block; border:1px solid #596170; border-radius:2px; padding:1px 6px; font-size:11px; } .pill.good { color:var(--green); border-color:var(--green); } .pill.bad { color:var(--red); border-color:var(--red); } .pill.warn { color:var(--orange); border-color:var(--orange); } .pill.missing { border-style:dashed; color:#ff9ba7; } @@ -98,7 +100,7 @@

Stage coverage

Recorded events; not a time axis

Candidate details

Select a row to inspect its trace
-
RoundCandidateRunGainQualityOutcomeEvidence
+
RoundCandidateRunStateVerdictGainQualityOutcomeDecisionEvidence
@@ -123,7 +125,7 @@ const byId = id => document.getElementById(id); const stageNames = {diagnosis:'Diagnosis', knowledge:'Knowledge', search:'Candidate search', execution:'Execution', validation:'Validation', review:'Review'}; const stageSelect = byId('stage-filter'), runSelect = byId('run-filter'), candidateSelect = byId('candidate-filter'); - let selected = null; + let selected = null, jumpNotice = ''; const node = (tag, text, cls) => { const el = document.createElement(tag); if (text !== undefined) el.textContent = text; if (cls) el.className = cls; return el; }; const clear = el => { while (el.firstChild) el.removeChild(el.firstChild); }; const option = (parent, value, text) => { const el = node('option', text); el.value = value; parent.appendChild(el); }; @@ -155,7 +157,7 @@ const root=byId('stats'); clear(root); const rows=data.candidates, o=data.overview; const missing=rows.flatMap(missingRefs).length; const figures=[ - ['Best gain',value(o.best_improvement_pct,'%'),o.comparison_state === 'incompatible' ? 'Incompatible workloads' : 'Quality and source checks required','green'], + ['Best gain',value(o.best_improvement_pct,'%'),o.comparison_state === 'incompatible' ? 'Incompatible records' : o.comparison_state === 'unverified' ? 'Comparison evidence missing or unsupported' : 'Quality and source checks required','green'], ['Target gap',value(o.target_gap_pct,' pp'),o.target_improvement_pct === null ? 'Target not recorded' : `Target ${value(o.target_improvement_pct,'%')}`,'orange'], ['Candidate / run views',rows.length ? String(rows.length) : 'Not recorded','Recorded candidate or run views',''], ['Within noise',rows.some(r=>['noise','improved','regressed'].includes(r.outcome)) ? String(rows.filter(r => r.outcome === 'noise').length) : 'Not recorded','Uses recorded noise band','orange'], @@ -172,7 +174,7 @@ button.type='button'; button.setAttribute('aria-pressed',String(stageSelect.value === stage.name)); button.appendChild(node('strong',`${index+1} ${stageNames[stage.name]}`)); button.appendChild(node('span',stage.state.replaceAll('_',' '))); - button.addEventListener('click',()=>{stageSelect.value=stageSelect.value === stage.name ? '' : stage.name;render();}); root.appendChild(button); + button.addEventListener('click',()=>{jumpNotice='';stageSelect.value=stageSelect.value === stage.name ? '' : stage.name;render();}); root.appendChild(button); } } function renderTimeline(visible) { @@ -185,8 +187,8 @@ if(!visible.some(x=>x.index===selected)) selected=visible.length ? visible[0].index : null; for(const {row,index} of visible){const tr=node('tr');if(index===selected)tr.className='selected'; const cell=(text)=>{const td=node('td',text);tr.appendChild(td);return td;}; - cell(row.round === null ? 'Not recorded' : String(row.round));const title=cell();const btn=node('button',row.title);btn.type='button';btn.setAttribute('aria-label',`Inspect ${identity(row,index)}`);btn.addEventListener('click',()=>{selected=index;render();});title.appendChild(btn); - cell(row.run_id || 'Not recorded');cell(value(row.improvement_pct,'%'));cell().appendChild(pill(row.quality));const outcome=cell();outcome.appendChild(pill(row.outcome));if(row.decision==='rolled_back')outcome.appendChild(pill(row.decision)); + cell(row.round === null ? 'Not recorded' : String(row.round));const title=cell();const btn=node('button',row.title);btn.type='button';btn.setAttribute('aria-label',`Inspect ${identity(row,index)}`);btn.addEventListener('click',()=>{jumpNotice='';selected=index;render();});title.appendChild(btn); + cell(row.run_id || 'Not recorded');cell(row.state.replaceAll('_',' '));cell().appendChild(pill(row.verdict));cell(value(row.improvement_pct,'%'));cell().appendChild(pill(row.quality));cell().appendChild(pill(row.outcome));cell(row.decision.replaceAll('_',' ')); const refs=Object.values(row.artifacts);cell(`${refs.filter(r=>r.state==='present').length} present / ${refs.filter(r=>r.state==='missing').length} missing`);root.appendChild(tr);} } function renderDetail() { @@ -194,15 +196,19 @@ if(selected === null){byId('trace-title').textContent='No selection';item(trace,'NOT RECORDED','No matching candidate.');item(evidence,'NOT RECORDED');return;} const row=data.candidates[selected];byId('trace-title').textContent=identity(row,selected); const events=row.events.filter(e=>!stageSelect.value || e.stage===stageSelect.value); + if(jumpNotice)item(trace,'TRACE LOCATION',jumpNotice); if(!events.length)item(trace,'NOT RECORDED','No events for this stage.'); - for(const event of events){const name=event.type || event.phase || 'unclassified event';const details=[event.time,event.stage,event.side,event.trial===null?null:`trial ${event.trial}`,event.tool].filter(Boolean).join(' · ');const refs=event.refs.map(refText).join(' | ');item(trace,`${event.source}:${event.line} · ${name}`,details,refs);} + for(const event of events){const name=event.type || event.phase || 'unclassified event';const details=[event.time,event.stage,event.side,event.trial===null?null:`trial ${event.trial}`,event.tool].filter(Boolean).join(' · ');const refs=event.refs.map(refText).join(' | ');const entry=item(trace,`${event.source}:${event.line} · ${name}`,details,refs);entry.id=`trace-${event.source}-${event.line}`;} if(row.artifacts.tool_calls.state==='missing')item(trace,'MISSING · tool calls',row.artifacts.tool_calls.missing_reason).classList.add('missing-item'); + if(row.agent_trace_links.unlinked.length)item(trace,'UNLINKED AGENT EVENTS',`Lines ${row.agent_trace_links.unlinked.join(', ')}`).classList.add('missing-item'); + if(row.agent_trace_links.mismatched.length)item(trace,'MISMATCHED AGENT EVENTS',`Lines ${row.agent_trace_links.mismatched.join(', ')}`).classList.add('missing-item'); + for(const event of row.unlinked_agent_events){const detail=[event.time,event.stage].filter(Boolean).join(' · ');item(trace,`UNLINKED agent_trace:${event.line} · ${event.type || 'unclassified event'}`,detail,event.refs.map(refText).join(' | ')).classList.add('missing-item');} if(row.corrupt_event_lines.length)item(trace,'CORRUPT EVENT LINES',row.corrupt_event_lines.join(', ')); for(const ref of Object.values(row.artifacts)){const entry=item(evidence,ref.state === 'present' ? `${ref.kind} · PRESENT` : `${ref.kind} · MISSING`,ref.state === 'present' ? ref.path : ref.missing_reason,ref.sha256 ? `sha256 ${ref.sha256}` : '');if(ref.state==='missing')entry.classList.add('missing-item');} } - function renderDiagnostics(){const root=byId('diagnostics');clear(root);if(!data.diagnostics.length)item(root,'NOT RECORDED','No evidence-backed diagnostic rule matched.');for(const d of data.diagnostics)item(root,d.kind.replaceAll('_',' '),d.candidate_id || 'Single run',d.refs.map(refText).join(' | '));} + function renderDiagnostics(){const root=byId('diagnostics');clear(root);if(!data.diagnostics.length)item(root,'NOT RECORDED','No evidence-backed diagnostic rule matched.');for(const d of data.diagnostics){const entry=item(root,'',d.candidate_id || 'Single run',d.refs.map(refText).join(' | '));const button=node('button',d.kind.replaceAll('_',' '),'diagnostic-button');button.type='button';const matches=data.candidates.map((row,index)=>({row,index})).filter(({row})=>row.candidate_id===d.candidate_id && row.run_id===d.run_id && row.round===d.round);button.disabled=matches.length!==1;button.addEventListener('click',()=>{const {row,index}=matches[0];const phases=d.kind==='execution_failed'?['crashed','measurement']:d.kind==='quality_failed_with_gain'?['quality_validation']:d.kind==='rolled_back'?[]:['measurement'];let event=null;for(const phase of phases){event=[...row.events].reverse().find(e=>e.phase===phase);if(event)break;}runSelect.value='';candidateSelect.value=String(index);stageSelect.value=event && event.stage || '';selected=index;jumpNotice=event?'': 'No corresponding trace event was recorded; inspect the referenced evidence.';render();const target=event?byId(`trace-${event.source}-${event.line}`):byId('trace');if(target){target.setAttribute('tabindex','-1');target.focus({preventScroll:true});target.scrollIntoView({block:'center'});}});entry.querySelector('strong').replaceWith(button);}} function render(){const visible=filtered();renderWorkflow();renderTimeline(visible);renderRows(visible);renderDetail();} - for(const select of [stageSelect,runSelect,candidateSelect])select.addEventListener('change',render); + for(const select of [stageSelect,runSelect,candidateSelect])select.addEventListener('change',()=>{jumpNotice='';render();}); const redact = text => text.replace(/https?:\/\/\S+/gi,'[redacted-url]').replace(/\b[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}\b/g,'[redacted-email]').replace(/\b(?:\d{1,3}\.){3}\d{1,3}\b/g,'[redacted-address]').replace(/\b[A-Za-z]:[\\/][^\s]+/g,'[redacted-path]').replace(/(^|\s)\/(?:[^\s/]+\/)*[^\s]+/g,'$1[redacted-path]').replace(/\b(api[_-]?key|token|password|secret)\s*[:=]\s*\S+/gi,'$1=[redacted]'); byId('export').addEventListener('click',()=>{const row=selected===null?null:data.candidates[selected];const refs=row?Object.values(row.artifacts).map(refText).join('\n- '):'Not recorded';const events=row?row.events.filter(e=>!stageSelect.value || e.stage===stageSelect.value).map(e=>`${e.source}:${e.line}`).join(', '):'';const feedback=redact(`# InferRSI review\n\nCampaign: ${data.campaign_id}\nRound: ${row && row.round!==null?row.round:'Not recorded'}\nCandidate: ${row?identity(row,selected):'Not recorded'}\nStage: ${stageSelect.value || 'all'}\nTrace: ${events || 'Not recorded'}\n\n## Findings and recommendation\n\n${byId('feedback').value || 'Not recorded'}\n\n## Evidence references\n\n- ${refs}\n`);const preview=byId('feedback-preview');preview.textContent=feedback;preview.hidden=false;const url=URL.createObjectURL(new Blob([feedback],{type:'text/markdown'}));const link=node('a');link.href=url;link.download='inferrsi-review.md';document.body.appendChild(link);link.click();link.remove();setTimeout(()=>URL.revokeObjectURL(url),1000);}); renderStats();renderDiagnostics();render(); diff --git a/src/inferrsi/harness/dashboard.py b/src/inferrsi/harness/dashboard.py index 7db499209..c99d433c1 100644 --- a/src/inferrsi/harness/dashboard.py +++ b/src/inferrsi/harness/dashboard.py @@ -4,7 +4,9 @@ import json import math +import os import re +import tempfile from importlib.resources import files from pathlib import Path @@ -70,7 +72,11 @@ def _bounded_chain(spec: dict) -> bool: if not isinstance(axis, dict): return False step = _number(axis.get("step", 0.5)) - return step is not None and step > 0 and (max(ratios) - min(ratios)) / step <= 150 + if step is None or step <= 0 or max(ratios) + step == max(ratios): + return False + # The shared renderer adds a 1e-9 tolerance when iterating ticks. Include + # it in the bound, and leave room for floor/ceiling rounding. + return (max(ratios) - min(ratios) + 1e-9) / step + 3 <= 150 def _chart(source: Path, view: dict) -> tuple[str, str]: @@ -89,11 +95,11 @@ def _chart(source: Path, view: dict) -> tuple[str, str]: pass if spec is None: if view["overview"]["comparison_state"] != "comparable": - return "", "Candidate workloads or metrics cannot be compared." + return "", "Comparison evidence is missing, unsupported, or incompatible." trials = [] for index, row in enumerate(view["candidates"], 1): gain = row["improvement_pct"] - if gain is None or 1 + gain / 100 <= 0: + if gain is None: continue status = {"improved": "gain", "noise": "flat", "regressed": "regressed"}.get( row["outcome"], "open" @@ -105,7 +111,7 @@ def _chart(source: Path, view: dict) -> tuple[str, str]: return "", "No measured candidate results recorded." target = view["overview"]["target_improvement_pct"] gains = [row["improvement_pct"] for row in view["candidates"] - if row["improvement_pct"] is not None and 1 + row["improvement_pct"] / 100 > 0] + if row["improvement_pct"] is not None] if target is not None and 1 + target / 100 > 0: gains.append(target) spread = max([0.0, *gains]) - min([0.0, *gains]) @@ -122,7 +128,7 @@ def _chart(source: Path, view: dict) -> tuple[str, str]: } try: svg = build_svg(spec) - except (SpecError, TypeError, ValueError, OverflowError): + except (SpecError, TypeError, ValueError, OverflowError, ZeroDivisionError): return "", "The recorded chart specification is invalid or incomplete." # The shared renderer is unchanged. Adapt its fixed light palette for this panel. for light, dark in { @@ -157,5 +163,26 @@ def write_dashboard(source: Path, destination: Path | None = None) -> Path: ) output = Path(destination) if destination is not None else source / "dashboard.html" output.parent.mkdir(parents=True, exist_ok=True) - output.write_text(document, encoding="utf-8") + if output.is_symlink(): + raise ValueError("dashboard output cannot be a symbolic link") + if output.exists(): + if not output.is_file(): + raise ValueError("dashboard output is not a regular file") + if output.suffix.lower() != ".html": + raise ValueError("refusing to overwrite an existing non-dashboard file") + with output.open("r", encoding="utf-8", errors="replace") as existing: + if '' not in existing.read(1024): + raise ValueError("refusing to overwrite an existing non-dashboard file") + temporary = None + try: + with tempfile.NamedTemporaryFile( + mode="w", encoding="utf-8", dir=output.parent, prefix=".dashboard-", + suffix=".tmp", delete=False, + ) as stream: + temporary = Path(stream.name) + stream.write(document) + os.replace(temporary, output) + finally: + if temporary is not None and temporary.exists(): + temporary.unlink() return output.resolve() diff --git a/src/inferrsi/harness/dashboard_data.py b/src/inferrsi/harness/dashboard_data.py index 5cde2a5d3..191e9f625 100644 --- a/src/inferrsi/harness/dashboard_data.py +++ b/src/inferrsi/harness/dashboard_data.py @@ -56,16 +56,40 @@ def _inside(root: Path, path: Path) -> bool: return False -def _object(path: Path, root: Path) -> tuple[dict, str]: - if not _inside(root, path): - return {}, "unsafe" +def _snapshot(root: Path, path: Path, kind: str) -> tuple[bytes | None, dict]: + try: + relative = path.resolve().relative_to(root.resolve()).as_posix() + except (OSError, ValueError): + return None, {"kind": kind, "state": "missing", "missing_reason": "unsafe_path"} try: - data = json.loads(path.read_text(encoding="utf-8")) + data = path.read_bytes() except FileNotFoundError: - return {}, "missing" - except (OSError, UnicodeError, json.JSONDecodeError): - return {}, "corrupt" - return (data, "present") if isinstance(data, dict) else ({}, "corrupt") + return None, {"kind": kind, "path": relative, "state": "missing", + "missing_reason": "file_absent"} + except OSError: + return None, {"kind": kind, "path": relative, "state": "missing", + "missing_reason": "unreadable"} + return data, {"kind": kind, "path": relative, "state": "present", + "sha256": hashlib.sha256(data).hexdigest()} + + +def _object_snapshot(path: Path, root: Path, kind: str) -> tuple[dict, str, dict]: + raw, ref = _snapshot(root, path, kind) + if raw is None: + state = "unsafe" if ref["missing_reason"] == "unsafe_path" else "missing" + return {}, state, ref + try: + data = json.loads(raw.decode("utf-8")) + if not isinstance(data, dict): + raise ValueError("object required") + except (UnicodeError, json.JSONDecodeError, ValueError): + return {}, "corrupt", {**ref, "state": "missing", "missing_reason": "corrupt"} + return data, "present", ref + + +def _object(path: Path, root: Path) -> tuple[dict, str]: + data, state, _ = _object_snapshot(path, root, "record") + return data, state def _file_ref(root: Path, path: Path, kind: str) -> dict: @@ -115,15 +139,17 @@ def _explicit_ref(root: Path, value: object, kind: str) -> dict: return ref -def _events(path: Path, root: Path) -> tuple[list[dict], list[int], str]: - if not _inside(root, path): - return [], [], "unsafe" +def _events(path: Path, root: Path, expected: dict | None = None + ) -> tuple[list[dict], list[int], str, dict[str, list[int]], dict]: + unlinked = {"unlinked": [], "mismatched": []} + raw, ref = _snapshot(root, path, "events") + if raw is None: + state = "unsafe" if ref["missing_reason"] == "unsafe_path" else "missing" + return [], [], state, unlinked, ref try: - lines = path.read_text(encoding="utf-8").splitlines() - except FileNotFoundError: - return [], [], "missing" - except (OSError, UnicodeError): - return [], [], "corrupt" + lines = raw.decode("utf-8").splitlines() + except UnicodeError: + return [], [], "corrupt", unlinked, {**ref, "state": "missing", "missing_reason": "corrupt"} events, corrupt = [], [] for line_number, line in enumerate(lines, 1): try: @@ -133,12 +159,34 @@ def _events(path: Path, root: Path) -> tuple[list[dict], list[int], str]: except (json.JSONDecodeError, ValueError): corrupt.append(line_number) continue + link_state = "linked" + if expected is not None: + identity = {key: record.get(key) for key in ( + "campaign_id", "candidate_id", "run_id", "round" + )} + valid = (type(record.get("schema_version")) is int + and record["schema_version"] == 1 + and all(isinstance(identity[key], str) and identity[key] + for key in ("campaign_id", "candidate_id", "run_id")) + and type(identity["round"]) is int and identity["round"] > 0 + and record.get("stage") in STAGES + and isinstance(record.get("time"), str) and bool(record["time"])) + if any(value is not None and expected[key] is not None and value != expected[key] + for key, value in identity.items()): + unlinked["mismatched"].append(line_number) + continue + if not valid or any(value is None for value in expected.values()): + unlinked["unlinked"].append(line_number) + link_state = "unlinked" phase = record.get("phase") if phase not in (*PHASE_STAGES, "complete", "crashed"): phase = None kind = record.get("type") event_type = kind if isinstance(kind, str) and kind in EVENT_TYPES else None - stage = record.get("stage") if record.get("stage") in STAGES else PHASE_STAGES.get(phase) + stage = None + if phase is not None or event_type is not None: + stage = (record.get("stage") if record.get("stage") in STAGES + else PHASE_STAGES.get(phase)) time = record.get("time") if not isinstance(time, str) or not re.fullmatch(r"[0-9TtZz:+.\-]{10,40}", time): time = None @@ -151,31 +199,78 @@ def _events(path: Path, root: Path) -> tuple[list[dict], list[int], str]: and 0 < trial <= 1_000_000 else None, "tool": record.get("name") if record.get("name") in ("read_file", "submit_answer") else None, + "_declared_refs": record.get("refs") if isinstance(record.get("refs"), list) else [], + "_link_state": link_state, }) - return events, corrupt, "present" + return events, corrupt, "present", unlinked, ref -def _run(root: Path, path: Path, candidate: dict, run_id: str | None, noise: float | None) -> dict: +def _environment_signature(environment: dict, devices: object, backend: object) -> tuple | None: + """Compare verified device/runtime identity privately; never expose it in the view.""" + if backend != "cuda" or not isinstance(devices, list) or not devices: + return None + device = environment.get("device") + host = environment.get("host") + target = environment.get("target_python") + target = target.get("environment") if isinstance(target, dict) else None + if (not isinstance(device, dict) or device.get("status") != "available" + or device.get("selection_verified") is not True + or not isinstance(device.get("devices"), list) + or not isinstance(host, dict) or not isinstance(host.get("platform"), str) + or not host["platform"] or not isinstance(target, dict) + or not isinstance(target.get("python"), str) or not target["python"] + or not isinstance(target.get("packages"), dict)): + return None + packages = target["packages"] + if not all(isinstance(key, str) and (value is None or isinstance(value, str)) + for key, value in packages.items()): + return None + if not all(isinstance(packages.get(key), str) and packages[key] for key in ("torch", "vllm")): + return None + selected = [] + for identity in devices: + matches = [row for row in device["devices"] if isinstance(row, dict) + and identity in (row.get("index"), row.get("uuid"))] + if len(matches) != 1: + return None + row = matches[0] + fields = tuple(row.get(key) for key in ("uuid", "name", "driver_version")) + if not all(isinstance(value, str) and value for value in fields): + return None + selected.append(fields) + return (host["platform"], target["python"], tuple(sorted(packages.items())), tuple(selected)) + + +def _run(root: Path, path: Path, candidate: dict, run_id: str | None, noise: float | None, + campaign_id: object = None) -> dict: lock, _ = _object(path / "run.lock.json", root) repositories, _ = _object(path / "repositories.lock.json", root) - evaluation, evaluation_state = _object(path / "evaluation.json", root) - status, status_state = _object(path / "status.json", root) - validation, validation_state = _object(path / "validation" / "execution.json", root) - events, corrupt, event_state = _events(path / "events.jsonl", root) + environment, _ = _object(path / "environment.json", root) + evaluation, evaluation_state, evaluation_ref = _object_snapshot( + path / "evaluation.json", root, "evaluation" + ) + status, status_state, status_ref = _object_snapshot(path / "status.json", root, "status") + validation, validation_state, quality_ref = _object_snapshot( + path / "validation" / "execution.json", root, "quality" + ) + events, corrupt, event_state, _, events_ref = _events(path / "events.jsonl", root) refs = { - "events": _file_ref(root, path / "events.jsonl", "events"), - "evaluation": _file_ref(root, path / "evaluation.json", "evaluation"), - "status": _file_ref(root, path / "status.json", "status"), - "quality": _file_ref(root, path / "validation" / "execution.json", "quality"), + "events": events_ref, + "evaluation": evaluation_ref, + "status": status_ref, + "quality": quality_ref, "agent_trace": _explicit_ref(root, candidate.get("agent_trace_ref"), "agent_trace"), "profile": _explicit_ref(root, candidate.get("profile_ref"), "profile"), } - for key, parse_state in (("evaluation", evaluation_state), ("status", status_state), - ("quality", validation_state), ("events", event_state)): - if parse_state in {"corrupt", "unsafe"}: - refs[key] = {**refs[key], "state": "missing", "missing_reason": - "unsafe_path" if parse_state == "unsafe" else "corrupt"} + def attach_declared(event: dict) -> None: + for item in event.pop("_declared_refs", []): + kind = item.get("kind") if isinstance(item, dict) else None + if not isinstance(kind, str) or not re.fullmatch(r"[A-Za-z0-9_-]{1,40}", kind): + kind = "artifact" + event["refs"].append(_explicit_ref(root, item, kind)) + for event in events: + event.pop("_link_state", None) event["source"] = "events" event["refs"] = [refs["events"]] if event["phase"] == "measurement" and event["side"] and event["trial"]: @@ -187,16 +282,33 @@ def _run(root: Path, path: Path, candidate: dict, run_id: str | None, noise: flo ) elif event["phase"] == "quality_validation": event["refs"].append(refs["quality"]) + attach_declared(event) + agent_links = {"unlinked": [], "mismatched": []} + unlinked_events = [] if refs["agent_trace"]["state"] == "present": - agent_events, agent_corrupt, agent_state = _events(root / refs["agent_trace"]["path"], root) - if agent_state in {"corrupt", "unsafe"}: + expected = {"campaign_id": campaign_id, "candidate_id": candidate.get("candidate_id"), + "run_id": run_id, "round": candidate.get("round")} + agent_events, agent_corrupt, agent_state, agent_links, agent_snapshot = _events( + root / refs["agent_trace"]["path"], root, expected + ) + if (agent_state == "present" and + agent_snapshot.get("sha256") != refs["agent_trace"].get("sha256")): + agent_state = "changed" + if agent_state != "present": refs["agent_trace"] = {**refs["agent_trace"], "state": "missing", "missing_reason": "unsafe_path" if agent_state == "unsafe" - else "corrupt"} + else "hash_mismatch" if agent_state == "changed" + else "file_absent" if agent_state == "missing" else "corrupt"} + agent_events = [] + agent_links = {"unlinked": [], "mismatched": []} for event in agent_events: event["source"] = "agent_trace" event["refs"] = [refs["agent_trace"]] - events.extend(agent_events) + attach_declared(event) + if event.pop("_link_state") == "unlinked": + unlinked_events.append(event) + else: + events.append(event) corrupt.extend(agent_corrupt) tool_events = [event for event in events if event["source"] == "agent_trace" and event["type"] == "agent.tool"] @@ -225,12 +337,16 @@ def _run(root: Path, path: Path, candidate: dict, run_id: str | None, noise: flo state = state_value if isinstance(state_value, str) and state_value in { "running", "completed", "failed", "crashed", "timeout", "cancelled", "termination_failed" } else "not_recorded" + verdict_value = evaluation.get("verdict") + verdict = verdict_value if verdict_value in ("PASS", "FAIL", "INCONCLUSIVE") else "not_recorded" if quality == "failed": outcome = "quality_failed" elif state in {"failed", "crashed", "timeout", "cancelled", "termination_failed"}: outcome = "failed" elif gain is None: outcome = "not_recorded" + elif state != "completed" or verdict not in ("PASS", "FAIL"): + outcome = "unverified" elif quality != "passed" or not verified: outcome = "unverified" elif noise is None: @@ -246,13 +362,17 @@ def _run(root: Path, path: Path, candidate: dict, run_id: str | None, noise: flo repository = repositories.get("baseline") snapshot = repository.get("snapshot_hash") if isinstance(repository, dict) else None command, devices = lock.get("baseline_command"), lock.get("device_ids") + validator = lock.get("validation_command") + environment_signature = _environment_signature(environment, devices, lock.get("backend")) valid_sequences = all( isinstance(seq, list) and seq and all(isinstance(item, str) and item for item in seq) - for seq in (command, devices) + for seq in (command, devices, validator) ) - signature = (tuple(signature_values) + (snapshot, tuple(command), tuple(devices))) if ( + signature = (tuple(signature_values) + (snapshot, tuple(command), tuple(devices), + tuple(validator), environment_signature)) if ( all(isinstance(value, str) and value for value in signature_values) and isinstance(snapshot, str) and snapshot and valid_sequences + and environment_signature is not None ) else None return { "candidate_id": _text(candidate_id) if isinstance(candidate_id, str) else None, @@ -270,22 +390,22 @@ def _run(root: Path, path: Path, candidate: dict, run_id: str | None, noise: flo "metric_unit": _text(evaluation.get("metric_unit")), "direction": evaluation.get("direction") if evaluation.get("direction") in ("minimize", "maximize") else None, - "verdict": evaluation.get("verdict") - if evaluation.get("verdict") in ("PASS", "FAIL", "INCONCLUSIVE") else "not_recorded", + "verdict": verdict, "outcome": outcome, "events": events, "corrupt_event_lines": corrupt, + "agent_trace_links": agent_links, "unlinked_agent_events": unlinked_events, "artifacts": refs, "_comparison_signature": signature, } -def _rounds(path: Path, root: Path) -> tuple[list[dict] | None, str, list[int]]: - if not _inside(root, path): - return None, "unsafe", [] +def _rounds(path: Path, root: Path) -> tuple[list[dict] | None, str, list[int], dict]: + raw, ref = _snapshot(root, path, "rounds") + if raw is None: + state = "unsafe" if ref["missing_reason"] == "unsafe_path" else "missing" + return None, state, [], ref try: - lines = path.read_text(encoding="utf-8").splitlines() - except FileNotFoundError: - return None, "missing", [] - except (OSError, UnicodeError): - return None, "corrupt", [] + lines = raw.decode("utf-8").splitlines() + except UnicodeError: + return None, "corrupt", [], {**ref, "state": "missing", "missing_reason": "corrupt"} result, corrupt = [], [] for line_number, line in enumerate(lines, 1): try: @@ -306,7 +426,7 @@ def _rounds(path: Path, root: Path) -> tuple[list[dict] | None, str, list[int]]: result.append(record) else: corrupt.append(line_number) - return result, "corrupt" if corrupt else "present", corrupt + return result, "corrupt" if corrupt else "present", corrupt, ref def load_dashboard(source: Path) -> dict: @@ -339,13 +459,14 @@ def load_dashboard(source: Path) -> dict: or run_id in {".", ".."}): run_id = None run_path = root / "runs" / run_id if run_id else directory / "missing-run" - rows.append(_run(root, run_path, metadata, run_id, noise)) + rows.append(_run(root, run_path, metadata, run_id, noise, + campaign.get("campaign_id"))) if not rows and _inside(root, root / "runs") and (root / "runs").is_dir(): for directory in sorted((root / "runs").iterdir()): if directory.is_dir() and not directory.is_symlink(): rows.append(_run(root, directory, {}, directory.name, noise)) - rounds, rounds_state, corrupt_round_lines = ( - (None, "missing", []) if single else _rounds(root / "rounds.jsonl", root) + rounds, rounds_state, corrupt_round_lines, rounds_ref = ( + (None, "missing", [], None) if single else _rounds(root / "rounds.jsonl", root) ) decisions = {(item["round"], item["decision"]["candidate_id"]): item["decision"]["outcome"] for item in (rounds or []) if "decision" in item} @@ -356,15 +477,17 @@ def load_dashboard(source: Path) -> dict: signature = row.pop("_comparison_signature") if row["improvement_pct"] is not None: signatures.append(signature) - comparable = len(signatures) < 2 or ( - all(signature is not None for signature in signatures) - and len(set(signatures)) == 1 - ) + comparison_state = "comparable" + if len(signatures) >= 2: + if any(signature is None for signature in signatures): + comparison_state = "unverified" + elif len(set(signatures)) != 1: + comparison_state = "incompatible" valid_gains = [row["improvement_pct"] for row in rows if row["improvement_pct"] is not None and row["quality"] == "passed" and row["source_verified"] is True and row["state"] == "completed" and row["verdict"] in ("PASS", "FAIL") and row["decision"] != "rolled_back"] - if not comparable: + if comparison_state != "comparable": valid_gains = [] best = max(valid_gains) if valid_gains else None goal = campaign.get("goal") if isinstance(campaign.get("goal"), dict) else {} @@ -373,26 +496,24 @@ def load_dashboard(source: Path) -> dict: target = rows[0]["minimum_improvement_pct"] diagnostics = [] for row in rows: + identity = {"candidate_id": row["candidate_id"], "run_id": row["run_id"], + "round": row["round"]} if row["outcome"] == "failed" and row["artifacts"]["status"]["state"] == "present": - diagnostics.append({"kind": "execution_failed", "candidate_id": row["candidate_id"], + diagnostics.append({**identity, "kind": "execution_failed", "refs": [row["artifacts"]["status"]]}) elif row["outcome"] == "quality_failed" and row["improvement_pct"] is not None \ and row["improvement_pct"] > 0: refs = [row["artifacts"][key] for key in ("evaluation", "quality")] if all(ref["state"] == "present" for ref in refs): - diagnostics.append({"kind": "quality_failed_with_gain", - "candidate_id": row["candidate_id"], "refs": refs}) + diagnostics.append({**identity, "kind": "quality_failed_with_gain", "refs": refs}) if row["outcome"] in {"noise", "regressed"}: ref = row["artifacts"]["evaluation"] if ref["state"] == "present": - diagnostics.append({"kind": "within_noise" if row["outcome"] == "noise" - else "measured_regression", "candidate_id": row["candidate_id"], - "refs": [ref]}) + diagnostics.append({**identity, "kind": "within_noise" if row["outcome"] == "noise" + else "measured_regression", "refs": [ref]}) if row["decision"] == "rolled_back": - ref = _file_ref(root, root / "rounds.jsonl", "rounds") - if ref["state"] == "present": - diagnostics.append({"kind": "rolled_back", "candidate_id": row["candidate_id"], - "refs": [ref]}) + if rounds_ref and rounds_ref["state"] == "present": + diagnostics.append({**identity, "kind": "rolled_back", "refs": [rounds_ref]}) recorded = {event["stage"] for row in rows for event in row["events"] if event["stage"]} declared = campaign.get("stages_declared") if not isinstance(declared, list): @@ -403,7 +524,7 @@ def load_dashboard(source: Path) -> dict: "rounds": rounds, "rounds_state": rounds_state, "corrupt_round_lines": corrupt_round_lines, "candidates": rows, - "overview": {"comparison_state": "comparable" if comparable else "incompatible", + "overview": {"comparison_state": comparison_state, "best_improvement_pct": best, "target_improvement_pct": target, "target_gap_pct": target - best if target is not None and best is not None else None}, diff --git a/src/inferrsi/harness/dashboard_timeline.py b/src/inferrsi/harness/dashboard_timeline.py index f7c570ba4..c2a89d241 100644 --- a/src/inferrsi/harness/dashboard_timeline.py +++ b/src/inferrsi/harness/dashboard_timeline.py @@ -127,7 +127,8 @@ def text(px: float, py: float, label: str, color: str = "#aab2bf", size: int = 1 gain, outcome = row["improvement_pct"], row["outcome"] if gain is not None and not comparable: continue - if gain is None and outcome != "failed": + if gain is None and outcome not in ("failed", "quality_failed") \ + and row["decision"] != "rolled_back": continue px, py = x(stamp), y(gain) if gain is not None else 242 label = f'{row["title"]}: {outcome.replace("_", " ")}' @@ -157,14 +158,17 @@ def text(px: float, py: float, label: str, color: str = "#aab2bf", size: int = 1 px = x(stamp) svg.append(f'') - label = datetime.fromtimestamp(stamp, timezone.utc).strftime("%Y-%m-%d %H:%M:%SZ") + try: + label = datetime.fromtimestamp(stamp, timezone.utc).strftime("%Y-%m-%d %H:%M:%SZ") + except (ValueError, OverflowError, OSError): + return "", "Recorded timestamps exceed the supported time-chart range." text(max(left, min(px - 60, right - 120)), 416, label) svg.append('') note = "Terminal events place results; stage dots are events, not durations." if untimed_results: note += f" {untimed_results} result(s) have no recorded terminal timestamp." if not comparable: - note += " Incompatible candidate measurements are not plotted together." + note += " Unverified or incompatible candidate measurements are not plotted together." elif measured and noise is None: note += " No common noise band recorded." return "".join(svg), note diff --git a/tests/test_campaign_records.py b/tests/test_campaign_records.py index 1bd2f6550..9f6f62eda 100644 --- a/tests/test_campaign_records.py +++ b/tests/test_campaign_records.py @@ -17,7 +17,10 @@ def test_record_candidates_in_one_round_and_reference_trace(tmp_path): (run / "run.lock.json").write_text("{}", encoding="utf-8") trace = tmp_path / "candidates/a/trace.jsonl" trace.parent.mkdir(parents=True) - trace.write_text('{"type":"agent.tool","stage":"search"}\n', encoding="utf-8") + trace.write_text(json.dumps({"type": "agent.tool", "stage": "search", "schema_version": 1, + "campaign_id": "campaign", "candidate_id": "a", "run_id": "a", + "round": 1, "time": "2026-01-01T00:00:00Z"}) + "\n", + encoding="utf-8") record_candidate(tmp_path, candidate_id="a", run_id="a", round_number=1, title="First", agent_trace="candidates/a/trace.jsonl") record_candidate(tmp_path, candidate_id="b", run_id="b", round_number=1, title="Second") diff --git a/tests/test_dashboard_data.py b/tests/test_dashboard_data.py index de1c27b27..2b5dfaf2b 100644 --- a/tests/test_dashboard_data.py +++ b/tests/test_dashboard_data.py @@ -1,11 +1,13 @@ """Dashboard aggregation uses recorded evidence and keeps missing data explicit.""" +import hashlib import json from pathlib import Path import pytest -from inferrsi.harness.dashboard import write_dashboard +from inferrsi.harness import dashboard_data +from inferrsi.harness.dashboard import _bounded_chain, _chart, write_dashboard from inferrsi.harness.dashboard_data import load_dashboard from inferrsi.harness.dashboard_timeline import temporal_chart @@ -21,9 +23,19 @@ def run(root: Path, name: str, gain: float, *, quality: str, verified: bool) -> "model": "example-model", "model_revision": "example-revision", "workload_id": "example-workload", "scenario": "http", "backend": "cuda", "comparison_scope": "http_e2e", "baseline_command": ["python", "bench.py"], - "device_ids": ["0"], + "validation_command": ["python", "validate.py"], "device_ids": ["0"], }) write_json(directory / "repositories.lock.json", {"baseline": {"snapshot_hash": "a" * 64}}) + write_json(directory / "environment.json", { + "host": {"platform": "synthetic-os"}, + "target_python": {"environment": {"python": "synthetic-python", "packages": { + "vllm": "synthetic-version", "torch": "synthetic-version", + }}}, + "device": {"status": "available", "selection_verified": True, "devices": [{ + "index": "0", "uuid": "synthetic-device", "name": "synthetic-gpu", + "driver_version": "synthetic-driver", + }]}, + }) write_json(directory / "status.json", {"state": "completed", "run_dir": "/private/run"}) write_json(directory / "evaluation.json", { "verdict": "PASS", "metric_key": "throughput", "metric_unit": "tokens/s", @@ -145,6 +157,61 @@ def test_campaign_does_not_compare_different_baseline_snapshots(tmp_path): assert load_dashboard(tmp_path)["overview"]["comparison_state"] == "incompatible" +@pytest.mark.parametrize("field", ["device", "model", "driver", "runtime", "validator", + "missing_environment", "unsupported_backend"]) +def test_campaign_does_not_compare_different_or_unverified_environments(tmp_path, field): + write_json(tmp_path / "campaign.json", {"campaign_id": "example"}) + for name in ("a", "b"): + run(tmp_path, name, 5, quality="completed", verified=True) + candidate(tmp_path, name, name) + environment_path = tmp_path / "runs/b/environment.json" + environment = json.loads(environment_path.read_text(encoding="utf-8")) + if field == "device": + environment["device"]["devices"][0]["uuid"] = "different-synthetic-device" + elif field == "model": + environment["device"]["devices"][0]["name"] = "different-synthetic-gpu" + elif field == "driver": + environment["device"]["devices"][0]["driver_version"] = "different-driver" + elif field == "runtime": + environment["target_python"]["environment"]["packages"]["vllm"] = "different-version" + elif field in ("validator", "unsupported_backend"): + lock = tmp_path / "runs/b/run.lock.json" + data = json.loads(lock.read_text(encoding="utf-8")) + if field == "validator": + data["validation_command"] = ["python", "other-validator.py"] + else: + data["backend"] = "cpu" + write_json(lock, data) + if field == "missing_environment": + environment_path.unlink() + else: + write_json(environment_path, environment) + + view = load_dashboard(tmp_path) + + assert view["overview"]["comparison_state"] == ( + "unverified" if field in ("missing_environment", "unsupported_backend") else "incompatible" + ) + assert view["overview"]["best_improvement_pct"] is None + assert "synthetic-device" not in json.dumps(view) + + +@pytest.mark.parametrize("state,verdict", [("running", "PASS"), ("completed", "INCONCLUSIVE")]) +def test_unfinished_or_inconclusive_run_is_not_marked_improved(tmp_path, state, verdict): + run(tmp_path, "a", 10, quality="completed", verified=True) + write_json(tmp_path / "runs/a/status.json", {"state": state}) + evaluation = tmp_path / "runs/a/evaluation.json" + data = json.loads(evaluation.read_text(encoding="utf-8")) + data["verdict"] = verdict + write_json(evaluation, data) + + row = load_dashboard(tmp_path / "runs/a")["candidates"][0] + + assert row["outcome"] == "unverified" + assert row["state"] == state + assert row["verdict"] == verdict + + def test_unmeasured_failed_candidate_does_not_hide_other_recorded_gains(tmp_path): write_json(tmp_path / "campaign.json", {"campaign_id": "example"}) run(tmp_path, "a", 5, quality="completed", verified=True) @@ -195,6 +262,113 @@ def test_unsafe_and_mismatched_artifact_refs_are_not_followed(tmp_path): assert "private contents" not in json.dumps(view) +def test_trace_requires_matching_correlation_and_unknown_events_do_not_cover_a_stage(tmp_path): + write_json(tmp_path / "campaign.json", {"campaign_id": "campaign"}) + run(tmp_path, "run-a", 1, quality="completed", verified=True) + trace = tmp_path / "candidates/a/agent_trace.jsonl" + trace.parent.mkdir(parents=True) + common = {"schema_version": 1, "campaign_id": "campaign", "candidate_id": "a", + "run_id": "run-a", "round": 1, "stage": "search", + "time": "2026-01-01T00:00:00Z"} + records = [ + {**common, "type": "agent.request"}, + {**common, "type": "agent.tool", "candidate_id": "other"}, + {"type": "agent.tool", "stage": "validation"}, + {**common, "type": "unknown", "stage": "validation"}, + ] + trace.write_text("".join(json.dumps(item) + "\n" for item in records), encoding="utf-8") + candidate(tmp_path, "a", "run-a", round=1, agent_trace_ref={ + "path": "candidates/a/agent_trace.jsonl", + "sha256": hashlib.sha256(trace.read_bytes()).hexdigest(), + }) + + view = load_dashboard(tmp_path) + row = view["candidates"][0] + + assert row["agent_trace_links"] == {"unlinked": [3], "mismatched": [2]} + assert [event["line"] for event in row["unlinked_agent_events"]] == [3] + assert row["artifacts"]["tool_calls"]["state"] == "missing" + assert [event["line"] for event in row["events"]] == [1, 4] + assert next(stage for stage in view["stages"] if stage["name"] == "validation")["state"] == ( + "stage_not_recorded" + ) + + +def test_trace_explicit_refs_are_checked_and_displayed(tmp_path): + write_json(tmp_path / "campaign.json", {"campaign_id": "campaign"}) + run(tmp_path, "run-a", 1, quality="completed", verified=True) + profile = tmp_path / "profile.json" + profile.write_text('{"synthetic":true}', encoding="utf-8") + profile_hash = hashlib.sha256(profile.read_bytes()).hexdigest() + trace = tmp_path / "candidates/a/agent_trace.jsonl" + trace.parent.mkdir(parents=True) + event = {"type": "agent.tool", "schema_version": 1, "campaign_id": "campaign", + "candidate_id": "a", "run_id": "run-a", "round": 1, "stage": "search", + "time": "2026-01-01T00:00:00Z", "refs": [ + {"kind": "profile", "path": "profile.json", "sha256": profile_hash}, + {"kind": "log", "path": "../private", "sha256": "0" * 64}, + ]} + trace.write_text(json.dumps(event) + "\n", encoding="utf-8") + candidate(tmp_path, "a", "run-a", round=1, agent_trace_ref={ + "path": "candidates/a/agent_trace.jsonl", + "sha256": hashlib.sha256(trace.read_bytes()).hexdigest(), + }) + + refs = load_dashboard(tmp_path)["candidates"][0]["events"][0]["refs"] + + assert any(ref.get("kind") == "profile" and ref.get("sha256") == profile_hash for ref in refs) + assert any(ref.get("kind") == "log" and ref.get("missing_reason") == "unsafe_path" + for ref in refs) + + +def test_record_hash_matches_bytes_used_for_rendering_even_if_file_changes(tmp_path, monkeypatch): + write_json(tmp_path / "run.lock.json", {}) + status = tmp_path / "status.json" + old = b'{"state":"completed"}' + status.write_bytes(old) + original = dashboard_data._snapshot + + def changing_snapshot(root, path, kind): + data, ref = original(root, path, kind) + if path == status: + status.write_text('{"state":"failed"}', encoding="utf-8") + return data, ref + + monkeypatch.setattr(dashboard_data, "_snapshot", changing_snapshot) + row = load_dashboard(tmp_path)["candidates"][0] + + assert row["state"] == "completed" + assert row["artifacts"]["status"]["sha256"] == hashlib.sha256(old).hexdigest() + + +def test_trace_changed_after_reference_check_is_not_linked(tmp_path, monkeypatch): + write_json(tmp_path / "campaign.json", {"campaign_id": "campaign"}) + run(tmp_path, "run-a", 1, quality="completed", verified=True) + trace = tmp_path / "candidates/a/agent_trace.jsonl" + trace.parent.mkdir(parents=True) + event = {"type": "agent.tool", "schema_version": 1, "campaign_id": "campaign", + "candidate_id": "a", "run_id": "run-a", "round": 1, "stage": "search", + "time": "2026-01-01T00:00:00Z"} + trace.write_text(json.dumps(event) + "\n", encoding="utf-8") + candidate(tmp_path, "a", "run-a", round=1, agent_trace_ref={ + "path": "candidates/a/agent_trace.jsonl", + "sha256": hashlib.sha256(trace.read_bytes()).hexdigest(), + }) + original = dashboard_data._snapshot + + def changing_snapshot(root, path, kind): + if path == trace: + trace.write_text(json.dumps({**event, "candidate_id": "other"}) + "\n", + encoding="utf-8") + return original(root, path, kind) + + monkeypatch.setattr(dashboard_data, "_snapshot", changing_snapshot) + row = load_dashboard(tmp_path)["candidates"][0] + + assert row["artifacts"]["agent_trace"]["missing_reason"] == "hash_mismatch" + assert not any(event["source"] == "agent_trace" for event in row["events"]) + + def test_run_symlink_outside_campaign_is_not_read(tmp_path): root = tmp_path / "campaign" outside = tmp_path / "outside" @@ -294,6 +468,53 @@ def test_unbounded_chart_axis_is_rejected(tmp_path): assert 'No measured candidate results recorded.' in html +def test_branch_chart_preserves_large_latency_regression(tmp_path): + write_json(tmp_path / "campaign.json", {"campaign_id": "example"}) + run(tmp_path, "a", -200, quality="completed", verified=True) + candidate(tmp_path, "a", "a", title="large-regression") + evaluation = tmp_path / "runs/a/evaluation.json" + data = json.loads(evaluation.read_text(encoding="utf-8")) + data["direction"] = "minimize" + write_json(evaluation, data) + + svg, _ = _chart(tmp_path, load_dashboard(tmp_path)) + + assert "large-regression" in svg + assert "-200" in svg + + +@pytest.mark.parametrize("point,step", [(1.0000000000000002, 1e-17), (1.0, 1e-100)]) +def test_chain_axis_cannot_stall_or_expand_from_float_rounding(point, step): + assert not _bounded_chain({ + "layout": "chain", "baseline": {"metrics": {"gain": 1}}, + "metrics": [{"key": "gain"}], "axis": {"step": step}, + "points": [{"phase": "a", "label": "x", "values": {"gain": point}}], + }) + + +def test_export_does_not_overwrite_evidence_or_follow_output_symlinks(tmp_path): + write_json(tmp_path / "run.lock.json", {}) + evidence = tmp_path / "evaluation.json" + evidence.write_text('{"verdict":"PASS"}', encoding="utf-8") + with pytest.raises(ValueError, match="non-dashboard"): + write_dashboard(tmp_path, evidence) + assert evidence.read_text(encoding="utf-8") == '{"verdict":"PASS"}' + output = tmp_path / "dashboard.html" + try: + output.symlink_to(evidence) + except OSError: + pytest.skip("file symlinks are unavailable") + with pytest.raises(ValueError, match="symbolic link"): + write_dashboard(tmp_path) + assert evidence.read_text(encoding="utf-8") == '{"verdict":"PASS"}' + + +def test_dashboard_can_be_regenerated(tmp_path): + write_json(tmp_path / "run.lock.json", {}) + output = write_dashboard(tmp_path) + assert write_dashboard(tmp_path) == output + + def test_time_chart_uses_terminal_events_and_marks_quality_failure_and_missing_values(tmp_path): write_json(tmp_path / "campaign.json", { "campaign_id": "example", "goal": {"target_improvement_pct": 20}, @@ -346,7 +567,7 @@ def test_time_chart_does_not_invent_timestamps_or_compare_incompatible_runs(tmp_ svg, note = temporal_chart(load_dashboard(tmp_path)) assert 'data-kind="marker-improved"' not in svg - assert 'Incompatible candidate measurements are not plotted together' in note + assert 'incompatible candidate measurements are not plotted together' in note (tmp_path / "runs/a/events.jsonl").write_text('{"phase":"complete"}\n', encoding="utf-8") (tmp_path / "runs/b/events.jsonl").write_text('{"phase":"complete"}\n', encoding="utf-8") assert temporal_chart(load_dashboard(tmp_path))[0] == "" @@ -364,3 +585,22 @@ def test_time_chart_escapes_untrusted_candidate_labels(tmp_path): assert ' diff --git a/src/inferrsi/harness/campaign_records.py b/src/inferrsi/harness/campaign_records.py index e3836b827..cc5fd0e42 100644 --- a/src/inferrsi/harness/campaign_records.py +++ b/src/inferrsi/harness/campaign_records.py @@ -9,7 +9,7 @@ import uuid from pathlib import Path -from .dashboard_data import _file_ref, _inside, _object +from .dashboard_data import _file_ref, _inside, _object, _relative_location, _safe_argv, _text def _identifier(value: str, name: str) -> str: @@ -85,7 +85,7 @@ def _write_rounds(root: Path, path: Path, records: list[dict]) -> None: def record_candidate(root: Path, *, candidate_id: str, run_id: str, round_number: int, title: str, agent_trace: str | None = None, - profile: str | None = None) -> Path: + profile: str | None = None, details: dict | None = None) -> Path: """Associate a run and optional explicit evidence without copying raw data.""" root = Path(root).resolve() _identifier(candidate_id, "candidate_id") @@ -94,6 +94,50 @@ def record_candidate(root: Path, *, candidate_id: str, run_id: str, round_number raise ValueError("round must be a positive integer") if not isinstance(title, str) or not title.strip(): raise ValueError("title is required") + if details is not None: + if not isinstance(details, dict) or set(details) - { + "change_kind", "summary", "hypothesis", "benchmark", "accuracy", "code_locations", + "commands" + }: + raise ValueError("candidate details contain unsupported fields") + if "change_kind" in details: + kind = details["change_kind"] + if not isinstance(kind, str) or kind not in {"code", "configuration", "mixed", "other"}: + raise ValueError("unknown candidate change kind") + for key in ("summary", "hypothesis", "benchmark", "accuracy"): + if key in details and (not isinstance(details[key], str) or len(details[key]) > 1500): + raise ValueError(f"{key} must be a string of at most 1500 characters") + locations = details.get("code_locations", []) + if not isinstance(locations, list) or len(locations) > 50: + raise ValueError("code_locations must be a list of at most 50 entries") + for location in locations: + if (not isinstance(location, dict) or set(location) - { + "path", "line_start", "line_end", "scope", "note", "revision" + } or _relative_location(location.get("path")) is None): + raise ValueError("code location must contain a safe relative path") + start, end = location.get("line_start"), location.get("line_end") + if ((start is not None and (type(start) is not int or not 0 < start <= 10_000_000)) + or (end is not None and (type(end) is not int or start is None + or not start <= end <= 10_000_000))): + raise ValueError("code location lines must be a valid positive range") + if "revision" in location and (not isinstance(location["revision"], str) + or not re.fullmatch(r"[a-fA-F0-9]{7,64}", + location["revision"])): + raise ValueError("code location revision must be a hexadecimal revision") + for key in ("scope", "note"): + if key in location and (not isinstance(location[key], str) + or len(location[key]) > 1000): + raise ValueError(f"code location {key} must be a string") + commands = details.get("commands", []) + if not isinstance(commands, list) or len(commands) > 20: + raise ValueError("commands must be a list of at most 20 entries") + for command in commands: + if (not isinstance(command, dict) or set(command) != {"label", "argv"} + or not isinstance(command["label"], str) or len(command["label"]) > 120 + or not isinstance(command["argv"], list) or not 1 <= len(command["argv"]) <= 100 + or not all(isinstance(arg, str) and len(arg) <= 4096 + for arg in command["argv"])): + raise ValueError("each command needs a label and a bounded argument list") _, state = _object(root / "campaign.json", root) if state != "present": raise ValueError("campaign.json is missing or invalid") @@ -105,6 +149,22 @@ def record_candidate(root: Path, *, candidate_id: str, run_id: str, round_number raise ValueError("candidate directory escapes the campaign") metadata = {"candidate_id": candidate_id, "run_id": run_id, "round": round_number, "title": title} + if details is not None: + sanitized = {key: _text(value, "", 1500) if key != "change_kind" else value + for key, value in details.items() + if key in ("change_kind", "summary", "hypothesis", "benchmark", "accuracy")} + if "code_locations" in details: + sanitized["code_locations"] = [ + {key: _text(value, "", 1000) if key in ("scope", "note") else value + for key, value in location.items()} + for location in details["code_locations"] + ] + if "commands" in details: + sanitized["commands"] = [ + {"label": _text(command["label"], "", 120), + "argv": _safe_argv(command["argv"])} for command in details["commands"] + ] + metadata["details"] = sanitized for key, value in (("agent_trace_ref", agent_trace), ("profile_ref", profile)): ref = _reference(root, value, key) if ref is not None: diff --git a/src/inferrsi/harness/dashboard_data.py b/src/inferrsi/harness/dashboard_data.py index aefd99d60..bc6f4298a 100644 --- a/src/inferrsi/harness/dashboard_data.py +++ b/src/inferrsi/harness/dashboard_data.py @@ -1,7 +1,8 @@ """Read recorded run and campaign evidence into a dashboard view model. -Only known fields and explicitly referenced artifacts are exposed. Raw command -lines, environments, error messages and log contents are deliberately omitted. +Only known fields and explicitly referenced artifacts are exposed. Commands +are reduced to a conservative display representation; environments, arbitrary +error messages and log contents are deliberately omitted. """ from __future__ import annotations @@ -10,6 +11,7 @@ import json import math import re +import shlex from pathlib import Path, PurePosixPath, PureWindowsPath STAGES = ("diagnosis", "knowledge", "search", "execution", "validation", "review") @@ -21,6 +23,7 @@ EVENT_TYPES = { "agent.context", "agent.request", "agent.response", "agent.tool", "agent.completed", "agent.failed", "agent.protocol", "agent.reservation", + "edit.artifact", "edit.protocol", "edit.slot", } @@ -33,7 +36,7 @@ def _number(value: object) -> float | None: return None -def _text(value: object, default: str = "not_recorded") -> str: +def _text(value: object, default: str = "not_recorded", limit: int = 240) -> str: if not isinstance(value, str) or not value: return default value = re.sub(r"https?://\S+", "[redacted-url]", value, flags=re.I) @@ -45,7 +48,261 @@ def _text(value: object, default: str = "not_recorded") -> str: r"(?i)\b(api[_-]?key|token|password|secret)\s*[:=]\s*\S+", r"\1=[redacted]", value, ) - return value[:240] + return value[:limit] + + +_SENSITIVE_NAME = re.compile( + r"(?i)(?:password|passwd|token|secret|api[_-]?key|access[_-]?key|auth|credential|cookie|header)" +) +_SENSITIVE_PATH = re.compile( + r"(?i)(?:^|[/._-])(?:password|passwd|token|secret|api[_-]?key|credential|cookie)(?:$|[/._-])" +) + + +def _relative_location(value: object) -> str | None: + """Accept a source location, never a host path or a credential-like filename.""" + if not isinstance(value, str) or not value or len(value) > 240: + return None + posix, windows = PurePosixPath(value), PureWindowsPath(value) + if (posix.is_absolute() or windows.is_absolute() or windows.drive or ".." in posix.parts + or ".." in windows.parts or "\\" in value or any(ord(c) < 32 for c in value) + or "://" in value or _text(value, "", 240) != value + or _SENSITIVE_PATH.search(value) or any(part.startswith(".") for part in posix.parts)): + return None + return posix.as_posix() + + +def _optimization(candidate: dict) -> dict: + data = candidate.get("details") + data = data if isinstance(data, dict) else {} + kind = data.get("change_kind") + if kind not in ("code", "configuration", "mixed", "other"): + kind = None + locations = [] + raw_locations = data.get("code_locations") + if isinstance(raw_locations, list): + for entry in raw_locations[:50]: + if not isinstance(entry, dict): + continue + path = _relative_location(entry.get("path")) + start = entry.get("line_start") + end = entry.get("line_end") + revision = entry.get("revision") + revision = (revision.lower() if isinstance(revision, str) + and re.fullmatch(r"[a-fA-F0-9]{7,64}", revision) else None) + start = start if type(start) is int and 0 < start <= 10_000_000 else None + end = end if (start is not None and type(end) is int + and start <= end <= 10_000_000) else None + locations.append({"path": path, "line_start": start, "line_end": end, + "revision": revision, + "scope": _text(entry.get("scope"), "Not recorded"), + "note": _text(entry.get("note"), "Not recorded", 1000)}) + commands = [] + raw_commands = data.get("commands") + if isinstance(raw_commands, list): + for entry in raw_commands[:20]: + if isinstance(entry, dict): + command = _display_command(entry.get("argv")) + if command is not None: + commands.append({"label": _text(entry.get("label"), "Caller-recorded command"), + "argv": command}) + return {"kind": kind, "summary": _text(data.get("summary"), "Not recorded", 1500), + "hypothesis": _text(data.get("hypothesis"), "Not recorded", 1500), + "benchmark": _text(data.get("benchmark"), "Not recorded", 1500), + "accuracy": _text(data.get("accuracy"), "Not recorded", 1500), + "code_locations": locations, "commands": commands} + + +def _safe_argv(value: object) -> list[str] | None: + """Keep useful argv structure without retaining arbitrary argument values.""" + if not isinstance(value, list) or not value or len(value) > 100: + return None + if not all(isinstance(arg, str) and len(arg) <= 4096 for arg in value): + return None + safe_words = {"baseline", "candidate", "mtp", "quality-baseline", "quality-mtp", "smoke", + "true", "false", "auto", "vllm", "float16", "bfloat16", "float32"} + safe_options = {"--mode", "--dtype", "--max-tokens", "--max-model-len", + "--num-speculative-tokens", "--max-num-batched-tokens", "--input-len", + "--output-len", "--batch-size", "--num-prompts", "--port", + "--tensor-parallel-size", "--gpu-memory-utilization", "--max-num-seqs", + "--generation-config"} + known_options = safe_options | {"-m", "--model", "--served-model-name", "--host", "--result", + "--speculative-config", "--disable-prefix-caching", + "--disable-log-requests", "--trust-remote-code", + "--enforce-eager", + "--dataset-name", "--dataset-path", "--backend", "--endpoint"} + result = [] + hide_next = False + value_option = None + for index, arg in enumerate(value): + if hide_next: + result.append("[redacted]") + hide_next = False + value_option = None + continue + if arg in ("-H", "-c", "--header", "--data", "--data-raw"): + result.append(arg) + hide_next = True + continue + if arg.startswith("-"): + value_option = None + name, equal, val = arg.partition("=") + if not re.fullmatch(r"--?[A-Za-z0-9][A-Za-z0-9_-]*", name): + result.append("[redacted]") + continue + if _SENSITIVE_NAME.search(name) and name not in safe_options: + result.append("[sensitive-option]" + ("=[redacted]" if equal else "")) + hide_next = not equal + continue + if name not in known_options: + result.append("[redacted-option]") + continue + if equal: + safe = name in safe_options and (val in safe_words or re.fullmatch(r"[0-9.]+", val)) + shown = val if safe else "[redacted]" + result.append(name + "=" + shown) + else: + result.append(name) + hide_next = bool(_SENSITIVE_NAME.search(name)) and name not in safe_options + accepts_value = name in safe_options or name == "--speculative-config" + value_option = name if accepts_value else None + continue + if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*=.*", arg): + result.append(arg.split("=", 1)[0] + "=[redacted]") + continue + base = PureWindowsPath(arg).name if "\\" in arg else PurePosixPath(arg).name + if (index == 0 and re.fullmatch(r"(?:python(?:[0-9.]+)?|vllm|pytest|bash|sh)", base)): + result.append(base) + elif (index == 1 and base.endswith(".py") + and re.fullmatch(r"[A-Za-z0-9_.-]{1,100}", base) + and not _SENSITIVE_NAME.search(base)): + result.append(base if base == arg else "[path]/" + base) + elif index == 2 and value[1] == "-m" and arg == "vllm.entrypoints.openai.api_server": + result.append(arg) + elif value_option == "--speculative-config": + try: + config = json.loads(arg) + except (ValueError, RecursionError): + config = None + if (isinstance(config, dict) and set(config) == {"method", "num_speculative_tokens"} + and config.get("method") == "mtp" + and type(config.get("num_speculative_tokens")) is int + and 0 < config["num_speculative_tokens"] <= 128): + result.append(json.dumps(config, separators=(",", ":"))) + else: + result.append("[redacted]") + elif value_option and (arg in safe_words or re.fullmatch(r"[0-9.]+", arg)): + result.append(arg) + else: + result.append("[redacted]") + value_option = None + return result + + +def _display_command(value: object) -> str | None: + safe = _safe_argv(value) + return shlex.join(safe) if safe is not None else None + + +def _event_details(record: dict) -> dict: + """Retain structured trace metadata, never arbitrary payloads or messages.""" + details = {} + for key in ("request", "requests", "request_bytes", "bytes", "revision"): + value = record.get(key) + if type(value) is int and 0 <= value <= 1_000_000_000_000: + details[key] = value + for key in ("request_sha256", "result_sha256", "artifact_sha256", "sha256", "manifest_sha256", + "actor_sha256", "check_receipt", "benchmark_receipt"): + value = record.get(key) + if isinstance(value, str) and re.fullmatch(r"[a-fA-F0-9]{64}", value): + details[key] = value.lower() + usage = record.get("usage") + if isinstance(usage, dict): + keys = ("input_tokens", "output_tokens", "cached_input_tokens") + values = {key: usage[key] for key in keys + if type(usage.get(key)) is int and 0 <= usage[key] <= 1_000_000_000_000} + if values: + details["usage"] = values + if record.get("client_kind") in ("anthropic-messages-agent-v1", + "anthropic-messages-full-context-v1", + "anthropic-messages-edit-check-v1"): + details["client_kind"] = record["client_kind"] + files = record.get("files") + if isinstance(files, list): + entries = [] + for file in files[:50]: + if not isinstance(file, dict): + continue + path = _relative_location(file.get("path")) + size, digest = file.get("bytes"), file.get("sha256") + if (path and type(size) is int and 0 <= size <= 1_000_000_000_000 + and isinstance(digest, str) and re.fullmatch(r"[a-fA-F0-9]{64}", digest)): + entries.append({"path": path, "bytes": size, "sha256": digest.lower()}) + if entries: + details["files"] = entries + reason = record.get("reason") + reasons = {"timeout", "interrupted", "agent_error", "edit_agent_error", + "readiness_or_worker_error", "request_bytes_limit", "input_token_limit", + "output_token_limit", "tool_call_limit", "invalid_tool_call", + "candidate_bytes_limit", + "credential_echo", "credential_in_payload", "deadline_exceeds_frozen_limit", + "feedback_bytes_limit", "incompatible_response", "invalid_candidate", + "invalid_credential_shape", "invalid_or_premature_answer", "invalid_response_json", + "invalid_tool_utf8", "invalid_visible_feedback", "invalid_workspace_file", + "missing_agent_tool_call", "missing_task", "protected_workspace_changed", + "provider_transport_error", "request_allocation_unavailable", + "request_authorization_denied", "request_limit", "response_bytes_limit", + "revision_retention_error", "stale_or_missing_receipt", "task_not_read", + "tool_read_limit", "transport_error", "unexposed_path", "unknown_usage", + "unsupported_tool", "visible_callback_error", "workspace_symlink"} + if isinstance(reason, str) and (reason in reasons + or re.fullmatch(r"http_[1-5][0-9]{2}", reason)): + details["reason"] = reason + for key, allowed in (("stop_reason", {"tool_use", "end_turn", "max_tokens", "stop_sequence", + "pause_turn", "refusal"}), + ("state", {"running", "completed", "failed", "crashed", "timeout", + "cancelled"}), + ("verdict", {"PASS", "FAIL", "INCONCLUSIVE"})): + value = record.get(key) + if isinstance(value, str) and value in allowed: + details[key] = value + path = _relative_location(record.get("path")) + if path is not None: + details["path"] = path + artifact = _relative_location(record.get("artifact")) + if artifact is not None: + details["artifact"] = artifact + receipt = record.get("receipt") + expected_kind = {"run_visible_checks": "checks", "run_visible_benchmark": "benchmark"}.get( + record.get("name") if isinstance(record.get("name"), str) else None + ) + if isinstance(receipt, dict) and expected_kind and receipt.get("kind") == expected_kind: + summary = {"kind": expected_kind} + digest = receipt.get("receipt_sha256") + if isinstance(digest, str) and re.fullmatch(r"[a-fA-F0-9]{64}", digest): + summary["receipt_sha256"] = digest.lower() + feedback = receipt.get("feedback") + if isinstance(feedback, dict): + for key in ("verdict", "quality_verdict"): + value = feedback.get(key) + if isinstance(value, str) and value in {"PASS", "FAIL", "INCONCLUSIVE"}: + summary[key] = value + improvement = _number(feedback.get("improvement_pct")) + if improvement is not None: + summary["improvement_pct"] = improvement + checks = feedback.get("checks") + if (isinstance(checks, dict) and len(checks) <= 1000 + and all(type(value) is bool for value in checks.values())): + summary["checks_total"] = len(checks) + summary["checks_passed"] = sum(checks.values()) + for key in ("baseline_values", "candidate_values"): + values = feedback.get(key) + if isinstance(values, list) and len(values) <= 100: + numbers = [_number(value) for value in values] + if all(value is not None and value > 0 for value in numbers): + summary[key] = numbers + details["recorded_receipt"] = summary + return details def _inside(root: Path, path: Path) -> bool: @@ -205,8 +462,11 @@ def _events(path: Path, root: Path, expected: dict | None = None, unique_candida if record.get("side") in ("baseline", "candidate") else None, "trial": trial if isinstance(trial, int) and not isinstance(trial, bool) and 0 < trial <= 1_000_000 else None, - "tool": record.get("name") if record.get("name") in ("read_file", "submit_answer") - else None, + "tool": record.get("name") if record.get("name") in ( + "read_file", "submit_answer", "submit_candidate", "write_candidate", + "run_visible_checks", "run_visible_benchmark", + ) else None, + "details": _event_details(record), "_declared_refs": record.get("refs") if isinstance(record.get("refs"), list) else [], "_link_state": link_state, }) @@ -252,7 +512,7 @@ def _environment_signature(environment: dict, devices: object, backend: object) def _run(root: Path, path: Path, candidate: dict, run_id: str | None, noise: float | None, campaign_id: object = None, unique_candidate: bool = False, candidate_ref: dict | None = None) -> dict: - lock, _ = _object(path / "run.lock.json", root) + lock, _, lock_ref = _object_snapshot(path / "run.lock.json", root, "run_lock") repositories, _ = _object(path / "repositories.lock.json", root) environment, _ = _object(path / "environment.json", root) evaluation, evaluation_state, evaluation_ref = _object_snapshot( @@ -263,7 +523,9 @@ def _run(root: Path, path: Path, candidate: dict, run_id: str | None, noise: flo path / "validation" / "execution.json", root, "quality" ) events, corrupt, event_state, _, events_ref = _events(path / "events.jsonl", root) + corrupt_sources = {"events": list(corrupt), "agent_trace": []} refs = { + "run_lock": lock_ref, "events": events_ref, "evaluation": evaluation_ref, "status": status_ref, @@ -311,6 +573,7 @@ def attach_declared(event: dict) -> None: else "hash_mismatch" if agent_state == "changed" else "file_absent" if agent_state == "missing" else "corrupt"} agent_events = [] + agent_corrupt = [] agent_links = {"unlinked": [], "mismatched": []} for event in agent_events: event["source"] = "agent_trace" @@ -321,6 +584,7 @@ def attach_declared(event: dict) -> None: else: events.append(event) corrupt.extend(agent_corrupt) + corrupt_sources["agent_trace"] = agent_corrupt tool_events = [event for event in events if event["source"] == "agent_trace" and event["type"] == "agent.tool"] refs["tool_calls"] = ({**refs["agent_trace"], "kind": "tool_calls"} if tool_events else { @@ -404,7 +668,24 @@ def attach_declared(event: dict) -> None: if evaluation.get("direction") in ("minimize", "maximize") else None, "verdict": verdict, "outcome": outcome, "events": events, "corrupt_event_lines": corrupt, + "corrupt_event_sources": corrupt_sources, "agent_trace_links": agent_links, "unlinked_agent_events": unlinked_events, + "optimization": _optimization(candidate), + "benchmark_context": {"scenario": _text(lock.get("scenario"), "Not recorded"), + "workload": _text(lock.get("workload_id"), "Not recorded"), + "repetitions": lock.get("repetitions") + if type(lock.get("repetitions")) is int + and 0 < lock["repetitions"] <= 1_000_000 else None, + "baseline_samples": [_number(v) + for v in baseline.get("values", [])[:100]] + if isinstance(baseline.get("values"), list) else [], + "candidate_samples": [_number(v) + for v in measured.get("values", [])[:100]] + if isinstance(measured.get("values"), list) else []}, + "commands": {key: _display_command(lock.get(source)) for key, source in ( + ("baseline", "baseline_command"), ("candidate", "candidate_command"), + ("accuracy", "validation_command"), + )}, "candidate_metadata_state": (candidate_ref.get("missing_reason", "present") if candidate_ref else "not_applicable"), "artifacts": refs, "_comparison_signature": signature, diff --git a/tests/test_campaign_records.py b/tests/test_campaign_records.py index 9f6f62eda..66cbc69e3 100644 --- a/tests/test_campaign_records.py +++ b/tests/test_campaign_records.py @@ -72,3 +72,42 @@ def test_round_decision_is_recorded_once_and_visible(tmp_path): assert load_dashboard(tmp_path)["candidates"][0]["decision"] == "rolled_back" with pytest.raises(ValueError, match="already has a decision"): record_decision(tmp_path, candidate_id="a", round_number=1, outcome="adopted") + + +def test_record_cli_accepts_optimization_details_and_rejects_unsafe_location(tmp_path): + root = tmp_path / "campaign" + create_campaign(root, "example") + run = root / "runs/run-a" + run.mkdir(parents=True) + (run / "run.lock.json").write_text("{}", encoding="utf-8") + details = tmp_path / "details.json" + details.write_text(json.dumps({"change_kind": "code", "summary": "Reduce launch overhead", + "code_locations": [{"path": "src/engine.py", "line_start": 7}], + "commands": [{"label": "Benchmark", + "argv": ["python", "bench.py", "--api-key", + "FAKE_TEST_SECRET"]}]}), + encoding="utf-8") + assert main(["campaign", "record", "--directory", str(root), "--candidate", "a", + "--run-id", "run-a", "--round", "1", "--title", "First", + "--details-json", str(details)]) == 0 + row = load_dashboard(root)["candidates"][0] + assert row["optimization"]["summary"] == "Reduce launch overhead" + assert row["optimization"]["code_locations"][0]["line_start"] == 7 + assert row["optimization"]["commands"][0]["argv"].startswith("python bench.py") + assert "FAKE_TEST_SECRET" not in (root / "candidates/a/candidate.json").read_text() + assert "FAKE_TEST_SECRET" not in json.dumps(row) + with pytest.raises(ValueError, match="safe relative path"): + record_candidate(root, candidate_id="b", run_id="run-a", round_number=1, title="Second", + details={"code_locations": [{"path": "../private.py"}]}) + assert not (root / "candidates/b/candidate.json").exists() + + +def test_record_cli_rejects_excessively_nested_details_without_traceback(tmp_path, capsys): + details = tmp_path / "details.json" + details.write_text("[" * 4000 + "0" + "]" * 4000, encoding="utf-8") + with pytest.raises(SystemExit): + main(["campaign", "record", "--directory", str(tmp_path), "--candidate", "a", + "--run-id", "run-a", "--round", "1", "--title", "First", + "--details-json", str(details)]) + error = capsys.readouterr().err + assert "too deeply nested" in error and "Traceback" not in error diff --git a/tests/test_dashboard_data.py b/tests/test_dashboard_data.py index d8dda4ef0..00c784a70 100644 --- a/tests/test_dashboard_data.py +++ b/tests/test_dashboard_data.py @@ -684,3 +684,99 @@ def test_time_chart_marks_quality_failure_and_rollback_without_measurement(tmp_p ) _, note = temporal_chart(load_dashboard(tmp_path)) assert '1 result(s) have no recorded terminal timestamp' in note + + +def test_optimization_details_commands_and_privacy(tmp_path): + write_json(tmp_path / "campaign.json", {"campaign_id": "example"}) + run(tmp_path, "run-a", 5, quality="completed", verified=True) + lock = tmp_path / "runs/run-a/run.lock.json" + data = json.loads(lock.read_text(encoding="utf-8")) + data["candidate_command"] = [ + "/private/bin/python", "/private/trial.py", "--mode", "mtp", "--token", "s3cr3t", + "--api-key=rawkey", "-H", "Authorization: Bearer hidden", "--max-tokens", "128", + "unknown-private-value", "-c", "print('hidden source')", + ] + write_json(lock, data) + candidate(tmp_path, "a", "run-a", details={ + "change_kind": "code", "summary": "Improve decode token=private-value", + "hypothesis": "Reduce launch overhead", "benchmark": "Paired trials", + "accuracy": "Evaluate a fixed test set", "code_locations": [ + {"path": "vllm/tokenizer.py", "line_start": 5, "line_end": 8, + "revision": "a" * 40, "scope": "vllm", "note": "Decode path"}, + {"path": "../private.py"}, + ], + "commands": [{"label": "Candidate server", "argv": [ + "python", "-m", "vllm.entrypoints.openai.api_server", "--speculative-config", + '{"method":"mtp","num_speculative_tokens":1}', + ]}], + }) + + row = load_dashboard(tmp_path)["candidates"][0] + command = row["commands"]["candidate"] + assert "trial.py" in command and "--mode mtp" in command and "--max-tokens 128" in command + for secret in ("/private", "s3cr3t", "rawkey", "Bearer hidden", "unknown-private-value", + "hidden source", "private-value"): + assert secret not in json.dumps(row) + assert row["optimization"]["summary"] == "Improve decode token=[redacted]" + assert row["optimization"]["code_locations"][0]["path"] == "vllm/tokenizer.py" + assert row["optimization"]["code_locations"][0]["revision"] == "a" * 40 + assert row["optimization"]["code_locations"][1]["path"] is None + assert '"method":"mtp"' in row["optimization"]["commands"][0]["argv"] + assert row["artifacts"]["run_lock"]["sha256"] == hashlib.sha256(lock.read_bytes()).hexdigest() + for args in (["python", "x.py", "--value", "--sk-abc123XYZ"], + ["python", "x.py", "--size", "-1122334455667788"]): + display = dashboard_data._display_command(args) + assert "--sk-abc123XYZ" not in display + assert "-1122334455667788" not in display + + +def test_trace_preserves_safe_event_fields_and_corrupt_source(tmp_path): + write_json(tmp_path / "campaign.json", {"campaign_id": "example"}) + run(tmp_path, "run-a", 5, quality="completed", verified=True) + events = tmp_path / "runs/run-a/events.jsonl" + events.write_text('bad-runner\n', encoding="utf-8") + trace = tmp_path / "candidates/a/trace.jsonl" + trace.parent.mkdir(parents=True) + event = { + "schema_version": 1, "campaign_id": "example", "candidate_id": "a", + "run_id": "run-a", "round": 1, "stage": "search", "type": "agent.tool", + "time": "2026-01-01T00:00:00Z", "name": "write_candidate", "request": 2, + "bytes": 100, "result_sha256": "b" * 64, + "usage": {"input_tokens": 10, "output_tokens": 2, "secret": "DO_NOT_EXPORT"}, + "prompt": "DO_NOT_EXPORT", "reason": {"secret": "DO_NOT_EXPORT"}, + } + trace.write_text(json.dumps(event) + '\n' + json.dumps({ + **event, "type": "edit.artifact", "name": "ignored", "revision": 3, + "artifact": "candidate.py", + }) + '\n' + json.dumps({**event, "type": "agent.tool", "name": "submit_candidate"}) + + '\n' + json.dumps({ + **event, "name": "run_visible_benchmark", + "reason": "provider_transport_error", + "receipt": {"kind": "benchmark", "receipt_sha256": "c" * 64, + "feedback": {"verdict": "PASS", "quality_verdict": "PASS", + "improvement_pct": 20.5, + "baseline_values": [2.0], + "candidate_values": [1.5], + "secret": "DO_NOT_EXPORT"}}, + }) + + '\nbad-agent\n', encoding="utf-8") + candidate(tmp_path, "a", "run-a", round=1, agent_trace_ref={ + "path": "candidates/a/trace.jsonl", + "sha256": hashlib.sha256(trace.read_bytes()).hexdigest(), + }) + + row = load_dashboard(tmp_path)["candidates"][0] + event = row["events"][0] + assert event["tool"] == "write_candidate" + assert event["details"]["request"] == 2 and event["details"]["bytes"] == 100 + assert event["details"]["usage"] == {"input_tokens": 10, "output_tokens": 2} + assert event["details"]["result_sha256"] == "b" * 64 + assert "DO_NOT_EXPORT" not in json.dumps(row) + assert row["events"][1]["type"] == "edit.artifact" and row["events"][1]["stage"] == "search" + assert row["events"][1]["details"]["artifact"] == "candidate.py" + assert row["events"][2]["tool"] == "submit_candidate" + receipt = row["events"][3]["details"]["recorded_receipt"] + assert receipt["quality_verdict"] == "PASS" and receipt["improvement_pct"] == 20.5 + assert receipt["receipt_sha256"] == "c" * 64 + assert row["events"][3]["details"]["reason"] == "provider_transport_error" + assert row["corrupt_event_sources"] == {"events": [1], "agent_trace": [5]} From bde2799762ef70caa1331dff03fe2be0fe83b44e Mon Sep 17 00:00:00 2001 From: David Chen <530634352@qq.com> Date: Mon, 2 Nov 2026 21:07:08 +0800 Subject: [PATCH 06/23] Clarify missing validation commands Agent-assisted-by: Codex (implementation) Signed-off-by: David Chen <530634352@qq.com> --- src/inferrsi/data/dashboard/template.html | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/inferrsi/data/dashboard/template.html b/src/inferrsi/data/dashboard/template.html index 31b9547eb..47f373323 100644 --- a/src/inferrsi/data/dashboard/template.html +++ b/src/inferrsi/data/dashboard/template.html @@ -234,7 +234,8 @@ detailLine(benchmark,'Accuracy methodology / result · caller recorded',info.accuracy); detailLine(benchmark,'Validation command · redacted argv',row.commands.accuracy || 'Not recorded',true); for(const command of info.commands)detailLine(benchmark,`${command.label} · caller recorded, redacted argv`,command.argv,true); - detailLine(benchmark,'Recorded validation state',`${row.quality.replaceAll('_',' ')} · only the recorded validation command is covered`); + const validationNote=row.commands.accuracy?'only the recorded validation command is covered':'validation command not recorded in run.lock.json'; + detailLine(benchmark,'Recorded validation state',`${row.quality.replaceAll('_',' ')} · ${validationNote}`); detailLine(benchmark,'Command evidence',refText(row.artifacts.run_lock),true); if(row.artifacts.candidate_metadata)detailLine(benchmark,'Caller detail evidence',refText(row.artifacts.candidate_metadata),true); } From 0c1cec645b841921408da2ec9182f3ded32edfe9 Mon Sep 17 00:00:00 2001 From: David Chen <530634352@qq.com> Date: Mon, 2 Nov 2026 21:07:49 +0800 Subject: [PATCH 07/23] Clarify unavailable command displays Agent-assisted-by: Codex (implementation) Signed-off-by: David Chen <530634352@qq.com> --- src/inferrsi/data/dashboard/template.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/inferrsi/data/dashboard/template.html b/src/inferrsi/data/dashboard/template.html index 47f373323..aeb7898db 100644 --- a/src/inferrsi/data/dashboard/template.html +++ b/src/inferrsi/data/dashboard/template.html @@ -229,12 +229,12 @@ detailLine(benchmark,'Recorded medians',`Baseline ${measurement(row.baseline_median)} · Candidate ${measurement(row.candidate_median)} ${row.metric_unit}`); const samples=values=>values.length?values.map(measurement).join(', '):'Not recorded'; detailLine(benchmark,'Recorded samples',`Baseline: ${samples(context.baseline_samples)}\nCandidate: ${samples(context.candidate_samples)} ${row.metric_unit}`); - detailLine(benchmark,'Baseline command · redacted argv',row.commands.baseline || 'Not recorded',true); - detailLine(benchmark,'Candidate command · redacted argv',row.commands.candidate || 'Not recorded',true); + detailLine(benchmark,'Baseline command · redacted argv',row.commands.baseline || 'Not recorded or unavailable for display',true); + detailLine(benchmark,'Candidate command · redacted argv',row.commands.candidate || 'Not recorded or unavailable for display',true); detailLine(benchmark,'Accuracy methodology / result · caller recorded',info.accuracy); - detailLine(benchmark,'Validation command · redacted argv',row.commands.accuracy || 'Not recorded',true); + detailLine(benchmark,'Validation command · redacted argv',row.commands.accuracy || 'Not recorded or unavailable for display',true); for(const command of info.commands)detailLine(benchmark,`${command.label} · caller recorded, redacted argv`,command.argv,true); - const validationNote=row.commands.accuracy?'only the recorded validation command is covered':'validation command not recorded in run.lock.json'; + const validationNote=row.commands.accuracy?'only the recorded validation command is covered':'validation command not recorded or unavailable for display'; detailLine(benchmark,'Recorded validation state',`${row.quality.replaceAll('_',' ')} · ${validationNote}`); detailLine(benchmark,'Command evidence',refText(row.artifacts.run_lock),true); if(row.artifacts.candidate_metadata)detailLine(benchmark,'Caller detail evidence',refText(row.artifacts.candidate_metadata),true); From c100c3652537fdef37f064fdf74ccb8d6a0780fa Mon Sep 17 00:00:00 2001 From: David Chen <530634352@qq.com> Date: Tue, 3 Nov 2026 09:29:23 +0800 Subject: [PATCH 08/23] Record reviewed optimization trace evidence Agent-assisted-by: Codex (implementation) Signed-off-by: David Chen <530634352@qq.com> --- src/inferrsi/arguments.py | 2 + src/inferrsi/cli.py | 3 +- src/inferrsi/harness/campaign_records.py | 15 +- src/inferrsi/harness/dashboard_data.py | 9 + src/inferrsi/harness/optimization_trace.py | 228 ++++++++++++++++++ tests/test_optimization_trace.py | 258 +++++++++++++++++++++ 6 files changed, 512 insertions(+), 3 deletions(-) create mode 100644 src/inferrsi/harness/optimization_trace.py create mode 100644 tests/test_optimization_trace.py diff --git a/src/inferrsi/arguments.py b/src/inferrsi/arguments.py index ecd7f0d4a..3c540696b 100644 --- a/src/inferrsi/arguments.py +++ b/src/inferrsi/arguments.py @@ -137,6 +137,8 @@ def parser() -> argparse.ArgumentParser: help="JSON file describing the optimization and code locations") record.add_argument("--agent-trace", help="Existing campaign-relative JSONL trace") record.add_argument("--profile", help="Existing campaign-relative profile artifact") + record.add_argument("--optimization-trace", + help="Caller-reviewed campaign-relative optimization trace JSON") decision = campaign_actions.add_parser("decide", help="Record one round decision") decision.add_argument("--directory", type=Path, required=True) decision.add_argument("--candidate", required=True) diff --git a/src/inferrsi/cli.py b/src/inferrsi/cli.py index 9286ebf6f..f925a1047 100644 --- a/src/inferrsi/cli.py +++ b/src/inferrsi/cli.py @@ -308,7 +308,8 @@ def main(argv: list[str] | None = None) -> int: path = record_candidate(args.directory, candidate_id=args.candidate, run_id=args.run_id, round_number=args.round, title=args.title, agent_trace=args.agent_trace, - profile=args.profile, details=details) + profile=args.profile, details=details, + optimization_trace=args.optimization_trace) else: path = record_decision(args.directory, candidate_id=args.candidate, round_number=args.round, outcome=args.outcome) diff --git a/src/inferrsi/harness/campaign_records.py b/src/inferrsi/harness/campaign_records.py index cc5fd0e42..a1fa4b676 100644 --- a/src/inferrsi/harness/campaign_records.py +++ b/src/inferrsi/harness/campaign_records.py @@ -10,6 +10,7 @@ from pathlib import Path from .dashboard_data import _file_ref, _inside, _object, _relative_location, _safe_argv, _text +from .optimization_trace import read_optimization_trace def _identifier(value: str, name: str) -> str: @@ -85,7 +86,8 @@ def _write_rounds(root: Path, path: Path, records: list[dict]) -> None: def record_candidate(root: Path, *, candidate_id: str, run_id: str, round_number: int, title: str, agent_trace: str | None = None, - profile: str | None = None, details: dict | None = None) -> Path: + profile: str | None = None, details: dict | None = None, + optimization_trace: str | None = None) -> Path: """Associate a run and optional explicit evidence without copying raw data.""" root = Path(root).resolve() _identifier(candidate_id, "candidate_id") @@ -138,7 +140,7 @@ def record_candidate(root: Path, *, candidate_id: str, run_id: str, round_number or not all(isinstance(arg, str) and len(arg) <= 4096 for arg in command["argv"])): raise ValueError("each command needs a label and a bounded argument list") - _, state = _object(root / "campaign.json", root) + campaign, state = _object(root / "campaign.json", root) if state != "present": raise ValueError("campaign.json is missing or invalid") run = root / "runs" / run_id @@ -169,6 +171,15 @@ def record_candidate(root: Path, *, candidate_id: str, run_id: str, round_number ref = _reference(root, value, key) if ref is not None: metadata[key] = ref + if optimization_trace is not None: + _, ref = read_optimization_trace( + root, {"path": optimization_trace}, campaign_id=campaign.get("campaign_id"), + candidate_id=candidate_id, round_number=round_number, unique_candidate=True, + require_hash=False, + ) + if ref["state"] != "present": + raise ValueError(f"optimization trace: {ref['missing_reason']}") + metadata["optimization_trace_ref"] = ref rounds_path = root / "rounds.jsonl" if not _inside(root, rounds_path): raise ValueError("rounds file escapes the campaign") diff --git a/src/inferrsi/harness/dashboard_data.py b/src/inferrsi/harness/dashboard_data.py index bc6f4298a..716fa380a 100644 --- a/src/inferrsi/harness/dashboard_data.py +++ b/src/inferrsi/harness/dashboard_data.py @@ -14,6 +14,8 @@ import shlex from pathlib import Path, PurePosixPath, PureWindowsPath +from .optimization_trace import read_optimization_trace + STAGES = ("diagnosis", "knowledge", "search", "execution", "validation", "review") PHASE_STAGES = { "source_snapshot": "execution", @@ -535,6 +537,12 @@ def _run(root: Path, path: Path, candidate: dict, run_id: str | None, noise: flo } if candidate_ref is not None: refs["candidate_metadata"] = candidate_ref + optimization_trace, refs["optimization_trace"] = read_optimization_trace( + root, candidate.get("optimization_trace_ref"), campaign_id=campaign_id, + candidate_id=candidate.get("candidate_id"), round_number=candidate.get("round"), + unique_candidate=unique_candidate, + round_ambiguous=candidate.get("_round_state") == "ambiguous", + ) def attach_declared(event: dict) -> None: for item in event.pop("_declared_refs", []): kind = item.get("kind") if isinstance(item, dict) else None @@ -671,6 +679,7 @@ def attach_declared(event: dict) -> None: "corrupt_event_sources": corrupt_sources, "agent_trace_links": agent_links, "unlinked_agent_events": unlinked_events, "optimization": _optimization(candidate), + "optimization_trace": optimization_trace, "benchmark_context": {"scenario": _text(lock.get("scenario"), "Not recorded"), "workload": _text(lock.get("workload_id"), "Not recorded"), "repetitions": lock.get("repetitions") diff --git a/src/inferrsi/harness/optimization_trace.py b/src/inferrsi/harness/optimization_trace.py new file mode 100644 index 000000000..aafa2adcb --- /dev/null +++ b/src/inferrsi/harness/optimization_trace.py @@ -0,0 +1,228 @@ +"""Explicit, caller-reviewed optimization narratives and bounded artifact reads. + +The record is a caller assertion, not proof that an agent consumed its context. +It is never populated from raw requests, logs, or the process environment. +""" + +from __future__ import annotations + +import hashlib +import json +import re +import stat +from pathlib import Path, PurePosixPath, PureWindowsPath + +MAX_OPTIMIZATION_TRACE_BYTES = 2 * 1024 * 1024 +ATTRIBUTION = "caller_declared_not_proof_of_model_consumption" +_STATES = {"complete", "partial", "redacted"} +_KINDS = {"observation", "attempt", "decision"} +_OUTCOMES = {"succeeded", "failed", "inconclusive", "skipped", "not_recorded"} +_IDENTIFIER = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]{0,127}") +_SHA256 = re.compile(r"[a-fA-F0-9]{64}") + + +class TraceError(ValueError): + """A fixed error code that never includes caller-supplied text.""" + + +def _object(value: object, fields: set[str]) -> dict: + if not isinstance(value, dict) or set(value) != fields: + raise TraceError("invalid_format") + return value + + +def _string(value: object, *, required: bool = False) -> str: + if not isinstance(value, str) or (required and not value.strip()): + raise TraceError("invalid_format") + if any(ord(char) < 32 and char not in "\n\r\t" for char in value): + raise TraceError("invalid_format") + try: + value.encode("utf-8") + except UnicodeError as exc: + raise TraceError("invalid_format") from exc + return value + + +def _strings(value: object) -> list[str]: + if not isinstance(value, list): + raise TraceError("invalid_format") + return [_string(item) for item in value] + + +def _section(value: object, fields: set[str]) -> dict: + section = _object(value, fields | {"state", "missing_reason"}) + state = section["state"] + if not isinstance(state, str) or state not in _STATES: + raise TraceError("invalid_format") + reason = _string(section["missing_reason"]) + if state != "complete" and not reason.strip(): + raise TraceError("missing_explanation") + if state == "complete" and reason.strip(): + raise TraceError("invalid_format") + return section + + +def _unique_object(pairs: list[tuple[str, object]]) -> dict: + result = {} + for key, value in pairs: + if key in result: + raise TraceError("invalid_format") + result[key] = value + return result + + +def _constant(_: str) -> None: + raise TraceError("invalid_format") + + +def parse_optimization_trace(raw: bytes) -> dict: + """Validate one complete UTF-8 JSON snapshot, without truncating any text.""" + if not isinstance(raw, bytes): + raise TraceError("invalid_format") + if len(raw) > MAX_OPTIMIZATION_TRACE_BYTES: + raise TraceError("too_large") + try: + value = json.loads(raw.decode("utf-8"), object_pairs_hook=_unique_object, + parse_constant=_constant) + except (UnicodeError, ValueError, RecursionError) as exc: + if isinstance(exc, TraceError): + raise + raise TraceError("invalid_format") from exc + if not isinstance(value, dict): + raise TraceError("invalid_format") + required = {"schema_version", "reviewed_by_caller", "campaign_id", "candidate_id", + "context", "search_space", "steps", "summary"} + if not required <= set(value) or set(value) - (required | {"round"}): + raise TraceError("invalid_format") + if type(value["schema_version"]) is not int or value["schema_version"] != 1: + raise TraceError("invalid_format") + if value["reviewed_by_caller"] is not True: + raise TraceError("not_reviewed") + for key in ("campaign_id", "candidate_id"): + if not isinstance(value[key], str) or not _IDENTIFIER.fullmatch(value[key]): + raise TraceError("invalid_format") + round_number = value.get("round") + if round_number is not None and (type(round_number) is not int or round_number < 1): + raise TraceError("invalid_format") + + context = _section(value["context"], {"text", "queries", "sources"}) + _string(context["text"], required=context["state"] == "complete") + _strings(context["queries"]) + if not isinstance(context["sources"], list): + raise TraceError("invalid_format") + for source in context["sources"]: + source = _object(source, {"ref", "revision", "locator"}) + _string(source["ref"], required=True) + _string(source["revision"]) + _string(source["locator"]) + + space = _section(value["search_space"], {"items"}) + if not isinstance(space["items"], list): + raise TraceError("invalid_format") + space_ids = set() + for item in space["items"]: + item = _object(item, {"id", "description"}) + identity = _string(item["id"], required=True) + _string(item["description"], required=True) + if identity in space_ids: + raise TraceError("invalid_format") + space_ids.add(identity) + + steps = _section(value["steps"], {"items"}) + if not isinstance(steps["items"], list): + raise TraceError("invalid_format") + previous = 0 + for item in steps["items"]: + item = _object(item, {"sequence", "kind", "action", "motivation", "outcome", "result", + "search_id"}) + sequence = item["sequence"] + if type(sequence) is not int or sequence <= previous: + raise TraceError("invalid_format") + previous = sequence + if (not isinstance(item["kind"], str) or item["kind"] not in _KINDS + or not isinstance(item["outcome"], str) or item["outcome"] not in _OUTCOMES): + raise TraceError("invalid_format") + _string(item["action"], required=True) + _string(item["motivation"], required=True) + _string(item["result"]) + search_id = _string(item["search_id"]) + if search_id and space["state"] == "complete" and search_id not in space_ids: + raise TraceError("invalid_format") + + summary = _section(value["summary"], {"successes", "failures", "limitations"}) + for key in ("successes", "failures", "limitations"): + _strings(summary[key]) + return {**value, "round": round_number, "attribution": ATTRIBUTION} + + +def _read(root: Path, raw_path: object) -> tuple[bytes | None, dict]: + if not isinstance(raw_path, str) or not raw_path: + return None, _missing("not_collected") + posix, windows = PurePosixPath(raw_path), PureWindowsPath(raw_path) + if (posix.is_absolute() or windows.is_absolute() or windows.drive or "\\" in raw_path + or any(ord(char) < 32 for char in raw_path) + or ".." in posix.parts or ".." in windows.parts): + return None, _missing("unsafe_path") + try: + root = Path(root).resolve() + path = root.joinpath(*posix.parts).resolve() + relative = path.relative_to(root).as_posix() + if path == root: + return None, _missing("unsafe_path") + if not stat.S_ISREG(path.stat().st_mode): + return None, _missing("unreadable") + with path.open("rb") as stream: + raw = stream.read(MAX_OPTIMIZATION_TRACE_BYTES + 1) + except (ValueError, RuntimeError): + return None, _missing("unsafe_path") + except FileNotFoundError: + return None, _missing("file_absent") + except OSError: + return None, _missing("unreadable") + if len(raw) > MAX_OPTIMIZATION_TRACE_BYTES: + return None, _missing("too_large") + return raw, {"kind": "optimization_trace", "state": "present", "path": relative, + "sha256": hashlib.sha256(raw).hexdigest()} + + +def _missing(reason: str) -> dict: + return {"kind": "optimization_trace", "state": "missing", "missing_reason": reason} + + +def read_optimization_trace(root: Path, reference: object, *, campaign_id: object, + candidate_id: object, round_number: object, + unique_candidate: bool, round_ambiguous: bool = False, + require_hash: bool = True) -> tuple[dict | None, dict]: + """Read, hash, validate and bind the exact same bounded snapshot.""" + if not isinstance(reference, dict): + return None, _missing("not_collected") + if reference.get("state") == "missing": + reason = reference.get("missing_reason") + allowed = {"not_collected", "file_absent", "redacted", "not_applicable"} + result = _missing(reason if isinstance(reason, str) and reason in allowed + else "not_collected") + digest = reference.get("sha256") + if isinstance(digest, str) and _SHA256.fullmatch(digest): + result["sha256"] = digest.lower() + return None, result + if reference.get("state") not in (None, "present"): + return None, _missing("invalid_format") + digest = reference.get("sha256") + if require_hash and (not isinstance(digest, str) or not _SHA256.fullmatch(digest)): + return None, _missing("hash_not_recorded") + raw, ref = _read(root, reference.get("path")) + if raw is None: + return None, ref + if require_hash and digest.lower() != ref["sha256"]: + return None, _missing("hash_mismatch") + try: + value = parse_optimization_trace(raw) + except TraceError as exc: + return None, _missing(str(exc)) + if unique_candidate is not True or round_ambiguous: + return None, _missing("identity_ambiguous") + if (not isinstance(campaign_id, str) or not isinstance(candidate_id, str) + or value["campaign_id"] != campaign_id or value["candidate_id"] != candidate_id + or (value["round"] is not None and value["round"] != round_number)): + return None, _missing("identity_mismatch") + return value, ref diff --git a/tests/test_optimization_trace.py b/tests/test_optimization_trace.py new file mode 100644 index 000000000..862b02d19 --- /dev/null +++ b/tests/test_optimization_trace.py @@ -0,0 +1,258 @@ +"""Caller-reviewed optimization narratives preserve their bytes and provenance.""" + +import hashlib +import json + +import pytest + +from inferrsi.cli import main +from inferrsi.harness import optimization_trace +from inferrsi.harness.campaign_records import create_campaign, record_candidate +from inferrsi.harness.dashboard import write_dashboard +from inferrsi.harness.dashboard_data import load_dashboard +from inferrsi.harness.optimization_trace import ( + MAX_OPTIMIZATION_TRACE_BYTES, + TraceError, + parse_optimization_trace, + read_optimization_trace, +) + + +def trace_document(): + return { + "schema_version": 1, "reviewed_by_caller": True, + "campaign_id": "campaign", "candidate_id": "candidate-a", "round": 1, + "context": {"state": "complete", "text": "Reviewed context " * 50, + "queries": ["search query"], + "sources": [{"ref": "package:example", "revision": "sha256:example", + "locator": "core"}], "missing_reason": ""}, + "search_space": {"state": "complete", "items": [ + {"id": "option-a", "description": "A considered configuration"}, + {"id": "option-b", "description": "An option not yet attempted"}, + ], "missing_reason": ""}, + "steps": {"state": "complete", "items": [ + {"sequence": 1, "kind": "attempt", "action": "Measure option A", + "motivation": "Check its hypothesis", "outcome": "failed", + "result": "Quality check failed", "search_id": "option-a"}, + ], "missing_reason": ""}, + "summary": {"state": "partial", "successes": [], "failures": ["Quality failed"], + "limitations": ["Only one option measured"], + "missing_reason": "Remaining options were not run"}, + } + + +def write_trace(path, document): + path.parent.mkdir(parents=True, exist_ok=True) + raw = json.dumps(document, ensure_ascii=False).encode("utf-8") + path.write_bytes(raw) + return hashlib.sha256(raw).hexdigest() + + +def setup_campaign(root): + create_campaign(root, "campaign") + run = root / "runs/run-a" + run.mkdir(parents=True) + (run / "run.lock.json").write_text("{}", encoding="utf-8") + + +def record(root, relative="review/optimization.json"): + return record_candidate(root, candidate_id="candidate-a", run_id="run-a", round_number=1, + title="Candidate A", optimization_trace=relative) + + +def test_cli_preserves_full_reviewed_context_and_escapes_script(tmp_path, capsys): + setup_campaign(tmp_path) + document = trace_document() + document["context"]["text"] += "\n" + digest = write_trace(tmp_path / "review/optimization.json", document) + + assert main(["campaign", "record", "--directory", str(tmp_path), "--candidate", "candidate-a", + "--run-id", "run-a", "--round", "1", "--title", "Candidate A", + "--optimization-trace", "review/optimization.json"]) == 0 + metadata = json.loads((tmp_path / "candidates/candidate-a/candidate.json").read_text()) + assert metadata["optimization_trace_ref"]["sha256"] == digest + row = load_dashboard(tmp_path)["candidates"][0] + assert row["optimization_trace"]["context"]["text"] == document["context"]["text"] + assert len(row["optimization_trace"]["context"]["text"]) > 240 + assert row["optimization_trace"]["attribution"] == ( + "caller_declared_not_proof_of_model_consumption" + ) + assert len(row["optimization_trace"]["search_space"]["items"]) == 2 + assert len(row["optimization_trace"]["steps"]["items"]) == 1 + assert row["artifacts"]["optimization_trace"]["sha256"] == digest + html = write_dashboard(tmp_path).read_text(encoding="utf-8") + assert "" not in html + payload = html.split('", 1)[0] + assert json.loads(payload)["candidates"][0]["optimization_trace"]["context"]["text"] == ( + document["context"]["text"] + ) + assert capsys.readouterr().out + + +def test_old_candidates_and_explicit_redaction_stay_visible(tmp_path): + setup_campaign(tmp_path) + record_candidate(tmp_path, candidate_id="candidate-a", run_id="run-a", round_number=1, + title="Candidate A") + row = load_dashboard(tmp_path)["candidates"][0] + assert row["optimization_trace"] is None + assert row["artifacts"]["optimization_trace"]["missing_reason"] == "not_collected" + path = tmp_path / "candidates/candidate-a/candidate.json" + metadata = json.loads(path.read_text(encoding="utf-8")) + metadata["optimization_trace_ref"] = {"state": "missing", "missing_reason": "redacted", + "sha256": "a" * 64} + path.write_text(json.dumps(metadata), encoding="utf-8") + row = load_dashboard(tmp_path)["candidates"][0] + assert row["optimization_trace"] is None + assert row["artifacts"]["optimization_trace"]["missing_reason"] == "redacted" + assert row["artifacts"]["optimization_trace"]["sha256"] == "a" * 64 + + +def test_optional_round_and_redacted_section_preserve_the_caller_explanation(tmp_path): + setup_campaign(tmp_path) + document = trace_document() + del document["round"] + document["context"].update(state="redacted", text="", missing_reason="Private context withheld") + write_trace(tmp_path / "review/optimization.json", document) + record(tmp_path) + trace = load_dashboard(tmp_path)["candidates"][0]["optimization_trace"] + assert trace["round"] is None + assert trace["context"]["state"] == "redacted" + assert trace["context"]["missing_reason"] == "Private context withheld" + + +def test_hash_mismatch_and_invalid_format_do_not_expose_text(tmp_path): + setup_campaign(tmp_path) + path = tmp_path / "review/optimization.json" + write_trace(path, trace_document()) + record(tmp_path) + path.write_text('{"private": "FIXTURE_PRIVATE_TEXT"}', encoding="utf-8") + row = load_dashboard(tmp_path)["candidates"][0] + assert row["optimization_trace"] is None + assert row["artifacts"]["optimization_trace"]["missing_reason"] == "hash_mismatch" + assert "FIXTURE_PRIVATE_TEXT" not in json.dumps(row) + candidate = tmp_path / "candidates/candidate-a/candidate.json" + metadata = json.loads(candidate.read_text(encoding="utf-8")) + metadata["optimization_trace_ref"]["sha256"] = hashlib.sha256(path.read_bytes()).hexdigest() + candidate.write_text(json.dumps(metadata), encoding="utf-8") + row = load_dashboard(tmp_path)["candidates"][0] + assert row["optimization_trace"] is None + assert row["artifacts"]["optimization_trace"]["missing_reason"] == "invalid_format" + assert "FIXTURE_PRIVATE_TEXT" not in json.dumps(row) + + +def test_invalid_reference_state_is_not_treated_as_present(tmp_path): + setup_campaign(tmp_path) + write_trace(tmp_path / "review/optimization.json", trace_document()) + record(tmp_path) + candidate = tmp_path / "candidates/candidate-a/candidate.json" + metadata = json.loads(candidate.read_text(encoding="utf-8")) + metadata["optimization_trace_ref"]["state"] = "invalid" + candidate.write_text(json.dumps(metadata), encoding="utf-8") + row = load_dashboard(tmp_path)["candidates"][0] + assert row["optimization_trace"] is None + assert row["artifacts"]["optimization_trace"]["missing_reason"] == "invalid_format" + + +@pytest.mark.parametrize("identity", ["candidate_id", "campaign_id", "round"]) +def test_record_rejects_wrong_identity_without_writing_or_echoing(tmp_path, identity): + setup_campaign(tmp_path) + document = trace_document() + document[identity] = 2 if identity == "round" else "other" + document["context"]["text"] = "FIXTURE_PRIVATE_TEXT" + write_trace(tmp_path / "review/optimization.json", document) + with pytest.raises(ValueError, match="identity_mismatch") as error: + record(tmp_path) + assert "FIXTURE_PRIVATE_TEXT" not in str(error.value) + assert not (tmp_path / "candidates/candidate-a/candidate.json").exists() + assert not (tmp_path / "rounds.jsonl").exists() + + +def test_duplicate_candidate_and_ambiguous_round_withhold_trace(tmp_path): + setup_campaign(tmp_path) + write_trace(tmp_path / "review/optimization.json", trace_document()) + record(tmp_path) + original = tmp_path / "candidates/candidate-a/candidate.json" + duplicate = tmp_path / "candidates/duplicate/candidate.json" + duplicate.parent.mkdir(parents=True) + duplicate.write_text(original.read_text(encoding="utf-8"), encoding="utf-8") + for row in load_dashboard(tmp_path)["candidates"]: + assert row["optimization_trace"] is None + assert row["artifacts"]["optimization_trace"]["missing_reason"] == "identity_ambiguous" + duplicate.unlink() + with (tmp_path / "rounds.jsonl").open("a", encoding="utf-8") as stream: + stream.write(json.dumps({"round": 2, "candidates": ["candidate-a"]}) + "\n") + row = load_dashboard(tmp_path)["candidates"][0] + assert row["round_state"] == "ambiguous" + assert row["artifacts"]["optimization_trace"]["missing_reason"] == "identity_ambiguous" + + +@pytest.mark.parametrize("relative", ["../outside.json", "/outside.json", "C:\\outside.json", + "review\\..\\outside.json"]) +def test_unsafe_reference_never_reads_outside_campaign(tmp_path, relative): + setup_campaign(tmp_path) + with pytest.raises(ValueError, match="unsafe_path"): + record(tmp_path, relative) + assert not (tmp_path / "candidates/candidate-a/candidate.json").exists() + + +def test_symlink_escape_is_refused(tmp_path): + root = tmp_path / "campaign" + setup_campaign(root) + outside = tmp_path / "outside.json" + write_trace(outside, trace_document()) + link = root / "outside-link.json" + try: + link.symlink_to(outside) + except OSError: + pytest.skip("file symlinks are unavailable") + with pytest.raises(ValueError, match="unsafe_path"): + record(root, "outside-link.json") + + +@pytest.mark.parametrize("field", ["review", "partial", "contradiction", "order", "duplicate_key"]) +def test_invalid_review_state_and_malformed_records_are_rejected(field): + document = trace_document() + if field == "review": + document["reviewed_by_caller"] = False + elif field == "partial": + document["context"]["state"] = "partial" + elif field == "contradiction": + document["context"]["missing_reason"] = "Some context was withheld" + elif field == "order": + document["steps"]["items"].append({**document["steps"]["items"][0]}) + raw = json.dumps(document).encode("utf-8") + if field == "duplicate_key": + raw = raw[:-1] + b', "schema_version": 1}' + with pytest.raises(TraceError): + parse_optimization_trace(raw) + + +def test_oversized_trace_is_rejected_without_partial_record(tmp_path): + setup_campaign(tmp_path) + path = tmp_path / "review/optimization.json" + path.parent.mkdir(parents=True) + path.write_bytes(b"x" * (MAX_OPTIMIZATION_TRACE_BYTES + 1)) + with pytest.raises(ValueError, match="too_large"): + record(tmp_path) + assert not (tmp_path / "candidates/candidate-a/candidate.json").exists() + + +def test_hash_and_parse_use_same_snapshot_even_if_file_changes(tmp_path, monkeypatch): + path = tmp_path / "optimization.json" + original = trace_document() + digest = write_trace(path, original) + reader = optimization_trace._read + + def changing_reader(root, relative): + data, reference = reader(root, relative) + path.write_text('{"changed": true}', encoding="utf-8") + return data, reference + + monkeypatch.setattr(optimization_trace, "_read", changing_reader) + value, ref = read_optimization_trace( + tmp_path, {"path": "optimization.json", "sha256": digest}, campaign_id="campaign", + candidate_id="candidate-a", round_number=1, unique_candidate=True, + ) + assert value["context"]["text"] == original["context"]["text"] + assert ref["sha256"] == digest From 5a1a487a0aa9aaa4245dda6ca3d75e1c0b67cc87 Mon Sep 17 00:00:00 2001 From: David Chen <530634352@qq.com> Date: Tue, 3 Nov 2026 09:56:49 +0800 Subject: [PATCH 09/23] Capture and inspect optimization decision traces Agent-assisted-by: Codex (implementation) Signed-off-by: David Chen <530634352@qq.com> --- docs/dashboard.md | 120 ++++++++++++- plugins/inferrsi/skills/inferrsi/SKILL.md | 18 ++ src/inferrsi/arguments.py | 8 + src/inferrsi/cli.py | 10 +- src/inferrsi/data/dashboard/template.html | 166 ++++++++++++++---- src/inferrsi/harness/campaign_records.py | 41 ++++- src/inferrsi/harness/dashboard_data.py | 35 ++-- src/inferrsi/harness/optimization_trace.py | 86 ++++++++-- tests/test_optimization_trace.py | 191 +++++++++++++++++++-- 9 files changed, 596 insertions(+), 79 deletions(-) diff --git a/docs/dashboard.md b/docs/dashboard.md index 84188052c..7e7d31e49 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -45,6 +45,113 @@ campaign-relative files; the recorder hashes them and does not copy them. The caller assigns IDs and must serialize campaign record updates. Neither command starts an agent or validates the experiment. +### Capture the optimization trace + +The dashboard can show the assembled knowledge text and the caller's account of +the search and optimization process for each candidate. Collect this while +performing the work; old run logs and request hashes cannot reconstruct an +unrecorded context or reasoning. The runtime does not intercept an unrelated +agent or prove that a model received a prepared context. + +First save the exact assembled context that was prepared for the agent. For +example, `inferrsi context ... --output ` writes the text +produced by that operation; a host agent that combines other material must save +its own final assembly. Inspect the complete file and the narrative for +credentials, private values and unrelated material **before** recording. If a +safe copy omits or redacts anything, mark its state and explain the gap; do not +call it the complete original. Public source attribution may be retained. + +Create a UTF-8 JSON template like the following. Every field shown is required +except `round`, `run_id`, and `event_ref`. Values below are illustrative; record the actual search, +attempts and conclusions, including failures and omissions. Set +`reviewed_by_caller` to true only after reviewing the template and context. + +```json +{ + "schema_version": 1, + "reviewed_by_caller": true, + "campaign_id": "", + "candidate_id": "", + "round": 1, + "run_id": "", + "context": { + "state": "complete", + "text": "", + "queries": [""], + "sources": [ + {"ref": "", "revision": "", "locator": ""} + ], + "missing_reason": "" + }, + "search_space": { + "state": "partial", + "items": [{"id": "option-a", "description": ""}], + "missing_reason": "" + }, + "steps": { + "state": "partial", + "items": [ + {"sequence": 1, "kind": "attempt", "action": "", + "motivation": "", "outcome": "inconclusive", "result": "", + "search_id": "option-a", + "event_ref": {"source": "events", "line": 1, "sha256": ""}} + ], + "missing_reason": "" + }, + "summary": { + "state": "partial", + "successes": [""], + "failures": [""], + "limitations": [""], + "missing_reason": "" + } +} +``` + +Each section state is `complete`, `partial` or `redacted`; partial and +redacted sections require a nonempty `missing_reason`. The context text must +be nonempty when complete. Empty lists indicate no entries were supplied. +Source revisions and locators are caller-provided; use an empty string when +unknown. Search-space entries describe options, not necessarily attempts. +Steps have increasing sequence numbers, kinds `observation`, `attempt`, or +`decision`, and outcomes `succeeded`, `failed`, `inconclusive`, `skipped`, +or `not_recorded`. The `search_id` field is required; leave it empty when +there is no linked option. An optional `event_ref` names the `events` or `agent_trace` +source, its one-based line number and the source file's SHA-256; a jump is +offered only when that exact linked event and hash are present. Narrative +conclusions are not experiment verdicts. If `run_id` is omitted, the snapshot +is candidate-level and is not asserted to describe the selected run. + +Insert the complete, reviewed context file without manually escaping or +truncating it; this creates a **new** snapshot and never overwrites an existing +file: + +```sh +inferrsi campaign assemble-trace --directory \ + --template --context-file \ + --output optimization-v1.json +``` + +Use `--optimization-trace optimization-v1.json` with `campaign record`. +An existing candidate record may instead contain an explicit +`optimization_trace_ref` with the snapshot filename and SHA-256: + +```json +{"path": "optimization-v1.json", "sha256": ""} +``` + +The CLI does not update an existing candidate's reference; use a new campaign +record or explicitly update the record and its hash. Preserve earlier snapshots. +After modifying a referenced file its hash no longer matches, so it is +withheld rather than silently displayed. +Alternatively, a caller may create the complete JSON itself. The entire JSON +snapshot is limited to 2 MiB; larger snapshots are rejected, never silently +truncated. Trace snapshots must be single files directly in the campaign root. +Treat the campaign root itself as a trusted directory that is not concurrently +replaced. IDs, optional round/run, filename and SHA-256 are +checked before the content is shown. Source references in the narrative remain +caller assertions; inspect the separate evidence artifacts for measured claims. + To explain a candidate, pass `--details-json ` to `campaign record`. Supported fields are filtered and saved in the candidate record. The JSON can contain these optional fields (text is caller-supplied, not inferred or verified @@ -90,6 +197,7 @@ Record one decision for a round with `inferrsi campaign decide --directory campaign.json rounds.jsonl summary-chart.json # optional, explicitly recorded chain + optimization-v1.json # optional, reviewed narrative snapshot candidates// candidate.json agent_trace.jsonl # optional, explicitly referenced and hashed @@ -104,7 +212,7 @@ An undeclared stage is marked `not_applicable` when `stages_declared` is supplie otherwise a stage with no recorded events is marked `stage_not_recorded`. Each `candidate.json` may provide `candidate_id`, `title`, `round`, `run_id`, -`agent_trace_ref`, and `profile_ref`. An explicit reference is an object with a +`agent_trace_ref`, `optimization_trace_ref`, and `profile_ref`. An explicit reference is an object with a campaign-relative `path` and the file's SHA-256 in `sha256`. Missing references can instead specify `state: "missing"` and `missing_reason`. Paths escaping the campaign, including through symlinks, are rejected. A missing hash or hash @@ -173,8 +281,8 @@ markers. Unrelated candidate measurements are not joined into a cumulative line. measurement. Missing stages are not diagnosed as successful. Profiling interpretation, automatic root-cause attribution, automatic capture of unrelated agents and framework write-back are not implemented. -- The stage matrix describes recorded coverage. The separate time chart does not - infer durations or result timestamps when they are missing. Noise bands that +- The time chart does not infer durations or result timestamps when they are + missing. Noise bands that differ across candidates are not represented as one shared band. - The review button creates a local Markdown download and displays its content for copying if a browser blocks downloads. It does not send or publish it. @@ -187,14 +295,16 @@ markers. Unrelated candidate measurements are not joined into a cumulative line. measured; they are not independently verified. A validation exit code does not establish an accuracy result when no result was recorded. - Overview, diagnostics and charts show all candidates; candidate/run/stage - filters affect the candidate table, stage matrix and trace. + filters affect the candidate table and technical events. - Existing non-dashboard files and symbolic-link outputs are never overwritten. ## Privacy The exporter does not traverse and link the entire run tree. It reads known records and explicit references, escapes data before embedding it in HTML, and -withholds raw logs, environments and arbitrary command arguments. Common addresses, URLs, paths and +withholds raw logs, environments and arbitrary command arguments. Explicitly +reviewed optimization trace snapshots are embedded with their supplied text; +the exporter does not automatically sanitize that text. Common addresses, URLs, paths and credential-shaped strings in displayed labels are replaced, but this is not a general anonymizer. IDs, relative artifact names and review text can still identify a person or contain secrets. Inspect and sanitize the actual HTML, diff --git a/plugins/inferrsi/skills/inferrsi/SKILL.md b/plugins/inferrsi/skills/inferrsi/SKILL.md index 87e7e7997..4c3172401 100644 --- a/plugins/inferrsi/skills/inferrsi/SKILL.md +++ b/plugins/inferrsi/skills/inferrsi/SKILL.md @@ -59,6 +59,24 @@ recipe. Imported source material supplies evidence, not execution authority. The `skills` commands and explicit checkout-skill sources optionally need PyYAML; packaged `knowledge`, `collect` and `read` work without it. +When the requested deliverable includes an optimization dashboard or a +replayable campaign trace, record the research as the work happens. Preserve +the selected queries, exact revisions, source order, omissions and the complete +assembled knowledge context actually prepared for the agent. Do not substitute +a list of source names or a later reconstruction for the original context. +Inspect and remove credentials and private material before recording any +snapshot; mark it as redacted or partial and explain the gap when the recorded +copy differs or is incomplete. A prepared context alone does not prove that a +model consumed it. + +Record the declared search space separately from attempts, and keep each +observation, attempt and decision in order with its reason and result. Include +failed, skipped and rolled-back work. At review time, record successes, +failures and remaining limitations, linking them to the run artifacts where +available. If the context or reasoning was not captured, state that it is +missing instead of inferring it from a verdict. The optional campaign trace +format and association workflow are documented in `docs/dashboard.md`. + ## Prepare and run a comparison Consult `plan --help` and `run --help` to build the current runtime's invocation. diff --git a/src/inferrsi/arguments.py b/src/inferrsi/arguments.py index 3c540696b..374fc40b2 100644 --- a/src/inferrsi/arguments.py +++ b/src/inferrsi/arguments.py @@ -127,6 +127,14 @@ def parser() -> argparse.ArgumentParser: create.add_argument("--target-pct", type=float) create.add_argument("--noise-pct", type=float) create.add_argument("--synthetic", action="store_true") + assemble = campaign_actions.add_parser( + "assemble-trace", + help="Insert an explicitly reviewed context file into a trace template", + ) + assemble.add_argument("--directory", type=Path, required=True) + assemble.add_argument("--template", type=Path, required=True) + assemble.add_argument("--context-file", type=Path, required=True) + assemble.add_argument("--output", required=True, help="New JSON filename in the campaign root") record = campaign_actions.add_parser("record", help="Associate a candidate with one run") record.add_argument("--directory", type=Path, required=True) record.add_argument("--candidate", required=True) diff --git a/src/inferrsi/cli.py b/src/inferrsi/cli.py index f925a1047..2681cfa1d 100644 --- a/src/inferrsi/cli.py +++ b/src/inferrsi/cli.py @@ -287,11 +287,19 @@ def main(argv: list[str] | None = None) -> int: return 0 if args.command == "campaign": - from .harness.campaign_records import create_campaign, record_candidate, record_decision + from .harness.campaign_records import ( + assemble_trace, + create_campaign, + record_candidate, + record_decision, + ) if args.campaign_action == "init": path = create_campaign(args.directory, args.id, target=args.target_pct, noise=args.noise_pct, synthetic=args.synthetic) + elif args.campaign_action == "assemble-trace": + path = assemble_trace(args.directory, template=args.template, + context_file=args.context_file, output=args.output) elif args.campaign_action == "record": details = None if args.details_json is not None: diff --git a/src/inferrsi/data/dashboard/template.html b/src/inferrsi/data/dashboard/template.html index aeb7898db..d2323a10b 100644 --- a/src/inferrsi/data/dashboard/template.html +++ b/src/inferrsi/data/dashboard/template.html @@ -58,14 +58,30 @@ .item:last-child { border-bottom:0; } .item small, .item code { display:block; color:var(--muted); margin-top:3px; } .item.missing-item { border-left:2px dashed var(--red); } code, .mono { font:11px/1.55 Consolas,monospace; overflow-wrap:anywhere; } -.timeline { display:grid; grid-template-columns:130px repeat(6,minmax(95px,1fr)); gap:3px; min-width:730px; padding:8px 12px 12px; } -.timeline > div { padding:5px; background:var(--panel2); color:var(--muted); font-size:11px; } -.timeline .recorded { color:var(--green); } .timeline .missing { border:1px dashed #596170; } textarea { display:block; width:100%; min-height:85px; resize:vertical; background:var(--bg); border:1px solid var(--line); border-radius:2px; padding:8px; } .preview { white-space:pre-wrap; overflow-wrap:anywhere; max-height:260px; overflow:auto; border:1px solid var(--line); background:var(--bg); padding:8px; } -.details-grid { display:grid; grid-template-columns:repeat(2,minmax(0,1fr)); gap:8px; } +.candidate-inspection { padding:12px; } +.candidate-heading { display:flex; align-items:center; flex-wrap:wrap; gap:8px; padding-bottom:12px; } +.candidate-heading h3 { font-size:15px; margin:0; } +.candidate-heading small { color:var(--muted); } +.insight-grid { display:grid; grid-template-columns:repeat(2,minmax(0,1fr)); gap:8px; } +.insight { background:#14171b; border:1px solid var(--line); min-width:0; } +.insight.wide { grid-column:1 / -1; } +.trace-heading { grid-column:1 / -1; padding:6px 0 0; } +.trace-heading h4 { display:inline; margin:0; font-size:14px; } +.trace-heading small { color:var(--muted); margin-left:8px; } +.insight-head { display:flex; gap:8px; align-items:baseline; padding:10px 12px 4px; } +.insight-head h4 { margin:0; font-size:13px; } +.insight-head small { color:var(--muted); } +.context-text { max-height:360px; overflow:auto; white-space:pre-wrap; overflow-wrap:anywhere; + color:var(--ink); background:#101217; padding:10px; margin:8px 12px 12px; border:1px solid var(--line); } +.context-detail { margin:8px 12px 12px; } +.context-detail summary { cursor:pointer; color:#bdd5ff; } +.context-detail .context-text { margin:8px 0 0; } +.step { margin:8px 12px; padding:8px 10px; border-left:2px solid #477bd2; background:var(--panel2); } +.step strong, .step small { display:block; } .step small { color:var(--muted); } .detail-line { padding:7px 12px; border-bottom:1px solid #292e36; } .detail-line strong { display:block; color:var(--muted); font-size:11px; margin-bottom:3px; } .detail-line p { margin:0; white-space:pre-wrap; overflow-wrap:anywhere; } @@ -77,7 +93,7 @@ .trace-event pre { white-space:pre-wrap; overflow-wrap:anywhere; color:var(--muted); margin:6px 0; } .button { border:1px solid #477bd2; border-radius:2px; background:#315eac; padding:5px 12px; margin-top:8px; } .footnote { color:var(--muted); font-size:11px; margin:12px 0 0; } -@media (max-width:900px) { .columns, .details-grid { grid-template-columns:1fr; } .top { flex-wrap:wrap; padding:10px 18px; } } +@media (max-width:900px) { .columns, .insight-grid { grid-template-columns:1fr; } .insight.wide { grid-column:auto; } .top { flex-wrap:wrap; padding:10px 18px; } } @@ -97,30 +113,24 @@
Overview · all candidates
-
Workflow Select a stage to filter candidates and trace
+
Workflow Technical events only; select a stage to filter candidates and events
-
Selected optimization Choose a candidate to inspect its recorded change and validation
-
-

What changed

Caller-recorded details
-

Benchmark and accuracy

Commands from run.lock.json are redacted
-
Performance and evidence Recorded values, including unsuccessful attempts
+

Candidate details

Select a row to inspect its trace
+
RoundCandidateRunStateVerdictGainQualityOutcomeDecisionEvidence
+ +

Performance trend · all candidates

__DASHBOARD_CHART__

Results and stages over time · all candidates

__DASHBOARD_TEMPORAL__

-

Stage coverage

Recorded events; not a time axis
-
-

Candidate details

Select a row to inspect its trace
-
RoundCandidateRunStateVerdictGainQualityOutcomeDecisionEvidence
-

Diagnosis · all candidates

Only rules backed by referenced evidence
-

Trace

+

Technical events

@@ -197,33 +207,125 @@ button.addEventListener('click',()=>{jumpNotice='';stageSelect.value=stageSelect.value === stage.name ? '' : stage.name;render();}); root.appendChild(button); } } - function renderTimeline(visible) { - const root=byId('timeline');clear(root);root.appendChild(node('div','Candidate')); - for(const stage of data.stages) root.appendChild(node('div',stageNames[stage.name])); - for(const {row,index} of visible){root.appendChild(node('div',identity(row,index)));for(const stage of data.stages){const present=row.events.some(e=>e.stage===stage.name);const label=present?'RECORDED':stage.state==='not_applicable'?'NOT APPLICABLE':'NOT RECORDED';root.appendChild(node('div',label,present?'recorded':'missing'));}} - } function renderRows(visible) { const root=byId('candidate-rows');clear(root);byId('no-rows').hidden=visible.length>0; if(!visible.some(x=>x.index===selected)) selected=visible.length ? visible[0].index : null; for(const {row,index} of visible){const tr=node('tr');if(index===selected)tr.className='selected'; const cell=(text)=>{const td=node('td',text);tr.appendChild(td);return td;}; - cell(row.round === null ? row.round_state==='ambiguous'?'Ambiguous':'Not recorded' : String(row.round));const title=cell();const btn=node('button',row.title);btn.type='button';btn.setAttribute('aria-label',`Inspect ${identity(row,index)}`);btn.addEventListener('click',()=>{jumpNotice='';selected=index;render();});title.appendChild(btn);if(!['present','not_applicable'].includes(row.candidate_metadata_state))title.appendChild(pill(`metadata_${row.candidate_metadata_state}`)); + cell(row.round === null ? row.round_state==='ambiguous'?'Ambiguous':'Not recorded' : String(row.round));const title=cell();const btn=node('button',row.title);btn.type='button';btn.setAttribute('aria-label',`Inspect ${identity(row,index)}`);btn.setAttribute('aria-pressed',String(index===selected));btn.setAttribute('aria-controls','candidate-inspection');btn.addEventListener('click',()=>{jumpNotice='';selected=index;render();});title.appendChild(btn);if(!['present','not_applicable'].includes(row.candidate_metadata_state))title.appendChild(pill(`metadata_${row.candidate_metadata_state}`)); cell(row.run_id || 'Not recorded');cell(row.state.replaceAll('_',' '));cell().appendChild(pill(row.verdict));cell(value(row.improvement_pct,'%'));cell().appendChild(pill(row.quality));cell().appendChild(pill(row.outcome));cell(row.decision.replaceAll('_',' ')); const refs=Object.values(row.artifacts);cell(`${refs.filter(r=>r.state==='present').length} present / ${refs.filter(r=>r.state==='missing').length} missing`);root.appendChild(tr);} } - function renderOptimization(row) { - const optimization=byId('optimization'), benchmark=byId('benchmark');clear(optimization);clear(benchmark); - byId('optimization-title').textContent=row?identity(row,selected):'No selection'; - if(!row){detailLine(optimization,'Change','No matching candidate.');detailLine(benchmark,'Measurement','No matching candidate.');return;} + function insight(parent, title, note, wide=false) { + const section=node('section',undefined,`insight${wide?' wide':''}`); + const heading=node('div',undefined,'insight-head');heading.appendChild(node('h4',title)); + if(note)heading.appendChild(node('small',note));section.appendChild(heading);parent.appendChild(section); + return section; + } + function recordedState(section) { + if(!section)return 'Not recorded'; + const state=`Caller declared · ${section.state.replaceAll('_',' ')}`; + return section.missing_reason ? `${state} · ${section.missing_reason}` : state; + } + function renderCandidate(row) { + const root=byId('candidate-inspection');clear(root); + if(!row){root.appendChild(node('p','No matching candidate.','empty'));return;} + const heading=node('div',undefined,'candidate-heading'); + heading.appendChild(node('h3',row.title)); + heading.appendChild(node('small',`Candidate ${identity(row,selected)}`)); + if(data.synthetic)heading.appendChild(node('span','SYNTHETIC DATA','badge sample')); + heading.appendChild(pill(row.verdict)); + heading.appendChild(node('small','Narrative is caller recorded; verdicts and measurements are separate evidence.')); + root.appendChild(heading); + const grid=node('div',undefined,'insight-grid');root.appendChild(grid); const info=row.optimization, context=row.benchmark_context; + const optimization=insight(grid,'Change and rationale','Caller recorded',true); detailLine(optimization,'Change type',info.kind || 'Not recorded'); detailLine(optimization,'What was changed',info.summary); detailLine(optimization,'Why / hypothesis',info.hypothesis); if(!info.code_locations.length)detailLine(optimization,'Code / configuration locations','Not recorded'); for(const location of info.code_locations){const lines=location.line_start===null?'':`:${location.line_start}${location.line_end===null?'':`-${location.line_end}`}`;const revision=location.revision?` · revision ${location.revision}`:'';detailLine(optimization,'Code / configuration location',`${location.scope} · ${location.path || '[redacted or invalid path]'}${lines}${revision}\n${location.note}`);} - const traceCount=`${row.events.length} linked events · ${row.unlinked_agent_events.length} unlinked`; - detailLine(optimization,'Trace coverage',traceCount); - const actions=node('div',undefined,'detail-line'),jump=node('button','Open selected trace','button');jump.type='button';jump.addEventListener('click',()=>{const target=byId('trace-search');target.focus({preventScroll:true});target.scrollIntoView({block:'center'});});actions.appendChild(jump);optimization.appendChild(actions); + const trace=row.optimization_trace; + const traceHeading=node('div',undefined,'trace-heading'); + traceHeading.appendChild(node('h4','Optimization trace')); + traceHeading.appendChild(node('small',!trace ? 'Trace not recorded' : trace.run_id + ? `Caller bound this narrative to run ${trace.run_id}` + : 'Candidate-level narrative; no run binding recorded')); + grid.appendChild(traceHeading); + const knowledge=insight(grid,'Knowledge context','Caller-reviewed snapshot; not proof of model consumption',true); + const knowledgeContext=trace && trace.context; + detailLine(knowledge,'Capture status',recordedState(knowledgeContext)); + detailLine(knowledge,'Search queries',knowledgeContext && knowledgeContext.queries.length ? knowledgeContext.queries.join('\n') : 'Not recorded'); + if(knowledgeContext && knowledgeContext.sources.length){ + for(const source of knowledgeContext.sources)detailLine(knowledge,'Selected source',`${source.ref} · revision ${source.revision || 'Not recorded'}${source.locator ? ` · ${source.locator}` : ''}`,true); + }else detailLine(knowledge,'Selected sources','Not recorded'); + if(knowledgeContext && knowledgeContext.text){ + const fold=node('details',undefined,'context-detail');fold.open=true; + fold.appendChild(node('summary',`View recorded context · ${knowledgeContext.state}`)); + fold.appendChild(node('pre',knowledgeContext.text,'context-text'));knowledge.appendChild(fold); + }else detailLine(knowledge,'Context text','Not recorded'); + if(row.artifacts.optimization_trace)detailLine(knowledge,'Trace evidence',refText(row.artifacts.optimization_trace),true); + + const space=trace && trace.search_space; + const spacePanel=insight(grid,'Declared search space','Options are not necessarily attempts',true); + detailLine(spacePanel,'Capture status',recordedState(space)); + if(space && space.items.length){ + const attemptsByOption=new Map(); + for(const step of trace.steps.items){ + if(step.kind==='attempt' && step.search_id)attemptsByOption.set(step.search_id,(attemptsByOption.get(step.search_id) || 0)+1); + } + for(const option of space.items){ + const attempts=attemptsByOption.get(option.id) || 0; + detailLine(spacePanel,option.id,`${option.description}\n${attempts ? `${attempts} linked recorded attempt(s)` : 'No linked attempt recorded'}`); + } + }else detailLine(spacePanel,'Options','Not recorded'); + + const narrative=trace && trace.steps; + const process=insight(grid,'Optimization process','Recorded order and rationale',true); + detailLine(process,'Capture status',recordedState(narrative)); + if(narrative && narrative.items.length){ + const linkedEvents=new Set(row.events.map(event=>`${event.source}:${event.line}`)); + for(const step of narrative.items){ + const entry=node('div',undefined,'step'); + entry.appendChild(node('strong',`${step.sequence} · ${step.kind.replaceAll('_',' ')} · ${step.outcome.replaceAll('_',' ')}`)); + if(step.search_id)entry.appendChild(node('small',`Search option: ${step.search_id}`)); + detailLine(entry,'Action',step.action); + detailLine(entry,'Why',step.motivation); + detailLine(entry,'Recorded result',step.result); + const eventRef=step.event_ref; + if(!eventRef)detailLine(entry,'Technical event','Not recorded'); + else { + const artifact=row.artifacts[eventRef.source]; + const linked=linkedEvents.has(`${eventRef.source}:${eventRef.line}`); + const valid=linked && artifact && artifact.state==='present' + && artifact.sha256===eventRef.sha256.toLowerCase(); + if(!valid)detailLine(entry,'Technical event','Unavailable, unlinked, or source hash mismatch'); + else { + const action=node('div',undefined,'detail-line'); + action.appendChild(node('strong','Technical event')); + const jump=node('button',`Open ${eventRef.source}:${eventRef.line}`,'button'); + jump.type='button'; + jump.addEventListener('click',()=>{ + stageSelect.value='';byId('trace-search').value='';byId('trace-source').value=''; + render(); + const target=byId(`trace-${eventRef.source}-${eventRef.line}`); + if(target){target.open=true;target.setAttribute('tabindex','-1');target.focus({preventScroll:true});target.scrollIntoView({block:'center'});} + }); + action.appendChild(jump);entry.appendChild(action); + } + } + process.appendChild(entry); + } + }else detailLine(process,'Steps','Not recorded'); + + const conclusion=trace && trace.summary; + const conclusions=insight(grid,'Successes and failures','Caller conclusions; see evidence for verdicts',true); + detailLine(conclusions,'Capture status',recordedState(conclusion)); + detailLine(conclusions,'Success summary',conclusion && conclusion.successes.length ? conclusion.successes.join('\n\n') : 'Not recorded'); + detailLine(conclusions,'Failure summary',conclusion && conclusion.failures.length ? conclusion.failures.join('\n\n') : 'Not recorded'); + detailLine(conclusions,'Limitations',conclusion && conclusion.limitations.length ? conclusion.limitations.join('\n\n') : 'Not recorded'); + + const benchmark=insight(grid,'Benchmark and accuracy','Measurements and redacted commands',true); detailLine(benchmark,'Performance methodology · caller recorded',info.benchmark); detailLine(benchmark,'Workload',`${context.scenario} · ${context.workload} · ${context.repetitions===null?'repetitions not recorded':`${context.repetitions} repetitions`}`); detailLine(benchmark,'Recorded medians',`Baseline ${measurement(row.baseline_median)} · Candidate ${measurement(row.candidate_median)} ${row.metric_unit}`); @@ -249,7 +351,7 @@ } function renderDetail() { const trace=byId('trace'), evidence=byId('evidence');clear(trace);clear(evidence); - const row=selected===null?null:data.candidates[selected];renderOptimization(row); + const row=selected===null?null:data.candidates[selected]; if(!row){byId('trace-title').textContent='No selection';byId('trace-count').textContent='';item(trace,'NOT RECORDED','No matching candidate.');item(evidence,'NOT RECORDED');return;} byId('trace-title').textContent=identity(row,selected); const {visible,total}=traceEvents(row);byId('trace-count').textContent=`${visible.length} of ${total} events · time order, unknown time last`; @@ -263,7 +365,7 @@ for(const ref of Object.values(row.artifacts)){const entry=item(evidence,ref.state === 'present' ? `${ref.kind} · PRESENT` : `${ref.kind} · MISSING`,ref.state === 'present' ? ref.path : ref.missing_reason,ref.sha256 ? `sha256 ${ref.sha256}` : '');if(ref.state==='missing')entry.classList.add('missing-item');} } function renderDiagnostics(){const root=byId('diagnostics');clear(root);if(!data.diagnostics.length)item(root,'NOT RECORDED','No evidence-backed diagnostic rule matched.');for(const d of data.diagnostics){const entry=item(root,'',d.candidate_id || 'Single run',d.refs.map(refText).join(' | '));const button=node('button',d.kind.replaceAll('_',' '),'diagnostic-button');button.type='button';const matches=data.candidates.map((row,index)=>({row,index})).filter(({row})=>row.candidate_id===d.candidate_id && row.run_id===d.run_id && row.round===d.round);button.disabled=matches.length!==1;button.addEventListener('click',()=>{const {row,index}=matches[0];const phases=d.kind==='execution_failed'?['crashed','measurement']:d.kind==='quality_failed_with_gain'?['quality_validation']:d.kind==='rolled_back'?[]:['measurement'];let event=null;for(const phase of phases){event=[...row.events].reverse().find(e=>e.phase===phase);if(event)break;}runSelect.value='';candidateSelect.value=String(index);stageSelect.value=event && event.stage || '';byId('trace-search').value='';byId('trace-source').value='';selected=index;jumpNotice=event?'': 'No corresponding trace event was recorded; inspect the referenced evidence.';render();const target=event?byId(`trace-${event.source}-${event.line}`):byId('trace');if(target){if(target.tagName==='DETAILS')target.open=true;target.setAttribute('tabindex','-1');target.focus({preventScroll:true});target.scrollIntoView({block:'center'});}});entry.querySelector('strong').replaceWith(button);}} - function render(){const visible=filtered();renderWorkflow();renderTimeline(visible);renderRows(visible);renderDetail();} + function render(){const visible=filtered();renderWorkflow();renderRows(visible);renderCandidate(selected===null?null:data.candidates[selected]);renderDetail();} for(const select of [stageSelect,runSelect,candidateSelect])select.addEventListener('change',()=>{jumpNotice='';render();}); byId('trace-search').addEventListener('input',renderDetail); byId('trace-source').addEventListener('change',renderDetail); diff --git a/src/inferrsi/harness/campaign_records.py b/src/inferrsi/harness/campaign_records.py index a1fa4b676..fc6411bdc 100644 --- a/src/inferrsi/harness/campaign_records.py +++ b/src/inferrsi/harness/campaign_records.py @@ -10,7 +10,12 @@ from pathlib import Path from .dashboard_data import _file_ref, _inside, _object, _relative_location, _safe_argv, _text -from .optimization_trace import read_optimization_trace +from .optimization_trace import ( + MAX_OPTIMIZATION_TRACE_BYTES, + assemble_optimization_trace, + parse_optimization_trace, + read_optimization_trace, +) def _identifier(value: str, name: str) -> str: @@ -84,6 +89,39 @@ def _write_rounds(root: Path, path: Path, records: list[dict]) -> None: temporary.unlink() +def assemble_trace(root: Path, *, template: Path, context_file: Path, output: str) -> Path: + """Create a new caller-reviewed trace from exact context-file bytes.""" + root = Path(root).resolve() + campaign, state = _object(root / "campaign.json", root) + if state != "present": + raise ValueError("campaign.json is missing or invalid") + _identifier(output, "trace output") + target = root / output + + def read_input(path: Path) -> bytes: + try: + with Path(path).open("rb") as stream: + return stream.read(MAX_OPTIMIZATION_TRACE_BYTES + 1) + except OSError as exc: + raise ValueError("trace template or reviewed context is unreadable") from exc + + payload = assemble_optimization_trace(read_input(template), read_input(context_file)) + document = parse_optimization_trace(payload) + if document["campaign_id"] != campaign.get("campaign_id"): + raise ValueError("trace campaign identity does not match") + temporary = root / f".optimization-{os.getpid()}-{uuid.uuid4().hex}.tmp" + try: + with temporary.open("xb") as stream: + stream.write(payload) + os.link(temporary, target) + except FileExistsError as exc: + raise ValueError("trace output already exists") from exc + finally: + if temporary.exists(): + temporary.unlink() + return target + + def record_candidate(root: Path, *, candidate_id: str, run_id: str, round_number: int, title: str, agent_trace: str | None = None, profile: str | None = None, details: dict | None = None, @@ -175,6 +213,7 @@ def record_candidate(root: Path, *, candidate_id: str, run_id: str, round_number _, ref = read_optimization_trace( root, {"path": optimization_trace}, campaign_id=campaign.get("campaign_id"), candidate_id=candidate_id, round_number=round_number, unique_candidate=True, + run_id=run_id, require_hash=False, ) if ref["state"] != "present": diff --git a/src/inferrsi/harness/dashboard_data.py b/src/inferrsi/harness/dashboard_data.py index 716fa380a..d673bcecf 100644 --- a/src/inferrsi/harness/dashboard_data.py +++ b/src/inferrsi/harness/dashboard_data.py @@ -514,17 +514,28 @@ def _environment_signature(environment: dict, devices: object, backend: object) def _run(root: Path, path: Path, candidate: dict, run_id: str | None, noise: float | None, campaign_id: object = None, unique_candidate: bool = False, candidate_ref: dict | None = None) -> dict: - lock, _, lock_ref = _object_snapshot(path / "run.lock.json", root, "run_lock") - repositories, _ = _object(path / "repositories.lock.json", root) - environment, _ = _object(path / "environment.json", root) - evaluation, evaluation_state, evaluation_ref = _object_snapshot( - path / "evaluation.json", root, "evaluation" - ) - status, status_state, status_ref = _object_snapshot(path / "status.json", root, "status") - validation, validation_state, quality_ref = _object_snapshot( - path / "validation" / "execution.json", root, "quality" - ) - events, corrupt, event_state, _, events_ref = _events(path / "events.jsonl", root) + if run_id is None: + def missing(kind: str) -> dict: + return {"kind": kind, "state": "missing", "missing_reason": "not_collected"} + + lock = repositories = environment = evaluation = status = validation = {} + lock_ref, evaluation_ref, status_ref, quality_ref, events_ref = ( + missing(kind) for kind in ("run_lock", "evaluation", "status", "quality", "events") + ) + evaluation_state = status_state = validation_state = event_state = "missing" + events, corrupt = [], [] + else: + lock, _, lock_ref = _object_snapshot(path / "run.lock.json", root, "run_lock") + repositories, _ = _object(path / "repositories.lock.json", root) + environment, _ = _object(path / "environment.json", root) + evaluation, evaluation_state, evaluation_ref = _object_snapshot( + path / "evaluation.json", root, "evaluation" + ) + status, status_state, status_ref = _object_snapshot(path / "status.json", root, "status") + validation, validation_state, quality_ref = _object_snapshot( + path / "validation" / "execution.json", root, "quality" + ) + events, corrupt, event_state, _, events_ref = _events(path / "events.jsonl", root) corrupt_sources = {"events": list(corrupt), "agent_trace": []} refs = { "run_lock": lock_ref, @@ -540,7 +551,7 @@ def _run(root: Path, path: Path, candidate: dict, run_id: str | None, noise: flo optimization_trace, refs["optimization_trace"] = read_optimization_trace( root, candidate.get("optimization_trace_ref"), campaign_id=campaign_id, candidate_id=candidate.get("candidate_id"), round_number=candidate.get("round"), - unique_candidate=unique_candidate, + unique_candidate=unique_candidate, run_id=run_id, round_ambiguous=candidate.get("_round_state") == "ambiguous", ) def attach_declared(event: dict) -> None: diff --git a/src/inferrsi/harness/optimization_trace.py b/src/inferrsi/harness/optimization_trace.py index aafa2adcb..7e14e3ede 100644 --- a/src/inferrsi/harness/optimization_trace.py +++ b/src/inferrsi/harness/optimization_trace.py @@ -8,9 +8,10 @@ import hashlib import json +import os import re import stat -from pathlib import Path, PurePosixPath, PureWindowsPath +from pathlib import Path MAX_OPTIMIZATION_TRACE_BYTES = 2 * 1024 * 1024 ATTRIBUTION = "caller_declared_not_proof_of_model_consumption" @@ -92,7 +93,7 @@ def parse_optimization_trace(raw: bytes) -> dict: raise TraceError("invalid_format") required = {"schema_version", "reviewed_by_caller", "campaign_id", "candidate_id", "context", "search_space", "steps", "summary"} - if not required <= set(value) or set(value) - (required | {"round"}): + if not required <= set(value) or set(value) - (required | {"round", "run_id"}): raise TraceError("invalid_format") if type(value["schema_version"]) is not int or value["schema_version"] != 1: raise TraceError("invalid_format") @@ -104,6 +105,9 @@ def parse_optimization_trace(raw: bytes) -> dict: round_number = value.get("round") if round_number is not None and (type(round_number) is not int or round_number < 1): raise TraceError("invalid_format") + run_id = value.get("run_id") + if run_id is not None and (not isinstance(run_id, str) or not _IDENTIFIER.fullmatch(run_id)): + raise TraceError("invalid_format") context = _section(value["context"], {"text", "queries", "sources"}) _string(context["text"], required=context["state"] == "complete") @@ -133,8 +137,11 @@ def parse_optimization_trace(raw: bytes) -> dict: raise TraceError("invalid_format") previous = 0 for item in steps["items"]: - item = _object(item, {"sequence", "kind", "action", "motivation", "outcome", "result", - "search_id"}) + required_step = {"sequence", "kind", "action", "motivation", "outcome", "result", + "search_id"} + if (not isinstance(item, dict) or not required_step <= set(item) + or set(item) - (required_step | {"event_ref"})): + raise TraceError("invalid_format") sequence = item["sequence"] if type(sequence) is not int or sequence <= previous: raise TraceError("invalid_format") @@ -148,30 +155,73 @@ def parse_optimization_trace(raw: bytes) -> dict: search_id = _string(item["search_id"]) if search_id and space["state"] == "complete" and search_id not in space_ids: raise TraceError("invalid_format") + if "event_ref" in item: + event = _object(item["event_ref"], {"source", "line", "sha256"}) + if (event["source"] not in ("events", "agent_trace") or type(event["line"]) is not int + or not 0 < event["line"] <= 10_000_000 + or not isinstance(event["sha256"], str) + or not _SHA256.fullmatch(event["sha256"])): + raise TraceError("invalid_format") summary = _section(value["summary"], {"successes", "failures", "limitations"}) for key in ("successes", "failures", "limitations"): _strings(summary[key]) - return {**value, "round": round_number, "attribution": ATTRIBUTION} + return {**value, "round": round_number, "run_id": run_id, "attribution": ATTRIBUTION} + + +def assemble_optimization_trace(template: bytes, context: bytes) -> bytes: + """Insert an explicitly reviewed UTF-8 context into a trace template. + + This does not inspect an agent request or assert that a model consumed the + context. The template and context must both be reviewed by the caller. + """ + if (not isinstance(template, bytes) or not isinstance(context, bytes) + or len(template) > MAX_OPTIMIZATION_TRACE_BYTES + or len(context) > MAX_OPTIMIZATION_TRACE_BYTES): + raise TraceError("too_large") + try: + value = json.loads(template.decode("utf-8"), object_pairs_hook=_unique_object, + parse_constant=_constant) + text = context.decode("utf-8") + except (UnicodeError, ValueError, RecursionError) as exc: + if isinstance(exc, TraceError): + raise + raise TraceError("invalid_format") from exc + if not isinstance(value, dict) or value.get("reviewed_by_caller") is not True: + raise TraceError("not_reviewed") + section = value.get("context") + if not isinstance(section, dict) or section.get("text") != "": + raise TraceError("invalid_format") + section["text"] = text + try: + result = (json.dumps(value, ensure_ascii=False, allow_nan=False) + "\n").encode("utf-8") + except (TypeError, ValueError, UnicodeError, RecursionError) as exc: + raise TraceError("invalid_format") from exc + parse_optimization_trace(result) + return result def _read(root: Path, raw_path: object) -> tuple[bytes | None, dict]: if not isinstance(raw_path, str) or not raw_path: return None, _missing("not_collected") - posix, windows = PurePosixPath(raw_path), PureWindowsPath(raw_path) - if (posix.is_absolute() or windows.is_absolute() or windows.drive or "\\" in raw_path - or any(ord(char) < 32 for char in raw_path) - or ".." in posix.parts or ".." in windows.parts): + # The campaign root is the trust boundary. A single filename avoids + # resolving attacker-replaceable intermediate directories. + if not _IDENTIFIER.fullmatch(raw_path) or raw_path in {".", ".."}: return None, _missing("unsafe_path") try: root = Path(root).resolve() - path = root.joinpath(*posix.parts).resolve() - relative = path.relative_to(root).as_posix() - if path == root: + path = root / raw_path + before = path.lstat() + if not stat.S_ISREG(before.st_mode) or not before.st_ino: return None, _missing("unsafe_path") - if not stat.S_ISREG(path.stat().st_mode): - return None, _missing("unreadable") - with path.open("rb") as stream: + flags = os.O_RDONLY | getattr(os, "O_BINARY", 0) | getattr(os, "O_NOFOLLOW", 0) + flags |= getattr(os, "O_NONBLOCK", 0) + descriptor = os.open(path, flags) + with os.fdopen(descriptor, "rb") as stream: + opened = os.fstat(stream.fileno()) + if (not stat.S_ISREG(opened.st_mode) or not opened.st_ino + or (opened.st_dev, opened.st_ino) != (before.st_dev, before.st_ino)): + return None, _missing("unsafe_path") raw = stream.read(MAX_OPTIMIZATION_TRACE_BYTES + 1) except (ValueError, RuntimeError): return None, _missing("unsafe_path") @@ -181,7 +231,7 @@ def _read(root: Path, raw_path: object) -> tuple[bytes | None, dict]: return None, _missing("unreadable") if len(raw) > MAX_OPTIMIZATION_TRACE_BYTES: return None, _missing("too_large") - return raw, {"kind": "optimization_trace", "state": "present", "path": relative, + return raw, {"kind": "optimization_trace", "state": "present", "path": raw_path, "sha256": hashlib.sha256(raw).hexdigest()} @@ -192,6 +242,7 @@ def _missing(reason: str) -> dict: def read_optimization_trace(root: Path, reference: object, *, campaign_id: object, candidate_id: object, round_number: object, unique_candidate: bool, round_ambiguous: bool = False, + run_id: object = None, require_hash: bool = True) -> tuple[dict | None, dict]: """Read, hash, validate and bind the exact same bounded snapshot.""" if not isinstance(reference, dict): @@ -223,6 +274,7 @@ def read_optimization_trace(root: Path, reference: object, *, campaign_id: objec return None, _missing("identity_ambiguous") if (not isinstance(campaign_id, str) or not isinstance(candidate_id, str) or value["campaign_id"] != campaign_id or value["candidate_id"] != candidate_id - or (value["round"] is not None and value["round"] != round_number)): + or (value["round"] is not None and value["round"] != round_number) + or (value["run_id"] is not None and value["run_id"] != run_id)): return None, _missing("identity_mismatch") return value, ref diff --git a/tests/test_optimization_trace.py b/tests/test_optimization_trace.py index 862b02d19..a32ef2724 100644 --- a/tests/test_optimization_trace.py +++ b/tests/test_optimization_trace.py @@ -2,11 +2,12 @@ import hashlib import json +from pathlib import Path import pytest from inferrsi.cli import main -from inferrsi.harness import optimization_trace +from inferrsi.harness import campaign_records, optimization_trace from inferrsi.harness.campaign_records import create_campaign, record_candidate from inferrsi.harness.dashboard import write_dashboard from inferrsi.harness.dashboard_data import load_dashboard @@ -55,7 +56,7 @@ def setup_campaign(root): (run / "run.lock.json").write_text("{}", encoding="utf-8") -def record(root, relative="review/optimization.json"): +def record(root, relative="optimization.json"): return record_candidate(root, candidate_id="candidate-a", run_id="run-a", round_number=1, title="Candidate A", optimization_trace=relative) @@ -64,11 +65,11 @@ def test_cli_preserves_full_reviewed_context_and_escapes_script(tmp_path, capsys setup_campaign(tmp_path) document = trace_document() document["context"]["text"] += "\n" - digest = write_trace(tmp_path / "review/optimization.json", document) + digest = write_trace(tmp_path / "optimization.json", document) assert main(["campaign", "record", "--directory", str(tmp_path), "--candidate", "candidate-a", "--run-id", "run-a", "--round", "1", "--title", "Candidate A", - "--optimization-trace", "review/optimization.json"]) == 0 + "--optimization-trace", "optimization.json"]) == 0 metadata = json.loads((tmp_path / "candidates/candidate-a/candidate.json").read_text()) assert metadata["optimization_trace_ref"]["sha256"] == digest row = load_dashboard(tmp_path)["candidates"][0] @@ -90,6 +91,116 @@ def test_cli_preserves_full_reviewed_context_and_escapes_script(tmp_path, capsys assert capsys.readouterr().out +def test_assembled_context_round_trips_from_actual_knowledge_command(tmp_path, capsys): + setup_campaign(tmp_path) + context_path = tmp_path / "reviewed-context.md" + assert main(["context", "decode", "--repo-type", "vllm", "--backend", "cuda", + "--output", str(context_path)]) == 0 + exact_context = context_path.read_bytes().decode("utf-8") + assert exact_context.startswith("# InferRSI knowledge context") + document = trace_document() + document["context"]["text"] = "" + template = tmp_path / "template.json" + template.write_text(json.dumps(document), encoding="utf-8") + + assert main(["campaign", "assemble-trace", "--directory", str(tmp_path), + "--template", str(template), "--context-file", str(context_path), + "--output", "optimization.json"]) == 0 + record(tmp_path, "optimization.json") + row = load_dashboard(tmp_path)["candidates"][0] + assert row["optimization_trace"]["context"]["text"] == exact_context + assert row["artifacts"]["optimization_trace"]["state"] == "present" + html = write_dashboard(tmp_path).read_text(encoding="utf-8") + payload = html.split('", 1)[0] + assert json.loads(payload)["candidates"][0]["optimization_trace"]["context"]["text"] == ( + exact_context + ) + assert capsys.readouterr().out + + +def test_assemble_requires_review_and_never_overwrites(tmp_path, capsys): + setup_campaign(tmp_path) + document = trace_document() + document["context"]["text"] = "" + document["reviewed_by_caller"] = False + template = tmp_path / "template.json" + template.write_text(json.dumps(document), encoding="utf-8") + context_path = tmp_path / "context.txt" + context_path.write_bytes(b"first line\r\nsecond line\n") + args = ["campaign", "assemble-trace", "--directory", str(tmp_path), + "--template", str(template), "--context-file", str(context_path), + "--output", "optimization.json"] + with pytest.raises(SystemExit): + main(args) + assert not (tmp_path / "optimization.json").exists() + document["reviewed_by_caller"] = True + template.write_text(json.dumps(document), encoding="utf-8") + assert main(args) == 0 + raw = (tmp_path / "optimization.json").read_bytes() + assert parse_optimization_trace(raw)["context"]["text"] == "first line\r\nsecond line\n" + with pytest.raises(SystemExit): + main(args) + assert (tmp_path / "optimization.json").read_bytes() == raw + assert capsys.readouterr().out + + +def test_assemble_rejects_subdirectories_and_traversal(tmp_path): + setup_campaign(tmp_path) + document = trace_document() + document["context"]["text"] = "" + template = tmp_path / "template.json" + template.write_text(json.dumps(document), encoding="utf-8") + context_path = tmp_path / "context.txt" + context_path.write_text("Reviewed context", encoding="utf-8") + args = ["campaign", "assemble-trace", "--directory", str(tmp_path), + "--template", str(template), "--context-file", str(context_path), "--output"] + for unsafe in ("subdir/optimization.json", "../outside.json", "C:\\outside.json"): + with pytest.raises(SystemExit): + main(args + [unsafe]) + + +def test_assemble_cleans_temporary_file_if_publish_fails(tmp_path, monkeypatch): + setup_campaign(tmp_path) + document = trace_document() + document["context"]["text"] = "" + template = tmp_path / "template.json" + template.write_text(json.dumps(document), encoding="utf-8") + context = tmp_path / "context.txt" + context.write_text("Reviewed context", encoding="utf-8") + + def fail_link(*args): + raise OSError("simulated publish failure") + + monkeypatch.setattr(campaign_records.os, "link", fail_link) + with pytest.raises(OSError): + campaign_records.assemble_trace(tmp_path, template=template, context_file=context, + output="optimization.json") + assert not (tmp_path / "optimization.json").exists() + assert not list(tmp_path.glob(".optimization-*.tmp")) + + +def test_run_binding_and_event_references_are_validated(tmp_path): + setup_campaign(tmp_path) + document = trace_document() + document["run_id"] = "different-run" + document["steps"]["items"][0]["event_ref"] = { + "source": "events", "line": 1, "sha256": "a" * 64, + } + write_trace(tmp_path / "optimization.json", document) + with pytest.raises(ValueError, match="identity_mismatch"): + record(tmp_path) + document["run_id"] = "run-a" + write_trace(tmp_path / "optimization.json", document) + record(tmp_path) + row = load_dashboard(tmp_path)["candidates"][0] + assert row["optimization_trace"]["run_id"] == "run-a" + assert row["optimization_trace"]["steps"]["items"][0]["event_ref"]["sha256"] == "a" * 64 + document["steps"]["items"][0]["event_ref"]["line"] = 0 + with pytest.raises(TraceError): + parse_optimization_trace(json.dumps(document).encode()) + + def test_old_candidates_and_explicit_redaction_stay_visible(tmp_path): setup_campaign(tmp_path) record_candidate(tmp_path, candidate_id="candidate-a", run_id="run-a", round_number=1, @@ -113,7 +224,7 @@ def test_optional_round_and_redacted_section_preserve_the_caller_explanation(tmp document = trace_document() del document["round"] document["context"].update(state="redacted", text="", missing_reason="Private context withheld") - write_trace(tmp_path / "review/optimization.json", document) + write_trace(tmp_path / "optimization.json", document) record(tmp_path) trace = load_dashboard(tmp_path)["candidates"][0]["optimization_trace"] assert trace["round"] is None @@ -123,7 +234,7 @@ def test_optional_round_and_redacted_section_preserve_the_caller_explanation(tmp def test_hash_mismatch_and_invalid_format_do_not_expose_text(tmp_path): setup_campaign(tmp_path) - path = tmp_path / "review/optimization.json" + path = tmp_path / "optimization.json" write_trace(path, trace_document()) record(tmp_path) path.write_text('{"private": "FIXTURE_PRIVATE_TEXT"}', encoding="utf-8") @@ -143,7 +254,7 @@ def test_hash_mismatch_and_invalid_format_do_not_expose_text(tmp_path): def test_invalid_reference_state_is_not_treated_as_present(tmp_path): setup_campaign(tmp_path) - write_trace(tmp_path / "review/optimization.json", trace_document()) + write_trace(tmp_path / "optimization.json", trace_document()) record(tmp_path) candidate = tmp_path / "candidates/candidate-a/candidate.json" metadata = json.loads(candidate.read_text(encoding="utf-8")) @@ -160,7 +271,7 @@ def test_record_rejects_wrong_identity_without_writing_or_echoing(tmp_path, iden document = trace_document() document[identity] = 2 if identity == "round" else "other" document["context"]["text"] = "FIXTURE_PRIVATE_TEXT" - write_trace(tmp_path / "review/optimization.json", document) + write_trace(tmp_path / "optimization.json", document) with pytest.raises(ValueError, match="identity_mismatch") as error: record(tmp_path) assert "FIXTURE_PRIVATE_TEXT" not in str(error.value) @@ -170,7 +281,7 @@ def test_record_rejects_wrong_identity_without_writing_or_echoing(tmp_path, iden def test_duplicate_candidate_and_ambiguous_round_withhold_trace(tmp_path): setup_campaign(tmp_path) - write_trace(tmp_path / "review/optimization.json", trace_document()) + write_trace(tmp_path / "optimization.json", trace_document()) record(tmp_path) original = tmp_path / "candidates/candidate-a/candidate.json" duplicate = tmp_path / "candidates/duplicate/candidate.json" @@ -210,6 +321,64 @@ def test_symlink_escape_is_refused(tmp_path): record(root, "outside-link.json") +def test_replacement_between_lstat_and_open_is_withheld(tmp_path, monkeypatch): + root = tmp_path / "campaign" + root.mkdir() + target = root / "optimization.json" + outside = tmp_path / "outside.json" + write_trace(target, trace_document()) + outside.write_text('{"private":"FIXTURE_PRIVATE_TEXT"}', encoding="utf-8") + probe = root / "probe-link" + try: + probe.symlink_to(outside) + except OSError: + pytest.skip("file symlinks are unavailable") + probe.unlink() + original_open = optimization_trace.os.open + swapped = False + + def swap_before_open(path, flags, *args, **kwargs): + nonlocal swapped + if Path(path) == target and not swapped: + swapped = True + target.unlink() + target.symlink_to(outside) + return original_open(path, flags, *args, **kwargs) + + monkeypatch.setattr(optimization_trace.os, "open", swap_before_open) + value, ref = read_optimization_trace( + root, {"path": "optimization.json"}, campaign_id="campaign", + candidate_id="candidate-a", round_number=1, unique_candidate=True, require_hash=False, + ) + assert swapped + assert value is None + assert ref["missing_reason"] in {"unsafe_path", "unreadable"} + assert "FIXTURE_PRIVATE_TEXT" not in json.dumps(ref) + + +def test_missing_run_does_not_read_placeholder_directory(tmp_path): + setup_campaign(tmp_path) + directory = tmp_path / "candidates" / "candidate-a" + fake_run = directory / "missing-run" + fake_run.mkdir(parents=True) + (directory / "candidate.json").write_text( + json.dumps({"candidate_id": "candidate-a", "title": "Candidate A", "round": 1}), + encoding="utf-8", + ) + (fake_run / "events.jsonl").write_text( + json.dumps({"phase": "measurement", "side": "candidate", "trial": 1}) + "\n", + encoding="utf-8", + ) + (fake_run / "evaluation.json").write_text( + json.dumps({"verdict": "PASS", "improvement_pct": 999}), encoding="utf-8", + ) + row = load_dashboard(tmp_path)["candidates"][0] + assert row["run_id"] is None + assert row["events"] == [] + assert row["improvement_pct"] is None + assert row["artifacts"]["events"]["missing_reason"] == "not_collected" + + @pytest.mark.parametrize("field", ["review", "partial", "contradiction", "order", "duplicate_key"]) def test_invalid_review_state_and_malformed_records_are_rejected(field): document = trace_document() @@ -230,8 +399,8 @@ def test_invalid_review_state_and_malformed_records_are_rejected(field): def test_oversized_trace_is_rejected_without_partial_record(tmp_path): setup_campaign(tmp_path) - path = tmp_path / "review/optimization.json" - path.parent.mkdir(parents=True) + path = tmp_path / "optimization.json" + path.parent.mkdir(parents=True, exist_ok=True) path.write_bytes(b"x" * (MAX_OPTIMIZATION_TRACE_BYTES + 1)) with pytest.raises(ValueError, match="too_large"): record(tmp_path) From 6cc1d4e69ba4499e3cf9951d1f2250a5c8c204f3 Mon Sep 17 00:00:00 2001 From: David Chen <530634352@qq.com> Date: Tue, 3 Nov 2026 10:02:29 +0800 Subject: [PATCH 10/23] Document reviewed trace assembly Agent-assisted-by: Codex (implementation) Signed-off-by: David Chen <530634352@qq.com> --- docs/operations.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/operations.md b/docs/operations.md index 8c3860439..1cac4a91f 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -27,6 +27,7 @@ Effect classes used below: | Inspect/prepare experiments | CLI `doctor`; `plan` or `--dry-run` | Read-only inspection / preview | `doctor` may invoke bounded probes and does not certify target compatibility; `plan` previews a specification without persisting a campaign or authorizing execution | | Execute/inspect experiments | CLI `run`; `status`; `report` | Experiment-executing / read-only / report-generating | `run` executes caller-supplied commands and records evidence; `status` reads recorded progress; `report` generates HTML from recorded evidence | | Visualize a recorded run or campaign | CLI `dashboard`; [offline dashboard](dashboard.md) | Report-generating | Writes a self-contained read-only HTML snapshot from recorded evidence; does not run an agent or validate a claim | +| Assemble an optimization trace | CLI `campaign assemble-trace`; [offline dashboard](dashboard.md#capture-the-optimization-trace) | Local evidence-writing | Copies an explicitly reviewed context into a new bounded campaign-root snapshot; caller owns redaction and narrative accuracy | | Associate runs with a local campaign | CLI `campaign init` / `campaign record` / `campaign decide`; [offline dashboard](dashboard.md) | Local evidence-writing | Creates caller-named campaign, round and candidate records; references existing artifacts and never starts an agent | | Adapt existing Flare delivery evidence | [Offline qualification adapter](flare-qualification.md), `python -m inferrsi.model.validators.flare` | Read-only | Existing raw packets and a separate receipt; one HTTP delivery metric, no model/server/device execution | | Evaluate selected skill revisions | [Bounded skill-evaluation driver](skill-evaluation.md) | Experiment-executing | Explicitly authorized real Codex calls, independent task verdicts and private evidence; separate from numeric engine-performance scoring | From 76de3c0ebfbf2990c757c53de39b0703acbbc00e Mon Sep 17 00:00:00 2001 From: David Chen <530634352@qq.com> Date: Tue, 3 Nov 2026 10:04:22 +0800 Subject: [PATCH 11/23] Place performance charts before candidate details Agent-assisted-by: Codex (implementation) Signed-off-by: David Chen <530634352@qq.com> --- src/inferrsi/data/dashboard/template.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/inferrsi/data/dashboard/template.html b/src/inferrsi/data/dashboard/template.html index d2323a10b..166f7dfb5 100644 --- a/src/inferrsi/data/dashboard/template.html +++ b/src/inferrsi/data/dashboard/template.html @@ -118,15 +118,15 @@
Performance and evidence Recorded values, including unsuccessful attempts
-

Candidate details

Select a row to inspect its trace
-
RoundCandidateRunStateVerdictGainQualityOutcomeDecisionEvidence
- -

Performance trend · all candidates

__DASHBOARD_CHART__

Results and stages over time · all candidates

__DASHBOARD_TEMPORAL__

+

Candidate details

Select a row to inspect its trace
+
RoundCandidateRunStateVerdictGainQualityOutcomeDecisionEvidence
+ +

Diagnosis · all candidates

Only rules backed by referenced evidence
From 22b01d4ccdd6736faa6465f4c1c06a05b2e8df9f Mon Sep 17 00:00:00 2001 From: David Chen <530634352@qq.com> Date: Tue, 3 Nov 2026 10:33:59 +0800 Subject: [PATCH 12/23] Add offline English and Chinese dashboard views Signed-off-by: David Chen <530634352@qq.com> --- docs/dashboard.md | 28 +++ plugins/inferrsi/skills/inferrsi/SKILL.md | 7 + src/inferrsi/arguments.py | 3 + src/inferrsi/cli.py | 2 +- src/inferrsi/data/dashboard/template.html | 230 +++++++++++++++++++++- src/inferrsi/harness/dashboard.py | 47 ++++- tests/test_campaign_records.py | 5 +- tests/test_dashboard_data.py | 46 +++++ 8 files changed, 356 insertions(+), 12 deletions(-) diff --git a/docs/dashboard.md b/docs/dashboard.md index 7e7d31e49..e89fd650c 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -17,6 +17,34 @@ Omit `--output` to write `dashboard.html` in the input directory. Open the file in a browser. Filters and review export work locally. A generated snapshot does not update as an experiment runs; regenerate it to show new records. +### Language + +The header switches the offline interface, chart labels and review export between +English and Chinese. The language choice is saved in browser storage when +available. Filter and candidate selections stay in place when switching. + +Recorded narratives are caller-provided text; the dashboard does not send them +to a translation service or infer translations. To include reviewed translations, +pass `--translations ` when exporting. The UTF-8 file maps the exact +recorded text to its translation in each target language: + +```json +{ + "schema_version": 1, + "zh": {"Reduce launch overhead": "减少启动开销"}, + "en": {"减少启动开销": "Reduce launch overhead"} +} +``` + +Both language maps are optional. Each key must match the complete original +field, including whitespace; the file is limited to 2 MiB and duplicate keys +are rejected. Untranslated text remains in the original language. Commands, +identifiers and structured event fields are kept as recorded. When a knowledge +context has a supplied translation, the page labels it as a translation and +offers the complete original separately. The overlay is supplied and reviewed +by the caller, not verified by InferRSI; review it for accuracy and sensitive +content before exporting or sharing. It is embedded in the resulting HTML. + The input can be one existing run directory containing `run.lock.json`, or a campaign containing `campaign.json`. Existing runs need no migration. Missing events, validation, trace and other evidence are shown explicitly; no missing diff --git a/plugins/inferrsi/skills/inferrsi/SKILL.md b/plugins/inferrsi/skills/inferrsi/SKILL.md index 4c3172401..380b83ded 100644 --- a/plugins/inferrsi/skills/inferrsi/SKILL.md +++ b/plugins/inferrsi/skills/inferrsi/SKILL.md @@ -77,6 +77,13 @@ available. If the context or reasoning was not captured, state that it is missing instead of inferring it from a verdict. The optional campaign trace format and association workflow are documented in `docs/dashboard.md`. +For a bilingual dashboard, prepare and review a translation overlay for the +recorded narrative, including the knowledge context, search space, decisions, +benchmark and quality descriptions. Supply it to `dashboard --translations` as +documented in `docs/dashboard.md`. Keep the recorded context, commands and +identifiers intact; a translation is not the original evidence. Explicitly leave +unsupported or unavailable translations in their original language. + ## Prepare and run a comparison Consult `plan --help` and `run --help` to build the current runtime's invocation. diff --git a/src/inferrsi/arguments.py b/src/inferrsi/arguments.py index 374fc40b2..4279a0908 100644 --- a/src/inferrsi/arguments.py +++ b/src/inferrsi/arguments.py @@ -118,6 +118,9 @@ def parser() -> argparse.ArgumentParser: dashboard.add_argument( "--output", type=Path, help="HTML output (default: input/dashboard.html)" ) + dashboard.add_argument( + "--translations", type=Path, help="Reviewed bilingual text overlay in JSON" + ) campaign = commands.add_parser("campaign", help="Create local campaign correlation records") campaign_actions = campaign.add_subparsers(dest="campaign_action", required=True) diff --git a/src/inferrsi/cli.py b/src/inferrsi/cli.py index 2681cfa1d..83a9da865 100644 --- a/src/inferrsi/cli.py +++ b/src/inferrsi/cli.py @@ -283,7 +283,7 @@ def main(argv: list[str] | None = None) -> int: if args.command == "dashboard": from .harness.dashboard import write_dashboard - _print({"dashboard": str(write_dashboard(args.input, args.output))}) + _print({"dashboard": str(write_dashboard(args.input, args.output, args.translations))}) return 0 if args.command == "campaign": diff --git a/src/inferrsi/data/dashboard/template.html b/src/inferrsi/data/dashboard/template.html index 166f7dfb5..1e14141b9 100644 --- a/src/inferrsi/data/dashboard/template.html +++ b/src/inferrsi/data/dashboard/template.html @@ -20,6 +20,10 @@ .logo { width:19px; height:19px; border-radius:5px; background:linear-gradient(135deg,#f9d949,#f05a28); } .crumb, .hint { color:var(--muted); } .crumb strong { color:var(--ink); font-weight:600; } .spacer { flex:1; } .badge { border:1px solid var(--line); padding:3px 8px; border-radius:2px; } +.language-switch { display:flex; border:1px solid var(--line); border-radius:2px; overflow:hidden; } +.language-switch button { border:0; background:transparent; color:var(--muted); padding:4px 9px; } +.language-switch button[aria-pressed="true"] { background:#315eac; color:#fff; } +.translation-note { color:var(--muted); margin:8px 12px; font-size:11px; } .sample { color:#241d00; background:#fade2a; font-weight:700; border:0; } .vars { display:flex; gap:10px; flex-wrap:wrap; padding:13px 18px 5px; } .var { display:flex; align-items:stretch; min-height:29px; } @@ -93,7 +97,7 @@ .trace-event pre { white-space:pre-wrap; overflow-wrap:anywhere; color:var(--muted); margin:6px 0; } .button { border:1px solid #477bd2; border-radius:2px; background:#315eac; padding:5px 12px; margin-top:8px; } .footnote { color:var(--muted); font-size:11px; margin:12px 0 0; } -@media (max-width:900px) { .columns, .insight-grid { grid-template-columns:1fr; } .insight.wide { grid-column:auto; } .top { flex-wrap:wrap; padding:10px 18px; } } +@media (max-width:900px) { .columns, .insight-grid { grid-template-columns:1fr; } .insight.wide { grid-column:auto; } .top { flex-wrap:wrap; padding:10px 18px; } .stat-value { font-size:24px; white-space:nowrap; } } @@ -102,6 +106,10 @@ Dashboards › InferRSI › READ ONLY · OFFLINE +
+ + +
campaign
@@ -143,6 +151,7 @@

Review the exported text before sharing. Common private values are redacted; free text can still contain sensitive information.

+

Recorded text without a supplied translation remains in its original language. Commands, identifiers and raw event fields stay unchanged.

A successful quality command covers only that command. This view does not establish statistical significance. Raw logs and environment values are withheld; references and SHA-256 identify recorded artifacts.

@@ -151,6 +160,181 @@ 'use strict'; const data = JSON.parse(document.getElementById('dashboard-data').textContent); const byId = id => document.getElementById(id); + const chinese = { + 'Language':'语言', 'InferRSI · Optimization dashboard':'InferRSI · 优化看板', + 'Dashboards › InferRSI ›':'看板 › InferRSI ›', 'SYNTHETIC DATA':'合成数据', + 'READ ONLY · OFFLINE':'只读 · 离线', 'Dashboard filters':'看板筛选', + 'campaign':'任务', 'run':'运行', 'candidate':'候选方案', 'stage':'阶段', 'metric':'指标', + 'All runs':'全部运行', 'All candidates':'全部候选方案', 'All stages':'全部阶段', + 'Overview · all candidates':'概览 · 全部候选方案', 'Overview statistics':'概览统计', + 'Workflow':'工作流程', 'Technical events only; select a stage to filter candidates and events':'仅显示技术事件；选择阶段可筛选候选方案和事件', + 'Performance and evidence':'性能与证据', 'Recorded values, including unsuccessful attempts':'已记录的数据，包括未成功的尝试', + 'Performance trend · all candidates':'性能趋势 · 全部候选方案', + 'Results and stages over time · all candidates':'结果与阶段的时间分布 · 全部候选方案', + 'Candidate details':'候选方案详情', 'Select a row to inspect its trace':'选择一行查看优化轨迹', + 'Round':'轮次', 'Candidate':'候选方案', 'Run':'运行', 'State':'状态', 'Verdict':'判定', + 'Gain':'提升', 'Quality':'质量', 'Outcome':'结果', 'Decision':'决策', 'Evidence':'证据', + 'No matching recorded candidates.':'没有符合条件的候选方案记录。', + 'Diagnosis · all candidates':'诊断 · 全部候选方案', 'Only rules backed by referenced evidence':'仅显示有引用证据支持的规则', + 'Technical events':'技术事件', 'Search trace':'搜索轨迹', 'Search event, tool, field or reference':'搜索事件、工具、字段或引用', + 'Trace source':'轨迹来源', 'All sources':'全部来源', 'Runner':'运行器', 'Agent':'智能体', + 'Evidence completeness':'证据完整性', 'Missing records are explicit':'明确标出缺失的记录', + 'Human review':'人工审阅', 'Local Markdown export':'本地导出 Markdown', + 'Findings and suggested framework improvement':'发现的问题与框架改进建议', + 'Record your observation and recommendation.':'记录你的观察和建议。', + 'Export feedback · Markdown':'导出反馈 · Markdown', + 'Review the exported text before sharing. Common private values are redacted; free text can still contain sensitive information.':'分享前请检查导出内容。常见隐私值已脱敏，但自由文本仍可能包含敏感信息。', + 'Recorded text without a supplied translation remains in its original language. Commands, identifiers and raw event fields stay unchanged.':'未提供译文的记录保留原语言；命令、标识符和原始事件字段保持不变。', + 'A successful quality command covers only that command. This view does not establish statistical significance. Raw logs and environment values are withheld; references and SHA-256 identify recorded artifacts.':'质量命令成功只代表该命令覆盖的范围。本页面不证明统计显著性。原始日志和环境变量未展示；引用和 SHA-256 用于标识已记录的工件。', + 'Diagnosis':'诊断', 'Knowledge':'知识检索', 'Candidate search':'候选方案搜索', 'Execution':'执行', 'Validation':'验证', 'Review':'审阅', + 'recorded':'已记录', 'stage not recorded':'未记录阶段', 'not applicable':'不适用', 'not recorded':'未记录', + 'Not recorded':'未记录', 'NOT RECORDED':'未记录', 'Ambiguous':'存在歧义', 'ambiguous':'存在歧义', + 'present':'已记录', 'missing':'缺失', 'PRESENT':'已记录', 'MISSING':'缺失', + 'complete':'完整', 'partial':'部分记录', 'redacted':'已脱敏', 'unknown':'未知', + 'completed':'已完成', 'running':'运行中', 'failed':'失败', 'passed':'通过', 'skipped':'已跳过', + 'improved':'提升', 'regressed':'下降', 'noise':'噪声范围内', 'quality failed':'质量未通过', + 'rolled back':'已回滚', 'inconclusive':'结论不明确', 'succeeded':'成功', + 'PASS':'PASS（通过）', 'FAIL':'FAIL（失败）', 'INCONCLUSIVE':'INCONCLUSIVE（结论不明确）', + 'observation':'观察', 'attempt':'尝试', 'decision':'决策', + 'code':'代码', 'configuration':'配置', 'mixed':'混合', 'other':'其他', + 'execution failed':'执行失败', 'quality failed with gain':'性能提升但质量未通过', + 'within noise':'噪声范围内', 'measured regression':'测量结果下降', + 'Best gain':'最佳提升', 'Incompatible records':'记录不可比较', + 'Comparison evidence missing or unsupported':'比较证据缺失或不受支持', 'Quality and source checks required':'需要检查质量和来源', + 'Target gap':'距目标差距', 'Target not recorded':'未记录目标', + 'Candidate / run views':'候选方案 / 运行视图', 'Recorded candidate or run views':'已记录的候选方案或运行视图', + 'Within noise':'噪声范围内', 'Uses recorded noise band':'使用已记录的噪声区间', + 'Failed / regressed':'失败 / 下降', 'Includes failures and rollbacks':'包含失败和回滚', + 'Missing evidence':'缺失证据', 'See completeness panel':'查看证据完整性面板', + 'Rounds not recorded':'未记录轮次', 'No matching candidate.':'没有符合条件的候选方案。', + 'Narrative is caller recorded; verdicts and measurements are separate evidence.':'叙述由调用方记录；判定和测量值属于独立证据。', + 'Change and rationale':'改动与原因', 'Caller recorded':'调用方记录', 'Change type':'改动类型', + 'What was changed':'具体改动', 'Why / hypothesis':'原因 / 假设', + 'Code / configuration locations':'代码 / 配置位置', 'Code / configuration location':'代码 / 配置位置', + 'Optimization trace':'优化轨迹', 'Trace not recorded':'未记录优化轨迹', + 'Candidate-level narrative; no run binding recorded':'候选方案级叙述；未记录绑定的运行', + 'Knowledge context':'知识上下文', 'Caller-reviewed snapshot; not proof of model consumption':'调用方审核的快照；不能证明模型实际使用过', + 'Capture status':'采集状态', 'Caller declared':'调用方声明', 'Search queries':'搜索查询', + 'Selected source':'选定来源', 'Selected sources':'选定来源', 'Context text':'上下文原文', 'Trace evidence':'轨迹证据', + 'Translated context · caller-supplied':'上下文译文 · 调用方提供', 'View original recorded context':'查看记录的上下文原文', + 'Declared search space':'声明的搜索空间', 'Options are not necessarily attempts':'列出的选项不一定已经尝试', + 'No linked attempt recorded':'未记录关联的尝试', 'Options':'选项', + 'Optimization process':'优化过程', 'Recorded order and rationale':'记录的顺序与原因', + 'Action':'操作', 'Why':'原因', 'Recorded result':'记录的结果', 'Technical event':'技术事件', + 'Unavailable, unlinked, or source hash mismatch':'不可用、未关联或来源哈希不匹配', 'Steps':'步骤', + 'Successes and failures':'成功与失败', 'Caller conclusions; see evidence for verdicts':'调用方的总结；判定请查看证据', + 'Success summary':'成功总结', 'Failure summary':'失败总结', 'Limitations':'局限性', + 'Benchmark and accuracy':'性能与精度测试', 'Measurements and redacted commands':'测量结果和脱敏命令', + 'Performance methodology · caller recorded':'性能测试方法 · 调用方记录', 'Workload':'工作负载', + 'Recorded medians':'记录的中位数', 'Recorded samples':'记录的样本', + 'Baseline command · redacted argv':'基线命令 · 已脱敏参数', 'Candidate command · redacted argv':'候选方案命令 · 已脱敏参数', + 'Accuracy methodology / result · caller recorded':'精度测试方法 / 结果 · 调用方记录', + 'Validation command · redacted argv':'验证命令 · 已脱敏参数', 'caller recorded, redacted argv':'调用方记录，已脱敏参数', + 'Recorded validation state':'记录的验证状态', 'only the recorded validation command is covered':'仅覆盖已记录的验证命令', + 'validation command not recorded or unavailable for display':'验证命令未记录或无法展示', + 'Not recorded or unavailable for display':'未记录或无法展示', 'Command evidence':'命令证据', 'Caller detail evidence':'调用方详情证据', + 'No selection':'未选择', 'TRACE LOCATION':'轨迹位置', 'No events match the current filters.':'没有符合当前筛选条件的事件。', + 'unclassified event':'未分类事件', 'Time not recorded':'未记录时间', 'UNLINKED':'未关联', + 'No additional structured fields recorded.':'未记录其他结构化字段。', 'MISSING · tool calls':'缺失 · 工具调用', + 'UNLINKED AGENT EVENTS':'未关联的智能体事件', 'MISMATCHED AGENT EVENTS':'不匹配的智能体事件', + 'No evidence-backed diagnostic rule matched.':'没有匹配到有证据支持的诊断规则。', 'Single run':'单次运行', + 'No corresponding trace event was recorded; inspect the referenced evidence.':'未记录对应的轨迹事件；请检查引用的证据。', + 'Recorded cumulative campaign steps.':'记录的累计优化步骤。', + 'Independent candidate results relative to their paired baselines.':'各候选方案相对于其配对基线的独立结果。', + 'Comparison evidence is missing, unsupported, or incompatible.':'比较证据缺失、不受支持或互不兼容。', + 'No measured candidate results recorded.':'未记录候选方案的测量结果。', + 'Recorded values exceed the supported chart scale.':'记录的值超出图表支持的范围。', + 'The recorded chart specification is invalid or incomplete.':'记录的图表配置无效或不完整。', + 'No timestamped events recorded; a time axis cannot be drawn.':'未记录带时间戳的事件，无法绘制时间轴。', + 'Recorded values exceed the supported time-chart scale.':'记录的值超出时间图表支持的范围。', + 'Recorded timestamps exceed the supported time-chart range.':'记录的时间戳超出时间图表支持的范围。', + 'Terminal events place results; stage dots are events, not durations.':'结果按终止事件定位；阶段圆点表示事件，不表示持续时间。', + 'Unverified or incompatible candidate measurements are not plotted together.':'未验证或不兼容的候选方案测量值不会绘制在一起。', + 'No common noise band recorded.':'未记录共同的噪声区间。', + 'Candidate results and stage events over time':'候选方案结果和阶段事件的时间分布', + 'Results use terminal event timestamps. Stage dots are events, not durations. Crosses in the no-value row have no numeric measurement.':'结果使用终止事件时间戳。阶段圆点表示事件，不表示持续时间；无数值行中的叉号表示没有数值测量结果。', + 'Candidate gain versus paired baseline':'候选方案相对于配对基线的提升', 'paired baseline':'配对基线', 'baseline':'基线', + 'Candidates':'候选方案', 'gain':'提升', 'Target':'目标', 'target':'目标', + 'Gain vs baseline':'相对基线的提升', 'No numeric result':'无数值结果', 'no numeric result':'无数值结果', 'Stage events':'阶段事件', + 'Recorded optimization step':'已记录的优化步骤', 'Throughput vs baseline':'相对基线的吞吐量', + 'Throughput':'吞吐量', 'throughput':'吞吐量', 'Baseline':'基线', 'revision':'修订版本', + 'Recorded optimization':'记录的优化', 'Findings and recommendation':'发现与建议', 'Evidence references':'证据引用', + 'Visible trace':'可见轨迹', 'Stage':'阶段', 'Campaign':'任务', 'InferRSI review':'InferRSI 审阅', + 'file_absent':'文件缺失', 'not_collected':'未采集', 'hash_mismatch':'哈希不匹配', + 'hash_not_recorded':'未记录哈希', 'unreadable':'无法读取', 'unsafe_path':'不安全的路径', + 'invalid_format':'格式无效', 'identity_mismatch':'身份不匹配', 'identity_ambiguous':'身份不明确', + 'too_large':'文件过大', 'not_applicable':'不适用', 'redacted or invalid path':'路径已脱敏或无效' + }; + let language = 'en'; + try { if (localStorage.getItem('inferrsi.dashboard.language') === 'zh') language = 'zh'; } catch (_) { /* file storage may be disabled */ } + const originalText = new WeakMap(), originalAttributes = new WeakMap(); + function translated(text) { + if (typeof text !== 'string') return text; + const supplied = (data.translations || {})[language] || {}; + if (Object.prototype.hasOwnProperty.call(supplied, text)) return supplied[text]; + if (language !== 'zh') return text; + if (Object.prototype.hasOwnProperty.call(chinese, text)) return chinese[text]; + const temporalPrefix='Terminal events place results; stage dots are events, not durations.'; + if(text.startsWith(`${temporalPrefix} `)) { + return chinese[temporalPrefix] + ' ' + text.slice(temporalPrefix.length+1) + .replace(/(\d+) result\(s\) have no recorded terminal timestamp\./g, (_, count)=>`${count} 个结果未记录终止时间戳。`) + .replace('Unverified or incompatible candidate measurements are not plotted together.',chinese['Unverified or incompatible candidate measurements are not plotted together.']) + .replace('No common noise band recorded.',chinese['No common noise band recorded.']); + } + const patterns = [ + [/^(\d+) recorded rounds$/, m => `已记录 ${m[1]} 轮`], + [/^corrupt lines (.+)$/, m => `损坏的行 ${m[1]}`], + [/^Target (.+)$/, m => `目标 ${m[1]}`], + [/^Noise (.+)$/, m => `噪声 ${m[1]}`], + [/^(\d+) present \/ (\d+) missing$/, m => `已记录 ${m[1]} / 缺失 ${m[2]}`], + [/^Inspect (.+)$/, m => `查看 ${m[1]}`], + [/^Caller bound this narrative to run (.+)$/, m => `调用方将此叙述绑定到运行 ${m[1]}`], + [/^View recorded context · (.+)$/, m => `查看记录的上下文 · ${translated(m[1])}`], + [/^(\d+) linked recorded attempt\(s\)$/, m => `关联了 ${m[1]} 次已记录的尝试`], + [/^Search option: (.+)$/, m => `搜索选项：${m[1]}`], + [/^Open (.+)$/, m => `打开 ${m[1]}`], + [/^(\d+) repetitions$/, m => `重复 ${m[1]} 次`], + [/^repetitions not recorded$/, () => '未记录重复次数'], + [/^Baseline ([^·]+)$/, m => `基线 ${m[1]}`], [/^Candidate ([^·]+)$/, m => `候选方案 ${m[1]}`], + [/^Baseline: (.+)$/, m => `基线：${m[1]}`], [/^Candidate: (.+)$/, m => `候选方案：${m[1]}`], + [/^(\d+) of (\d+) events · time order, unknown time last$/, m => `${m[2]} 个事件中显示 ${m[1]} 个 · 按时间排序，时间未知的排在最后`], + [/^Lines (.+)$/, m => `行 ${m[1]}`], [/^CORRUPT (.+) LINES$/, m => `${m[1]} 损坏的行`], + [/^trial (.+)$/, m => `试验 ${m[1]}`], + [/^baseline: (.+)$/, m => `基线：${translated(m[1])}`], + [/^(\d+)\. (.+)$/, m => `${m[1]}. ${translated(m[2])}`], + [/^(\d+) (Diagnosis|Knowledge|Candidate search|Execution|Validation|Review)$/, m => `${m[1]} ${translated(m[2])}`], + [/^revision (.+)$/, m => `修订版本 ${m[1]}`], + [/^(.+?): (Diagnosis|Knowledge|Candidate search|Execution|Validation|Review) · (.+)$/, m => `${translated(m[1])}：${translated(m[2])} · ${m[3]}`], + [/^(.+?): (improved|regressed|failed|quality failed|noise)(.*)$/, m => `${translated(m[1])}：${translated(m[2])}${m[3]}`], + [/^(\d+) result\(s\) have no recorded terminal timestamp\.$/, m => `${m[1]} 个结果未记录终止时间戳。`] + ]; + for (const [pattern, format] of patterns) { const match=text.match(pattern); if(match)return format(match); } + if (text.includes('\n')) return text.split('\n').map(translated).join('\n'); + if (text.includes(' · ')) return text.split(' · ').map(translated).join(' · '); + return text; + } + function localizePage() { + document.documentElement.lang=language==='zh'?'zh-CN':'en'; + document.title=translated('InferRSI · Optimization dashboard'); + byId('language-en').setAttribute('aria-pressed',String(language==='en')); + byId('language-zh').setAttribute('aria-pressed',String(language==='zh')); + const walker=document.createTreeWalker(document.body,NodeFilter.SHOW_TEXT); + while(walker.nextNode()) { + const text=walker.currentNode, parent=text.parentElement; + if(!parent || parent.closest('script,style,pre,code,textarea,[data-no-translate]'))continue; + if(!originalText.has(text))originalText.set(text,text.nodeValue); + const original=originalText.get(text), trimmed=original.trim(); + if(trimmed){const direct=translated(original);text.nodeValue=direct===original?original.replace(trimmed,translated(trimmed)):direct;} + } + for(const element of document.querySelectorAll('[aria-label],[placeholder]')) { + if(!originalAttributes.has(element))originalAttributes.set(element,{}); + const original=originalAttributes.get(element); + for(const attr of ['aria-label','placeholder'])if(element.hasAttribute(attr)) { + if(!(attr in original))original[attr]=element.getAttribute(attr); + element.setAttribute(attr,translated(original[attr])); + } + } + } const stageNames = {diagnosis:'Diagnosis', knowledge:'Knowledge', search:'Candidate search', execution:'Execution', validation:'Validation', review:'Review'}; const stageSelect = byId('stage-filter'), runSelect = byId('run-filter'), candidateSelect = byId('candidate-filter'); let selected = null, jumpNotice = ''; @@ -257,12 +441,19 @@ detailLine(knowledge,'Capture status',recordedState(knowledgeContext)); detailLine(knowledge,'Search queries',knowledgeContext && knowledgeContext.queries.length ? knowledgeContext.queries.join('\n') : 'Not recorded'); if(knowledgeContext && knowledgeContext.sources.length){ - for(const source of knowledgeContext.sources)detailLine(knowledge,'Selected source',`${source.ref} · revision ${source.revision || 'Not recorded'}${source.locator ? ` · ${source.locator}` : ''}`,true); + for(const source of knowledgeContext.sources)detailLine(knowledge,'Selected source',`${source.ref} · ${translated('revision')} ${source.revision || translated('Not recorded')}${source.locator ? ` · ${translated(source.locator)}` : ''}`,true); }else detailLine(knowledge,'Selected sources','Not recorded'); if(knowledgeContext && knowledgeContext.text){ const fold=node('details',undefined,'context-detail');fold.open=true; - fold.appendChild(node('summary',`View recorded context · ${knowledgeContext.state}`)); - fold.appendChild(node('pre',knowledgeContext.text,'context-text'));knowledge.appendChild(fold); + fold.id='knowledge-context'; + const contextText=translated(knowledgeContext.text); + fold.appendChild(node('summary',contextText===knowledgeContext.text ? `View recorded context · ${knowledgeContext.state}` : 'Translated context · caller-supplied')); + fold.appendChild(node('pre',contextText,'context-text'));knowledge.appendChild(fold); + if(contextText!==knowledgeContext.text){ + const original=node('details',undefined,'context-detail');original.id='original-context'; + original.appendChild(node('summary','View original recorded context')); + original.appendChild(node('pre',knowledgeContext.text,'context-text'));knowledge.appendChild(original); + } }else detailLine(knowledge,'Context text','Not recorded'); if(row.artifacts.optimization_trace)detailLine(knowledge,'Trace evidence',refText(row.artifacts.optimization_trace),true); @@ -357,7 +548,7 @@ const {visible,total}=traceEvents(row);byId('trace-count').textContent=`${visible.length} of ${total} events · time order, unknown time last`; if(jumpNotice)item(trace,'TRACE LOCATION',jumpNotice); if(!visible.length)item(trace,'NOT RECORDED','No events match the current filters.'); - for(const event of visible){const name=event.type || event.phase || 'unclassified event';const metadata=[event.time || 'Time not recorded',event.stage,event.side,event.trial===null?null:`trial ${event.trial}`,event.tool].filter(Boolean).join(' · ');const entry=node('details',undefined,`item trace-event${event.unlinked?' missing-item':''}`);entry.id=`trace-${event.source}-${event.line}`;entry.appendChild(node('summary',`${event.unlinked?'UNLINKED · ':''}${event.source}:${event.line} · ${name}`));entry.appendChild(node('small',metadata));entry.appendChild(node('pre',Object.keys(event.details).length ? JSON.stringify(event.details,null,2) : 'No additional structured fields recorded.'));entry.appendChild(node('code',event.refs.map(refText).join(' | ')));trace.appendChild(entry);} + for(const event of visible){const name=event.type || event.phase || translated('unclassified event');const metadata=[event.time || 'Time not recorded',event.stage,event.side,event.trial===null?null:`trial ${event.trial}`,event.tool].filter(Boolean).join(' · ');const entry=node('details',undefined,`item trace-event${event.unlinked?' missing-item':''}`);entry.id=`trace-${event.source}-${event.line}`;const summary=node('summary');if(event.unlinked)summary.appendChild(node('span','UNLINKED · '));const rawName=node('span',`${event.source}:${event.line} · ${name}`);rawName.setAttribute('data-no-translate','');summary.appendChild(rawName);entry.appendChild(summary);entry.appendChild(node('small',metadata));entry.appendChild(node('pre',Object.keys(event.details).length ? JSON.stringify(event.details,null,2) : translated('No additional structured fields recorded.')));entry.appendChild(node('code',event.refs.map(refText).join(' | ')));trace.appendChild(entry);} if(row.artifacts.tool_calls.state==='missing')item(trace,'MISSING · tool calls',row.artifacts.tool_calls.missing_reason).classList.add('missing-item'); if(row.agent_trace_links.unlinked.length)item(trace,'UNLINKED AGENT EVENTS',`Lines ${row.agent_trace_links.unlinked.join(', ')}`).classList.add('missing-item'); if(row.agent_trace_links.mismatched.length)item(trace,'MISMATCHED AGENT EVENTS',`Lines ${row.agent_trace_links.mismatched.join(', ')}`).classList.add('missing-item'); @@ -365,12 +556,33 @@ for(const ref of Object.values(row.artifacts)){const entry=item(evidence,ref.state === 'present' ? `${ref.kind} · PRESENT` : `${ref.kind} · MISSING`,ref.state === 'present' ? ref.path : ref.missing_reason,ref.sha256 ? `sha256 ${ref.sha256}` : '');if(ref.state==='missing')entry.classList.add('missing-item');} } function renderDiagnostics(){const root=byId('diagnostics');clear(root);if(!data.diagnostics.length)item(root,'NOT RECORDED','No evidence-backed diagnostic rule matched.');for(const d of data.diagnostics){const entry=item(root,'',d.candidate_id || 'Single run',d.refs.map(refText).join(' | '));const button=node('button',d.kind.replaceAll('_',' '),'diagnostic-button');button.type='button';const matches=data.candidates.map((row,index)=>({row,index})).filter(({row})=>row.candidate_id===d.candidate_id && row.run_id===d.run_id && row.round===d.round);button.disabled=matches.length!==1;button.addEventListener('click',()=>{const {row,index}=matches[0];const phases=d.kind==='execution_failed'?['crashed','measurement']:d.kind==='quality_failed_with_gain'?['quality_validation']:d.kind==='rolled_back'?[]:['measurement'];let event=null;for(const phase of phases){event=[...row.events].reverse().find(e=>e.phase===phase);if(event)break;}runSelect.value='';candidateSelect.value=String(index);stageSelect.value=event && event.stage || '';byId('trace-search').value='';byId('trace-source').value='';selected=index;jumpNotice=event?'': 'No corresponding trace event was recorded; inspect the referenced evidence.';render();const target=event?byId(`trace-${event.source}-${event.line}`):byId('trace');if(target){if(target.tagName==='DETAILS')target.open=true;target.setAttribute('tabindex','-1');target.focus({preventScroll:true});target.scrollIntoView({block:'center'});}});entry.querySelector('strong').replaceWith(button);}} - function render(){const visible=filtered();renderWorkflow();renderRows(visible);renderCandidate(selected===null?null:data.candidates[selected]);renderDetail();} + function render(){const visible=filtered();renderWorkflow();renderRows(visible);renderCandidate(selected===null?null:data.candidates[selected]);renderDetail();localizePage();} for(const select of [stageSelect,runSelect,candidateSelect])select.addEventListener('change',()=>{jumpNotice='';render();}); - byId('trace-search').addEventListener('input',renderDetail); - byId('trace-source').addEventListener('change',renderDetail); + byId('trace-search').addEventListener('input',()=>{renderDetail();localizePage();}); + byId('trace-source').addEventListener('change',()=>{renderDetail();localizePage();}); + for(const choice of ['en','zh'])byId(`language-${choice}`).addEventListener('click',()=>{ + if(language===choice)return; + const openEvents=[...document.querySelectorAll('#trace details[open]')].map(el=>el.id); + const contextOpen=byId('knowledge-context')?.open, originalOpen=byId('original-context')?.open; + language=choice; + try { localStorage.setItem('inferrsi.dashboard.language',language); } catch (_) { /* file storage may be disabled */ } + renderStats();renderDiagnostics();render(); + for(const id of openEvents){const event=byId(id);if(event)event.open=true;} + if(byId('knowledge-context') && contextOpen!==undefined)byId('knowledge-context').open=contextOpen; + if(byId('original-context') && originalOpen!==undefined)byId('original-context').open=originalOpen; + if(!byId('feedback-preview').hidden)byId('feedback-preview').textContent=buildFeedback(); + }); const redact = text => text.replace(/https?:\/\/\S+/gi,'[redacted-url]').replace(/\b[\w.+-]+@[\w.-]+\.[A-Za-z]{2,}\b/g,'[redacted-email]').replace(/\b(?:\d{1,3}\.){3}\d{1,3}\b/g,'[redacted-address]').replace(/\b[A-Za-z]:[\\/][^\s]+/g,'[redacted-path]').replace(/(^|\s)\/(?:[^\s/]+\/)*[^\s]+/g,'$1[redacted-path]').replace(/\b(api[_-]?key|token|password|secret)\s*[:=]\s*\S+/gi,'$1=[redacted]'); - byId('export').addEventListener('click',()=>{const row=selected===null?null:data.candidates[selected];const visible=row?traceEvents(row).visible:[];const refs=row?[...new Set([...Object.values(row.artifacts),...visible.flatMap(event=>event.refs)].map(refText))].join('\n- '):'Not recorded';const events=visible.map(e=>`${e.unlinked?'UNLINKED ':''}${e.source}:${e.line}`).join(', ');const feedback=redact(`# InferRSI review\n\nCampaign: ${data.campaign_id}\nRound: ${row && row.round!==null?row.round:'Not recorded'}\nCandidate: ${row?identity(row,selected):'Not recorded'}\nStage: ${stageSelect.value || 'all'}\nVisible trace: ${events || 'Not recorded'}\n\n## Recorded optimization\n\n${row?row.optimization.summary:'Not recorded'}\n\n## Findings and recommendation\n\n${byId('feedback').value || 'Not recorded'}\n\n## Evidence references\n\n- ${refs}\n`);const preview=byId('feedback-preview');preview.textContent=feedback;preview.hidden=false;const url=URL.createObjectURL(new Blob([feedback],{type:'text/markdown'}));const link=node('a');link.href=url;link.download='inferrsi-review.md';document.body.appendChild(link);link.click();link.remove();setTimeout(()=>URL.revokeObjectURL(url),1000);}); + function buildFeedback(){ + const row=selected===null?null:data.candidates[selected], visible=row?traceEvents(row).visible:[]; + const refs=row?[...new Set([...Object.values(row.artifacts),...visible.flatMap(event=>event.refs)].map(refText))].join('\n- '):translated('Not recorded'); + const events=visible.map(e=>`${e.unlinked?`${translated('UNLINKED')} `:''}${e.source}:${e.line}`).join(', '); + return redact(`# ${translated('InferRSI review')}\n\n${translated('Campaign')}: ${data.campaign_id}\n${translated('Round')}: ${row && row.round!==null?row.round:translated('Not recorded')}\n${translated('Candidate')}: ${row?identity(row,selected):translated('Not recorded')}\n${translated('Stage')}: ${stageSelect.value?translated(stageNames[stageSelect.value]):translated('All stages')}\n${translated('Visible trace')}: ${events || translated('Not recorded')}\n\n## ${translated('Recorded optimization')}\n\n${row?translated(row.optimization.summary):translated('Not recorded')}\n\n## ${translated('Findings and recommendation')}\n\n${byId('feedback').value || translated('Not recorded')}\n\n## ${translated('Evidence references')}\n\n- ${refs}\n`); + } + byId('export').addEventListener('click',()=>{ + const feedback=buildFeedback(); + const preview=byId('feedback-preview');preview.textContent=feedback;preview.hidden=false;const url=URL.createObjectURL(new Blob([feedback],{type:'text/markdown'}));const link=node('a');link.href=url;link.download='inferrsi-review.md';document.body.appendChild(link);link.click();link.remove();setTimeout(()=>URL.revokeObjectURL(url),1000); + }); renderStats();renderDiagnostics();render(); })(); diff --git a/src/inferrsi/harness/dashboard.py b/src/inferrsi/harness/dashboard.py index 68d772c4a..ecc8a3842 100644 --- a/src/inferrsi/harness/dashboard.py +++ b/src/inferrsi/harness/dashboard.py @@ -139,7 +139,51 @@ def _chart(source: Path, view: dict) -> tuple[str, str]: return svg[svg.find(" Path: +def _translations(path: Path | None) -> dict: + """Load an explicitly supplied, bounded translation overlay.""" + if path is None: + return {"en": {}, "zh": {}} + with Path(path).open("rb") as stream: + raw = stream.read(2 * 1024 * 1024 + 1) + if len(raw) > 2 * 1024 * 1024: + raise ValueError("dashboard translations exceed 2 MiB") + + def unique(pairs: list[tuple[str, object]]) -> dict: + result = {} + for key, value in pairs: + if key in result: + raise ValueError("duplicate dashboard translation key") + result[key] = value + return result + + try: + document = json.loads(raw.decode("utf-8"), object_pairs_hook=unique) + except (UnicodeError, json.JSONDecodeError, RecursionError) as exc: + raise ValueError("invalid dashboard translations") from exc + if (not isinstance(document, dict) or set(document) - {"schema_version", "en", "zh"} + or type(document.get("schema_version")) is not int + or document["schema_version"] != 1): + raise ValueError("invalid dashboard translations") + result = {} + for language in ("en", "zh"): + entries = document.get(language, {}) + if (not isinstance(entries, dict) or any( + not isinstance(key, str) or not key.strip() or not isinstance(value, str) + or not value.strip() for key, value in entries.items() + )): + raise ValueError("invalid dashboard translations") + try: + for key, value in entries.items(): + key.encode("utf-8") + value.encode("utf-8") + except UnicodeError as exc: + raise ValueError("invalid dashboard translations") from exc + result[language] = entries + return result + + +def write_dashboard(source: Path, destination: Path | None = None, + translations: Path | None = None) -> Path: """Write one offline HTML document, excluding raw logs and environment data.""" source = Path(source).resolve() view = load_dashboard(source) @@ -147,6 +191,7 @@ def write_dashboard(source: Path, destination: Path | None = None) -> Path: temporal, temporal_note = temporal_chart(view) view["chart_note"] = chart_note view["temporal_note"] = temporal_note + view["translations"] = _translations(translations) payload = json.dumps(view, ensure_ascii=False, allow_nan=False) # The JSON data block cannot terminate its script element or create markup. payload = (payload.replace("&", "\\u0026").replace("<", "\\u003c") diff --git a/tests/test_campaign_records.py b/tests/test_campaign_records.py index 66cbc69e3..4ae9966b1 100644 --- a/tests/test_campaign_records.py +++ b/tests/test_campaign_records.py @@ -56,8 +56,11 @@ def test_campaign_and_dashboard_cli_round_trip(tmp_path, capsys): (run / "run.lock.json").write_text("{}", encoding="utf-8") assert main(["campaign", "record", "--directory", str(root), "--candidate", "a", "--run-id", "run-a", "--round", "1", "--title", "First"]) == 0 - assert main(["dashboard", "--input", str(root)]) == 0 + translations = tmp_path / "translations.json" + translations.write_text('{"schema_version":1,"zh":{"First":"第一项"}}', encoding="utf-8") + assert main(["dashboard", "--input", str(root), "--translations", str(translations)]) == 0 assert (root / "dashboard.html").is_file() + assert "第一项" in (root / "dashboard.html").read_text(encoding="utf-8") assert "dashboard" in capsys.readouterr().out diff --git a/tests/test_dashboard_data.py b/tests/test_dashboard_data.py index 00c784a70..427cbada2 100644 --- a/tests/test_dashboard_data.py +++ b/tests/test_dashboard_data.py @@ -2,6 +2,7 @@ import hashlib import json +import re from pathlib import Path import pytest @@ -590,6 +591,51 @@ def test_dashboard_can_be_regenerated(tmp_path): assert write_dashboard(tmp_path) == output +def test_reviewed_translations_are_embedded_without_changing_evidence(tmp_path): + write_json(tmp_path / "campaign.json", {"campaign_id": "example"}) + candidate(tmp_path, "a", "run-a", details={"summary": "Reduce launch overhead"}) + translations = tmp_path / "translations.json" + write_json(translations, {"schema_version": 1, "zh": { + "Reduce launch overhead": "减少启动开销", + "literal": "", + }}) + + html = write_dashboard(tmp_path, translations=translations).read_text(encoding="utf-8") + payload = re.search(r'', + html, re.DOTALL) + assert payload is not None + view = json.loads(payload.group(1)) + assert view["candidates"][0]["optimization"]["summary"] == "Reduce launch overhead" + assert view["translations"]["zh"]["Reduce launch overhead"] == "减少启动开销" + assert view["translations"]["zh"]["literal"] == "" + assert "" not in html + + +@pytest.mark.parametrize("raw", [ + '{"schema_version":1,"zh":{"a":"b","a":"c"}}', + '{"schema_version":true,"zh":{}}', + '{"schema_version":1,"zh":{"a":""}}', + '{"schema_version":1,"zh":[],"extra":{}}', + '{"schema_version":1,"zh":{"a":"\\ud800"}}', +]) +def test_invalid_translation_overlay_is_rejected(tmp_path, raw): + write_json(tmp_path / "run.lock.json", {}) + translations = tmp_path / "translations.json" + translations.write_text(raw, encoding="utf-8") + + with pytest.raises(ValueError, match="translation"): + write_dashboard(tmp_path, translations=translations) + assert not (tmp_path / "dashboard.html").exists() + + +def test_oversized_translation_overlay_is_rejected(tmp_path): + write_json(tmp_path / "run.lock.json", {}) + translations = tmp_path / "translations.json" + translations.write_bytes(b" " * (2 * 1024 * 1024 + 1)) + with pytest.raises(ValueError, match="2 MiB"): + write_dashboard(tmp_path, translations=translations) + + def test_time_chart_uses_terminal_events_and_marks_quality_failure_and_missing_values(tmp_path): write_json(tmp_path / "campaign.json", { "campaign_id": "example", "goal": {"target_improvement_pct": 20}, From 6b611821b9d807462e1cddff0a3a2ddc077f7ea8 Mon Sep 17 00:00:00 2001 From: David Chen <530634352@qq.com> Date: Tue, 3 Nov 2026 10:36:03 +0800 Subject: [PATCH 13/23] Keep trace event identifiers unchanged in translated views Signed-off-by: David Chen <530634352@qq.com> --- src/inferrsi/data/dashboard/template.html | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/inferrsi/data/dashboard/template.html b/src/inferrsi/data/dashboard/template.html index 1e14141b9..37da20e18 100644 --- a/src/inferrsi/data/dashboard/template.html +++ b/src/inferrsi/data/dashboard/template.html @@ -548,7 +548,19 @@ const {visible,total}=traceEvents(row);byId('trace-count').textContent=`${visible.length} of ${total} events · time order, unknown time last`; if(jumpNotice)item(trace,'TRACE LOCATION',jumpNotice); if(!visible.length)item(trace,'NOT RECORDED','No events match the current filters.'); - for(const event of visible){const name=event.type || event.phase || translated('unclassified event');const metadata=[event.time || 'Time not recorded',event.stage,event.side,event.trial===null?null:`trial ${event.trial}`,event.tool].filter(Boolean).join(' · ');const entry=node('details',undefined,`item trace-event${event.unlinked?' missing-item':''}`);entry.id=`trace-${event.source}-${event.line}`;const summary=node('summary');if(event.unlinked)summary.appendChild(node('span','UNLINKED · '));const rawName=node('span',`${event.source}:${event.line} · ${name}`);rawName.setAttribute('data-no-translate','');summary.appendChild(rawName);entry.appendChild(summary);entry.appendChild(node('small',metadata));entry.appendChild(node('pre',Object.keys(event.details).length ? JSON.stringify(event.details,null,2) : translated('No additional structured fields recorded.')));entry.appendChild(node('code',event.refs.map(refText).join(' | ')));trace.appendChild(entry);} + for(const event of visible){ + const name=event.type || event.phase || translated('unclassified event'); + const metadata=[event.time || 'Time not recorded',event.stage,event.side,event.trial===null?null:`trial ${event.trial}`,event.tool].filter(Boolean).join(' · '); + const entry=node('details',undefined,`item trace-event${event.unlinked?' missing-item':''}`); + entry.id=`trace-${event.source}-${event.line}`; + const summary=node('summary'); + if(event.unlinked){summary.appendChild(node('span','UNLINKED'));summary.appendChild(node('span',' · '));} + const rawName=node('span',`${event.source}:${event.line} · ${name}`); + rawName.setAttribute('data-no-translate','');summary.appendChild(rawName);entry.appendChild(summary); + entry.appendChild(node('small',metadata)); + entry.appendChild(node('pre',Object.keys(event.details).length ? JSON.stringify(event.details,null,2) : translated('No additional structured fields recorded.'))); + entry.appendChild(node('code',event.refs.map(refText).join(' | ')));trace.appendChild(entry); + } if(row.artifacts.tool_calls.state==='missing')item(trace,'MISSING · tool calls',row.artifacts.tool_calls.missing_reason).classList.add('missing-item'); if(row.agent_trace_links.unlinked.length)item(trace,'UNLINKED AGENT EVENTS',`Lines ${row.agent_trace_links.unlinked.join(', ')}`).classList.add('missing-item'); if(row.agent_trace_links.mismatched.length)item(trace,'MISMATCHED AGENT EVENTS',`Lines ${row.agent_trace_links.mismatched.join(', ')}`).classList.add('missing-item'); From d7a413e859a1b4095cd4ae9cbb521105976a151c Mon Sep 17 00:00:00 2001 From: David Chen <530634352@qq.com> Date: Tue, 3 Nov 2026 15:36:04 +0800 Subject: [PATCH 14/23] Add campaign optimization story with captured knowledge context The per-candidate trace relied on a caller-supplied context file and had no view of the whole campaign. Add a campaign-level trace that shows how the optimization unfolded and why. - campaign capture-context runs collect with a context budget and keeps the full envelope: the request, what search selected, what was left out and why, and every resource body with its SHA-256. The assembled text is re-derived on load, so an edited capture is withheld. - campaign log records a versioned optimization log: goal, retrievals and why, the declared search space with tried/pruned/untried options, each round's hypothesis, rationale and decision, and success and failure summaries. - The dashboard resolves every reference in the log against captures, rounds.jsonl and recorded runs and lists each disagreement under Narrative vs evidence, including knowledge that search selected but the context budget dropped before the agent read it. - Add an Optimization story section with process, knowledge context, search space and success/failure views, with Chinese labels. The narrative stays caller-authored; the dashboard does not verify reasoning. Refs #314 Co-Authored-By: Claude Opus 5.5 Signed-off-by: David Chen <530634352@qq.com> --- docs/dashboard.md | 59 +++- docs/operations.md | 2 + plugins/inferrsi/skills/inferrsi/SKILL.md | 9 + src/inferrsi/arguments.py | 26 ++ src/inferrsi/cli.py | 22 ++ src/inferrsi/data/dashboard/template.html | 225 ++++++++++++- src/inferrsi/harness/dashboard_data.py | 13 + src/inferrsi/harness/knowledge_capture.py | 166 ++++++++++ src/inferrsi/harness/optimization_log.py | 352 +++++++++++++++++++++ src/inferrsi/harness/optimization_story.py | 225 +++++++++++++ tests/test_optimization_story.py | 294 +++++++++++++++++ 11 files changed, 1389 insertions(+), 4 deletions(-) create mode 100644 src/inferrsi/harness/knowledge_capture.py create mode 100644 src/inferrsi/harness/optimization_log.py create mode 100644 src/inferrsi/harness/optimization_story.py create mode 100644 tests/test_optimization_story.py diff --git a/docs/dashboard.md b/docs/dashboard.md index e89fd650c..f9f97142e 100644 --- a/docs/dashboard.md +++ b/docs/dashboard.md @@ -73,7 +73,64 @@ campaign-relative files; the recorder hashes them and does not copy them. The caller assigns IDs and must serialize campaign record updates. Neither command starts an agent or validates the experiment. -### Capture the optimization trace +### Record the optimization story (campaign trace) + +The **Optimization story** section is the campaign-level trace a reviewer reads +to understand the whole optimization: what knowledge was retrieved and why, the +declared search space and how it was narrowed, each round's hypothesis and +reason, the decision, and the success and failure summaries. It has two parts. + +**1. Capture the knowledge context while working.** Run the same retrieval the +agent uses through `campaign capture-context`. It calls `collect` with a context +budget and keeps the complete envelope: the request, everything search selected, +everything left out and why, and every resource body with its SHA-256. The +dashboard re-derives the assembled text from those resources and withholds a +capture whose bytes or hashes no longer match. + +```sh +inferrsi campaign capture-context --directory --id r1-diagnosis \ + --relevance '' --backend --context-bytes 65536 +``` + +Each call writes a new `knowledge/.json` and prints its SHA-256; an +existing id is never overwritten. The capture proves what InferRSI prepared, +not that a model read it. Captured text comes from the selected knowledge +sources; review it before sharing if a store or checkout source was added. + +**2. Record the optimization log.** Write a JSON log (`schema_version: 2`) and +record it with `campaign log`. Every call adds a numbered version +(`optimization-log-vN.json`) and an entry in `optimization-log.jsonl`; earlier +versions remain, and the newest one whose hash matches is shown. + +```sh +inferrsi campaign log --directory --file +``` + +| Field | Content | +| --- | --- | +| `goal` | `statement`, `metric`, optional `baseline`, `target`, `constraints` | +| `knowledge[]` | `context_id` and `sha256` of a capture, `stage`, `purpose` (why it was retrieved), `used_refs`, `takeaways`, optional `rejected_refs[]` with `ref` and `reason` | +| `search_space` | `state`, `missing_reason`, `dimensions[]` with `id`, `name`, optional `rationale`, and `options[]` (`id`, `description`, `status` = `tried`/`pruned`/`untried`, `reason`, optional `source_refs`) | +| `iterations[]` | `round` (increasing), `hypothesis`, `rationale` (why this optimization), `knowledge_refs`, optional `context_ids`, `observations`, `next_step`; `candidates[]` with `candidate_id`, `option_ids` (`.