From a04722ac93606a029def52090d237ca0a7c9c59c Mon Sep 17 00:00:00 2001 From: Mani Sarkar Date: Tue, 25 Aug 2026 01:34:07 +0100 Subject: [PATCH 01/12] =?UTF-8?q?feat(slice-50):=20add=20Tessl=20Eval=20ro?= =?UTF-8?q?w=20with=20Scenario=E2=86=92Eval=20auto-chain?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Emit Eval as blocked until Scenario Generation completes and evals/ is populated, then run eval with resume/stale handling and project preflight. --- sandbox/scan_app.py | 21 +- sandbox/scanners.py | 548 +++++++++++++- sandbox/tests/test_scan_app.py | 21 + sandbox/tests/test_scanners_status.py | 871 ++++++++++++++++++++++- sandbox/tests/test_ship_path_coverage.py | 234 +++++- 5 files changed, 1626 insertions(+), 69 deletions(-) diff --git a/sandbox/scan_app.py b/sandbox/scan_app.py index 3d3fd14..c35c0b7 100644 --- a/sandbox/scan_app.py +++ b/sandbox/scan_app.py @@ -277,10 +277,11 @@ def _on_scanner_done(findings, scanner_rows, quality_score=None): ) def _on_scanner_progress(row: dict) -> None: - """Persist Tessl Scenario Generation checkpoints before Modal timeout.""" + """Persist Tessl mid-group rows (scenario checkpoint, Eval blocked→running).""" _persist_scanner_row(row, completed=False) tessl_scenario_resume = None + tessl_prior_eval = None try: resume_resp = ( supabase.table("scan_run_scanners") @@ -295,6 +296,23 @@ def _on_scanner_progress(row: dict) -> None: except Exception as exc: print(f"[scan] warning: could not load Tessl scenario resume_checkpoint: {exc}") + try: + eval_resp = ( + supabase.table("scan_run_scanners") + .select( + "status,tessl_run_id,tessl_run_id_at,completed_at," + "upstream_run_ids,detail,checks_run" + ) + .eq("scan_run_id", scan_run_id) + .eq("scanner_source", "Tessl: Eval") + .limit(1) + .execute() + ) + if eval_resp.data: + tessl_prior_eval = eval_resp.data[0] + except Exception as exc: + print(f"[scan] warning: could not load Tessl Eval prior row: {exc}") + try: results = run_all_scanners( workdir=workdir, @@ -304,6 +322,7 @@ def _on_scanner_progress(row: dict) -> None: on_scanner_start=_on_scanner_start, on_scanner_progress=_on_scanner_progress, tessl_scenario_resume=tessl_scenario_resume, + tessl_prior_eval=tessl_prior_eval, ) except Exception: _mark_failed() diff --git a/sandbox/scanners.py b/sandbox/scanners.py index 532ddd4..ac3802f 100644 --- a/sandbox/scanners.py +++ b/sandbox/scanners.py @@ -37,11 +37,11 @@ def _which(binary): return shutil.which(binary) is not None -def _run(cmd, timeout=SCAN_TIMEOUT): +def _run(cmd, timeout=SCAN_TIMEOUT, cwd=None): """Never raises on nonzero exit or timeout — callers map that to a scan_run_scanners status (unreachable) rather than a crash.""" try: - proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout, cwd=cwd) return proc.returncode, proc.stdout, proc.stderr except subprocess.TimeoutExpired as exc: return None, exc.stdout or "", f"timeout after {timeout}s" @@ -59,7 +59,10 @@ def _skipped(source, reason="skipped_missing_credential", detail=None, console_o def _snyk_collect_errors(path_result): - """Gather path-level and per-server errors from a Snyk path envelope.""" + """Gather path-level and per-component errors from a Snyk path envelope. + + Supports v0.5 (`servers`) and v0.6 (`server_risks` / `skill_risks`). + """ errors = [] path_err = path_result.get("error") if path_err: @@ -67,9 +70,79 @@ def _snyk_collect_errors(path_result): for server in path_result.get("servers") or []: if isinstance(server, dict) and server.get("error"): errors.append(server["error"]) + for component in (path_result.get("server_risks") or []) + ( + path_result.get("skill_risks") or [] + ): + if isinstance(component, dict) and component.get("error"): + errors.append(component["error"]) return errors +def _snyk_iter_path_results(root): + """Yield path-result dicts from v0.5 path-keyed maps or v0.6 envelopes.""" + responses = root.get("scan_path_responses") + if isinstance(responses, list): + for item in responses: + if isinstance(item, dict): + yield item + return + for _abs_path, path_result in root.items(): + if isinstance(path_result, dict): + yield path_result + + +def _snyk_severity_from_score(score): + """Map Agent Scan v0.6 risk score (0–1000) to Tripwire red/amber.""" + try: + numeric = int(score) + except (TypeError, ValueError): + return "amber" + return "red" if numeric >= 600 else "amber" + + +def _snyk_findings_from_path(path_result, source): + """Extract findings + check count from one path result (v0.5 issues or v0.6 risks).""" + findings = [] + checks = 0 + for issue in path_result.get("issues") or []: + if not isinstance(issue, dict): + continue + checks += 1 + code_ = issue.get("code", "") + severity = "red" if code_.startswith("E") else ("amber" if code_.startswith("W") else None) + if severity is None: + continue + findings.append( + { + "severity": severity, + "category": _SNYK_CODE_CATEGORY.get(code_, "unknown"), + "message": issue.get("message"), + "scanner_source": source, + } + ) + for component in (path_result.get("server_risks") or []) + ( + path_result.get("skill_risks") or [] + ): + if not isinstance(component, dict): + continue + risk_indexes = component.get("risk_indexes") or {} + if not isinstance(risk_indexes, dict): + continue + for risk_name, risk in risk_indexes.items(): + if not isinstance(risk, dict): + continue + checks += 1 + findings.append( + { + "severity": _snyk_severity_from_score(risk.get("score")), + "category": str(risk_name), + "message": risk.get("evidence") or str(risk_name), + "scanner_source": source, + } + ) + return findings, checks + + def _snyk_error_is_auth(err): """True when Snyk rejected credentials (401 / Unauthorized / SNYK_TOKEN hint).""" if isinstance(err, dict): @@ -458,6 +531,8 @@ def run_cisco_mcp_scanner(workdir, target): # ---- Snyk Agent Scan --------------------------------------------------------- # docs/research/adapters/scanner-output-adapters.md §2 # Prefer image-preinstalled `snyk-agent-scan` (uv tool install); fall back to uvx. +# JSON: Agent Scan v0.6+ uses scan_path_responses / risk_indexes; v0.5 path-keyed +# issues[] remains supported (see docs/research/adapters/scanner-output-adapters.md §2). def _snyk_cmd(workdir, item_type): @@ -492,29 +567,14 @@ def run_snyk(workdir, item_type="mcp_server"): findings, checks = [], 0 collected_errors = [] paths_seen = 0 - for _abs_path, path_result in root.items(): - if not isinstance(path_result, dict): - continue + for path_result in _snyk_iter_path_results(root): paths_seen += 1 path_errs = _snyk_collect_errors(path_result) if path_errs: collected_errors.extend(path_errs) - for issue in path_result.get("issues", []) or []: - checks += 1 - code_ = issue.get("code", "") - severity = ( - "red" if code_.startswith("E") else ("amber" if code_.startswith("W") else None) - ) - if severity is None: - continue - findings.append( - { - "severity": severity, - "category": _SNYK_CODE_CATEGORY.get(code_, "unknown"), - "message": issue.get("message"), - "scanner_source": source, - } - ) + path_findings, path_checks = _snyk_findings_from_path(path_result, source) + findings.extend(path_findings) + checks += path_checks if collected_errors: detail = "; ".join(_snyk_error_message(e) for e in collected_errors) @@ -970,6 +1030,407 @@ def _run_tessl_scenario_gen( return _scenario_download_and_finish(workdir, ctx, row, gen_id, consoles, on_progress) +_TESSL_EVAL_SOURCE = "Tessl: Eval" +_TESSL_EVAL_RUNS = 3 +_TESSL_EVAL_POLL_SLEEP_S = 2 +_TESSL_EVAL_POLL_MAX = 90 + + +def _new_blocked_eval_row() -> dict: + return { + "scanner_source": _TESSL_EVAL_SOURCE, + "status": "blocked", + "checks_run": 0, + "detail": "waiting for Scenario Generation to complete and populate evals/", + } + + +def _has_tessl_project_link(workdir: str) -> bool: + return os.path.isfile(os.path.join(workdir, "tessl.json")) + + +def _tessl_project_create_argv(workspace: str, project_name: str) -> list[str]: + return [ + "npx", + "--yes", + "tessl@latest", + "project", + "create", + "--workspace", + workspace, + project_name, + ] + + +def _tessl_project_repair_argv() -> list[str]: + return ["npx", "--yes", "tessl@latest", "project", "repair", "--yes"] + + +def _tessl_eval_run_argv(workdir: str, *, as_json: bool = False) -> list[str]: + argv = [ + "npx", + "--yes", + "tessl@latest", + "eval", + "run", + workdir, + "--runs", + str(_TESSL_EVAL_RUNS), + "-y", + ] + if as_json: + argv.append("--json") + return argv + + +def _tessl_eval_view_argv(eval_id: str) -> list[str]: + return ["npx", "--yes", "tessl@latest", "eval", "view", eval_id, "--json"] + + +def _parse_eval_status(parsed) -> str | None: + if not isinstance(parsed, dict): + return None + for key in ("status", "state"): + value = parsed.get(key) + if isinstance(value, str) and value.strip(): + return value.strip().lower() + nested = parsed.get("eval") or parsed.get("evaluation") + if isinstance(nested, dict): + return _parse_eval_status(nested) + return None + + +def _parse_eval_id(parsed) -> str | None: + if not isinstance(parsed, dict): + return None + for key in ("id", "runId", "run_id", "evalId", "eval_id"): + value = parsed.get(key) + if isinstance(value, str) and value.strip(): + return value.strip() + for list_key in ("evals", "runs", "evalRuns", "eval_runs"): + items = parsed.get(list_key) + if isinstance(items, list) and items: + first = _parse_eval_id(items[0]) if isinstance(items[0], dict) else None + if first: + return first + nested = parsed.get("eval") or parsed.get("evaluation") + if isinstance(nested, dict): + return _parse_eval_id(nested) + return _parse_tessl_run_id(parsed) + + +def _parse_eval_checks_run(parsed) -> int | None: + if not isinstance(parsed, dict): + return None + for key in ("scenarioCount", "scenario_count", "checks_run", "count"): + value = parsed.get(key) + if isinstance(value, int) and value >= 0: + return value + scenarios = parsed.get("scenarios") + if isinstance(scenarios, list): + return len(scenarios) + results = parsed.get("results") + if isinstance(results, list): + return len(results) + nested = parsed.get("eval") or parsed.get("evaluation") + if isinstance(nested, dict): + return _parse_eval_checks_run(nested) + return None + + +def _parse_eval_score_field(parsed, *keys: str) -> float | None: + if not isinstance(parsed, dict): + return None + for key in keys: + value = parsed.get(key) + if isinstance(value, int | float): + return float(value) + nested = parsed.get("eval") or parsed.get("evaluation") or parsed.get("scores") + if isinstance(nested, dict): + return _parse_eval_score_field(nested, *keys) + return None + + +def _format_eval_detail(parsed, fallback: str = "eval completed") -> str: + baseline = _parse_eval_score_field( + parsed, "baselineAvg", "baseline_avg", "baseline", "withoutContextAvg" + ) + with_ctx = _parse_eval_score_field( + parsed, "withContextAvg", "with_context_avg", "withContext", "contextAvg" + ) + delta = _parse_eval_score_field(parsed, "delta", "deltaAvg", "delta_avg") + parts: list[str] = [] + if baseline is not None: + parts.append(f"baseline avg={baseline:g}") + if with_ctx is not None: + parts.append(f"with-context avg={with_ctx:g}") + if delta is not None: + parts.append(f"delta={delta:g}") + runs = parsed.get("runs") if isinstance(parsed, dict) else None + if isinstance(runs, int): + parts.append(f"runs={runs}") + elif isinstance(parsed, dict) and isinstance(parsed.get("runCount"), int): + parts.append(f"runs={parsed['runCount']}") + if not parts: + return fallback + return "; ".join(parts) + + +def _ensure_tessl_project(workdir: str, workspace: str) -> tuple[bool, str]: + """Ensure tessl.json project link exists. Returns (ok, detail_on_failure).""" + if _has_tessl_project_link(workdir): + code, out, err = _run(_tessl_project_repair_argv(), cwd=workdir) + if code not in (0, None) and not _has_tessl_project_link(workdir): + return False, (err or out or "tessl project repair failed").strip() + return True, "" + + project_name = os.path.basename(os.path.abspath(workdir)) or "tripwire-scan" + code, out, err = _run(_tessl_project_create_argv(workspace, project_name), cwd=workdir) + if code == 0 and _has_tessl_project_link(workdir): + return True, "" + if code is None: + return False, (err or out or "tessl project create timed out").strip() + detail = (err or out or "tessl project create failed").strip() + return False, detail or "tessl.json missing — project create/repair required before eval" + + +def _eval_should_mark_stale(prior_eval: dict, scenario_row: dict) -> bool: + # Stale when upstream gen id changed, or gen stamp is newer than eval completion. + if prior_eval.get("status") != "completed": + return False + if scenario_row.get("status") != "completed": + return False + upstream = prior_eval.get("upstream_run_ids") or {} + old_gen = upstream.get("scenario_gen") if isinstance(upstream, dict) else None + new_gen = scenario_row.get("tessl_run_id") + if old_gen and new_gen and old_gen != new_gen: + return True + scenario_at = scenario_row.get("tessl_run_id_at") + eval_done = prior_eval.get("completed_at") + if scenario_at and eval_done and str(scenario_at) > str(eval_done): + return True + return False + + +def _poll_eval_until_terminal( + eval_id: str, + *, + max_attempts: int | None = None, + sleep_s: float | None = None, +) -> tuple[str | None, dict | None, str]: + attempts = _TESSL_EVAL_POLL_MAX if max_attempts is None else max_attempts + pause = _TESSL_EVAL_POLL_SLEEP_S if sleep_s is None else sleep_s + last_console = "" + last_parsed = None + for _ in range(attempts): + code, out, err = _run(_tessl_eval_view_argv(eval_id)) + last_console = _build_console(out, err) or last_console + parsed = _safe_json(out) + last_parsed = parsed if isinstance(parsed, dict) else last_parsed + status = _parse_eval_status(parsed) + if status in {"completed", "failed"}: + return status, last_parsed, last_console + if code not in (0, None) and status is None: + return "failed", last_parsed, last_console + if pause > 0: + time.sleep(pause) + return _parse_eval_status(last_parsed), last_parsed, last_console + + +def _finish_eval_row( + row: dict, + *, + status: str, + eval_id: str | None, + parsed: dict | None, + consoles: list[str], + detail: str | None = None, + on_progress=None, +) -> dict: + row["status"] = status + if detail: + row["detail"] = detail[:4000] + elif parsed is not None: + row["detail"] = _format_eval_detail(parsed)[:4000] + checks = _parse_eval_checks_run(parsed) if parsed else None + if checks is not None: + row["checks_run"] = checks + if eval_id: + _stamp_tessl_run_id(row, eval_id) + if consoles: + row["console_output"] = "\n".join(consoles)[:MAX_CONSOLE_CHARS] + _emit_tessl_row_progress(on_progress, row) + return row + + +def _run_tessl_eval( + workdir: str, + ctx: dict[str, str | None], + row: dict, + *, + resume_eval_id: str | None = None, + on_progress=None, +) -> dict: + """Auto-chain Eval: project preflight → eval run → stamp tessl_run_id.""" + _attach_upstream_run_ids(row, ctx, "review_quality", "scenario_gen") + row["status"] = "queued" + _emit_tessl_row_progress(on_progress, row) + + workspace = (os.environ.get("TESSL_WORKSPACE") or "").strip() + if not os.environ.get("TESSL_TOKEN") or not workspace: + row["status"] = "needs_setup" + row["detail"] = "TESSL_TOKEN and TESSL_WORKSPACE required for eval" + _emit_tessl_row_progress(on_progress, row) + return row + + ok, project_detail = _ensure_tessl_project(workdir, workspace) + if not ok: + row["status"] = "needs_setup" + row["detail"] = project_detail[:4000] + _emit_tessl_row_progress(on_progress, row) + return row + + row["status"] = "running" + _emit_tessl_row_progress(on_progress, row) + consoles: list[str] = [] + eval_id = resume_eval_id + + if not eval_id: + # --json returns IDs immediately (Modal-friendly); then poll via eval view. + code, out, err = _run(_tessl_eval_run_argv(workdir, as_json=True)) + console = _build_console(out, err) + if console: + consoles.append(console) + parsed_run = _safe_json(out) + eval_id = _parse_eval_id(parsed_run) if isinstance(parsed_run, dict) else None + if code is None: + return _finish_eval_row( + row, + status="interrupted" if eval_id else "timed_out", + eval_id=eval_id, + parsed=parsed_run if isinstance(parsed_run, dict) else None, + consoles=consoles, + detail=(err or out or "eval run timed out").strip(), + on_progress=on_progress, + ) + if code != 0 and not eval_id: + return _finish_eval_row( + row, + status="failed", + eval_id=None, + parsed=None, + consoles=consoles, + detail=(err or out or "eval run exited non-zero").strip(), + on_progress=on_progress, + ) + if not eval_id: + return _finish_eval_row( + row, + status="completed", + eval_id=None, + parsed=parsed_run if isinstance(parsed_run, dict) else None, + consoles=consoles, + detail=_format_eval_detail( + parsed_run if isinstance(parsed_run, dict) else {}, + fallback="eval completed (no run id captured)", + ), + on_progress=on_progress, + ) + _stamp_tessl_run_id(row, eval_id) + _emit_tessl_row_progress(on_progress, row) + + status, view_parsed, view_console = _poll_eval_until_terminal(eval_id) + if view_console: + consoles.append(view_console) + if status == "completed": + return _finish_eval_row( + row, + status="completed", + eval_id=eval_id, + parsed=view_parsed, + consoles=consoles, + on_progress=on_progress, + ) + if status == "failed": + return _finish_eval_row( + row, + status="failed", + eval_id=eval_id, + parsed=view_parsed, + consoles=consoles, + detail=_format_eval_detail(view_parsed or {}, fallback="eval failed"), + on_progress=on_progress, + ) + return _finish_eval_row( + row, + status="interrupted", + eval_id=eval_id, + parsed=view_parsed, + consoles=consoles, + detail=f"eval status={status or 'unknown'} — resume via eval view", + on_progress=on_progress, + ) + + +def _resolve_tessl_eval_row( + workdir: str, + ctx: dict[str, str | None], + eval_row: dict, + scenario_row: dict, + *, + prior_eval: dict | None = None, + on_progress=None, +) -> dict: + """Apply stale / resume / auto-chain rules after Scenario Generation.""" + prior = dict(prior_eval) if isinstance(prior_eval, dict) else None + + if prior and prior.get("status") == "completed": + if _eval_should_mark_stale(prior, scenario_row): + stale = { + **prior, + "scanner_source": _TESSL_EVAL_SOURCE, + "status": "stale", + "detail": ( + prior.get("detail") or "Scenario Generation re-run — eval result is stale" + ), + } + _emit_tessl_row_progress(on_progress, stale) + return stale + kept = {**prior, "scanner_source": _TESSL_EVAL_SOURCE} + _emit_tessl_row_progress(on_progress, kept) + return kept + + resume_id = None + if prior and isinstance(prior.get("tessl_run_id"), str): + if prior.get("status") in {"interrupted", "running", "queued", "timed_out"}: + resume_id = prior["tessl_run_id"] + eval_row = { + **eval_row, + **{ + k: prior[k] + for k in ( + "upstream_run_ids", + "tessl_run_id", + "tessl_run_id_at", + ) + if k in prior + }, + } + + if resume_id: + return _run_tessl_eval( + workdir, ctx, eval_row, resume_eval_id=resume_id, on_progress=on_progress + ) + + scenario_ok = scenario_row.get("status") == "completed" + has_scenarios = _count_evals_scenarios(workdir) > 0 + if scenario_ok and has_scenarios and eval_row.get("status") in {"blocked", "not_started"}: + return _run_tessl_eval(workdir, ctx, eval_row, on_progress=on_progress) + + _emit_tessl_row_progress(on_progress, eval_row) + return eval_row + + def _finish_tessl_review( judge_type: str, source: str, workspace: str, code, out: str, err: str ) -> tuple[float | None, dict]: @@ -1040,22 +1501,26 @@ def run_tessl( id_context: dict[str, str | None] | None = None, *, resume_checkpoint: dict | None = None, + prior_eval: dict | None = None, on_row_progress=None, ): - """Run Tessl Lint, Review Quality, then Scenario Generation. + """Run Tessl Lint, Review Quality, Scenario Generation, then Eval auto-chain. Lint is synchronous and never requires TESSL_TOKEN; it always runs when npx is available. Review Quality requires TESSL_TOKEN and TESSL_WORKSPACE; its row transitions to needs_setup when either is absent. Scenario Generation - requires TESSL_TOKEN and ``.tessl-plugin/plugin.json``. + requires TESSL_TOKEN and ``.tessl-plugin/plugin.json``. Eval starts ``blocked`` + and auto-chains after Scenario Generation completes with scenarios in + ``evals/`` (first run only; re-runs mark prior completed Eval as ``stale``). - Returns (quality_score, [lint_row, review_row, scenario_row]). quality_score - is None when Review Quality did not complete successfully. + Returns (quality_score, [lint_row, review_row, scenario_row, eval_row]). + quality_score is None when Review Quality did not complete successfully. id_context is the in-process Tessl ID bag for this invocation (GWT-47.5). Tests inject it to observe carry-forward; production seeds a fresh dict. resume_checkpoint resumes Scenario Generation after Modal detach/timeout. - on_row_progress receives partial Scenario Generation rows for mid-scan persist. + prior_eval rehydrates Eval for stale detection or eval-view resume. + on_row_progress receives partial Tessl rows for mid-scan persist. """ rows: list[dict] = [] ctx = id_context if id_context is not None else _new_tessl_id_context() @@ -1098,6 +1563,10 @@ def run_tessl( rows.append(review_row) _update_tessl_id_context(ctx, "review_quality", review_row.get("tessl_run_id")) + # --- Tessl: Eval (blocked until Scenario Generation completes) --- + eval_row = _new_blocked_eval_row() + _emit_tessl_row_progress(on_row_progress, eval_row) + # --- Tessl: Scenario Generation (token + plugin manifest) --- scenario_row = _run_tessl_scenario_gen( workdir, @@ -1106,6 +1575,16 @@ def run_tessl( on_progress=on_row_progress, ) rows.append(scenario_row) + + eval_row = _resolve_tessl_eval_row( + workdir, + ctx, + eval_row, + scenario_row, + prior_eval=prior_eval, + on_progress=on_row_progress, + ) + rows.append(eval_row) return score, rows @@ -1645,7 +2124,12 @@ def _collapse_severity(raw): MCP_SCANNER_SOURCES = list(_MCP_ANALYZER_LABEL.values()) -TESSL_SOURCES = ["Tessl: Lint", "Tessl: Review (Quality)", "Tessl: Scenario Generation"] +TESSL_SOURCES = [ + "Tessl: Lint", + "Tessl: Review (Quality)", + "Tessl: Scenario Generation", + "Tessl: Eval", +] SNYK_SOURCES = ["Snyk"] DEPSHIELD_SOURCES = ["DepShield"] OSSPREY_SOURCES = ["Ossprey"] @@ -1672,12 +2156,14 @@ def _run_tessl_group( target, *, resume_checkpoint: dict | None = None, + prior_eval: dict | None = None, on_row_progress=None, ): """Tessl group — quality axis only: (quality_score, rows), never findings.""" quality_score, rows = run_tessl( workdir, resume_checkpoint=resume_checkpoint, + prior_eval=prior_eval, on_row_progress=on_row_progress, ) return [], rows, quality_score @@ -1742,6 +2228,7 @@ def run_all_scanners( on_scanner_start=None, on_scanner_progress=None, tessl_scenario_resume=None, + tessl_prior_eval=None, ): """Run every applicable SCANNER_GROUPS entry, optionally relaying results. @@ -1756,8 +2243,10 @@ def run_all_scanners( dashboard shows progress scanner-by-scanner. When *on_scanner_progress* is provided it receives mid-group Tessl row - updates (e.g. Scenario Generation ``resume_checkpoint``) for partial persist. + updates (e.g. Scenario Generation ``resume_checkpoint``, Eval + ``blocked``→``queued``→``running``) for partial persist. *tessl_scenario_resume* rehydrates Scenario Generation after Modal timeout. + *tessl_prior_eval* rehydrates Eval for stale detection or eval-view resume. """ findings, scanner_rows = [], [] quality_score = None @@ -1773,6 +2262,7 @@ def run_all_scanners( item_type, target, resume_checkpoint=tessl_scenario_resume, + prior_eval=tessl_prior_eval, on_row_progress=on_scanner_progress, ) else: diff --git a/sandbox/tests/test_scan_app.py b/sandbox/tests/test_scan_app.py index 872f965..56d0990 100644 --- a/sandbox/tests/test_scan_app.py +++ b/sandbox/tests/test_scan_app.py @@ -260,3 +260,24 @@ def test_given_supabase_error_when_safe_rpc_then_raises_runtime_error() -> None: ### When / Then with pytest.raises(RuntimeError, match="rollup"): scan_app._safe_rpc(mock_sb, "tripwire_rollup_item", {"p_item_id": "item-1"}, "rollup") + + +@pytest.mark.parametrize( + "target", + [ + "http://github.com/org/repo.git", + "repo.git", + "path/to/repo.git/", + ], +) +def test_given_git_like_target_when_classified_then_is_git_url(target: str) -> None: + """ + Scenario: http://…git and bare .git suffixes count as git URLs. + Slice: 50 — scan_app git URL detection edge paths + + Given an http .git URL or a path ending in .git, + When _is_git_url classifies it, + Then it returns True. + """ + ### Given / When / Then + assert scan_app._is_git_url(target) is True diff --git a/sandbox/tests/test_scanners_status.py b/sandbox/tests/test_scanners_status.py index 686f764..10f0a7e 100644 --- a/sandbox/tests/test_scanners_status.py +++ b/sandbox/tests/test_scanners_status.py @@ -8,7 +8,8 @@ _build_console / _truncate_console; on_scanner_done callback relay; _completed console_output passthrough; Tessl ID context seed after Review Quality (GWT-47.5); - Tessl Scenario Generation + resume_checkpoint (GWT-49.*) + Tessl Scenario Generation + resume_checkpoint (GWT-49.*); + Tessl Eval auto-chain + stale + resume (GWT-50.*) """ from __future__ import annotations @@ -461,14 +462,16 @@ def test_run_tessl_logs_diagnostic_when_score_is_none(capsys) -> None: ### Then assert score is None - assert len(rows) == 3 - lint_row, review_row, scenario_row = rows + assert len(rows) == 4 + lint_row, review_row, scenario_row, eval_row = rows assert lint_row["scanner_source"] == "Tessl: Lint" assert lint_row["status"] == "completed" assert review_row["scanner_source"] == "Tessl: Review (Quality)" assert review_row["status"] == "unreachable" assert scenario_row["scanner_source"] == "Tessl: Scenario Generation" assert scenario_row["status"] == "failed" + assert eval_row["scanner_source"] == "Tessl: Eval" + assert eval_row["status"] == "blocked" captured = capsys.readouterr() assert "[tessl]" in captured.out assert "quality_score extraction failed" in captured.out @@ -501,8 +504,8 @@ def test_run_tessl_without_token_emits_lint_completed_and_review_needs_setup() - ### Then assert score is None - assert len(rows) == 3 - lint_row, review_row, scenario_row = rows + assert len(rows) == 4 + lint_row, review_row, scenario_row, eval_row = rows assert lint_row["scanner_source"] == "Tessl: Lint" assert lint_row["status"] == "completed" assert lint_row["checks_run"] == 12 @@ -512,6 +515,8 @@ def test_run_tessl_without_token_emits_lint_completed_and_review_needs_setup() - assert review_row["status"] == "needs_setup" assert scenario_row["scanner_source"] == "Tessl: Scenario Generation" assert scenario_row["status"] == "needs_setup" + assert eval_row["scanner_source"] == "Tessl: Eval" + assert eval_row["status"] == "blocked" def test_run_tessl_with_token_emits_lint_and_review_rows() -> None: @@ -547,8 +552,8 @@ def test_run_tessl_with_token_emits_lint_and_review_rows() -> None: ### Then assert score == 75 - assert len(rows) == 3 - lint_row, review_row, scenario_row = rows + assert len(rows) == 4 + lint_row, review_row, scenario_row, eval_row = rows assert lint_row["scanner_source"] == "Tessl: Lint" assert lint_row["status"] == "completed" assert lint_row.get("tessl_run_id") is None @@ -558,8 +563,10 @@ def test_run_tessl_with_token_emits_lint_and_review_rows() -> None: assert review_row["tessl_run_id_at"] assert scenario_row["scanner_source"] == "Tessl: Scenario Generation" assert scenario_row["status"] == "failed" + assert eval_row["scanner_source"] == "Tessl: Eval" + assert eval_row["status"] == "blocked" sources = {row["scanner_source"] for row in rows} - assert "Tessl: Eval" not in sources + assert "Tessl: Eval" in sources assert "Tessl: Review (Security)" not in sources @@ -583,13 +590,15 @@ def test_run_tessl_lint_failure_emits_failed_row() -> None: ### Then assert score is None - lint_row, review_row, scenario_row = rows + lint_row, review_row, scenario_row, eval_row = rows assert lint_row["scanner_source"] == "Tessl: Lint" assert lint_row["status"] == "failed" assert review_row["scanner_source"] == "Tessl: Review (Quality)" assert review_row["status"] == "needs_setup" assert scenario_row["scanner_source"] == "Tessl: Scenario Generation" assert scenario_row["status"] == "needs_setup" + assert eval_row["scanner_source"] == "Tessl: Eval" + assert eval_row["status"] == "blocked" def test_run_tessl_no_npx_emits_lint_unreachable() -> None: @@ -611,14 +620,16 @@ def test_run_tessl_no_npx_emits_lint_unreachable() -> None: ### Then assert score is None - assert len(rows) == 3 - lint_row, review_row, scenario_row = rows + assert len(rows) == 4 + lint_row, review_row, scenario_row, eval_row = rows assert lint_row["scanner_source"] == "Tessl: Lint" assert lint_row["status"] == "unreachable" assert review_row["scanner_source"] == "Tessl: Review (Quality)" assert review_row["status"] == "needs_setup" assert scenario_row["scanner_source"] == "Tessl: Scenario Generation" assert scenario_row["status"] == "needs_setup" + assert eval_row["scanner_source"] == "Tessl: Eval" + assert eval_row["status"] == "blocked" def test_parse_tessl_lint_detail_extracts_count_from_text() -> None: @@ -694,7 +705,7 @@ def test_run_tessl_review_quality_invokes_review_run_quality() -> None: ### Given captured: list[list[str]] = [] - def _capture_run(cmd, timeout=None): + def _capture_run(cmd, timeout=None, cwd=None): captured.append(cmd) if cmd[3:5] == ["skill", "lint"]: return 0, "1 check", "" @@ -702,6 +713,9 @@ def _capture_run(cmd, timeout=None): return 0, '{"score": 80, "id": "rev_from_run"}', "" if cmd[3:6] == ["review", "view", "--last"]: return 0, '{"id": "rev_from_view", "score": 80}', "" + eval_handled = _eval_ok(cmd, timeout, cwd) + if eval_handled is not None: + return eval_handled raise AssertionError(f"unexpected cmd: {cmd}") ### When @@ -924,10 +938,12 @@ def _make_tessl_plugin(tmp_path) -> str: manifest_dir = plugin_dir / ".tessl-plugin" manifest_dir.mkdir() (manifest_dir / "plugin.json").write_text('{"name":"demo","version":"0.0.1"}') + (plugin_dir / "tessl.json").write_text('{"project":"demo"}') return str(plugin_dir) -def _lint_and_quality_ok(cmd, timeout=None): +def _lint_and_quality_ok(cmd, timeout=None, cwd=None): + del timeout, cwd if cmd[3:5] == ["skill", "lint"]: return 0, "1 check", "" if cmd[3:6] == ["review", "run", "quality"]: @@ -937,6 +953,29 @@ def _lint_and_quality_ok(cmd, timeout=None): raise AssertionError(f"unexpected cmd before scenario: {cmd}") +def _eval_ok(cmd, timeout=None, cwd=None): + """Handle Tessl project repair + eval run/view for auto-chain tests.""" + del timeout, cwd + if cmd[3:5] == ["project", "repair"]: + return 0, '{"ok": true}', "" + if cmd[3:5] == ["project", "create"]: + return 0, '{"ok": true}', "" + if cmd[3:5] == ["eval", "run"]: + assert "--runs" in cmd and "3" in cmd + assert "-y" in cmd + return 0, '{"id": "eval_xyz789", "status": "pending"}', "" + if cmd[3:5] == ["eval", "view"]: + return ( + 0, + ( + '{"id": "eval_xyz789", "status": "completed", "scenarioCount": 3, ' + '"baselineAvg": 0.4, "withContextAvg": 0.7, "delta": 0.3, "runs": 3}' + ), + "", + ) + return None + + def test_given_plugin_when_scenario_gen_succeeds_then_download_stamps_and_clears_checkpoint( tmp_path, ) -> None: @@ -955,7 +994,7 @@ def test_given_plugin_when_scenario_gen_succeeds_then_download_stamps_and_clears progress: list[dict] = [] captured: list[list[str]] = [] - def _run(cmd, timeout=None): + def _run(cmd, timeout=None, cwd=None): captured.append(cmd) if ( cmd[3:5] in (["skill", "lint"],) @@ -979,6 +1018,9 @@ def _run(cmd, timeout=None): os.makedirs(os.path.join(out_dir, "s2"), exist_ok=True) os.makedirs(os.path.join(out_dir, "s3"), exist_ok=True) return 0, "downloaded 3", "" + eval_handled = _eval_ok(cmd, timeout, cwd) + if eval_handled is not None: + return eval_handled raise AssertionError(f"unexpected cmd: {cmd}") ### When @@ -1022,7 +1064,7 @@ def test_given_resume_generated_when_run_tessl_then_skips_generate_and_downloads workdir = _make_tessl_plugin(tmp_path) captured: list[list[str]] = [] - def _run(cmd, timeout=None): + def _run(cmd, timeout=None, cwd=None): captured.append(cmd) if ( cmd[3:5] in (["skill", "lint"],) @@ -1038,6 +1080,9 @@ def _run(cmd, timeout=None): os.makedirs(os.path.join(out_dir, "a"), exist_ok=True) os.makedirs(os.path.join(out_dir, "b"), exist_ok=True) return 0, "ok", "" + eval_handled = _eval_ok(cmd, timeout, cwd) + if eval_handled is not None: + return eval_handled raise AssertionError(f"unexpected cmd: {cmd}") ### When @@ -1079,7 +1124,7 @@ def test_given_in_progress_checkpoint_when_resumed_then_polls_before_download( ] captured: list[list[str]] = [] - def _run(cmd, timeout=None): + def _run(cmd, timeout=None, cwd=None): captured.append(cmd) if ( cmd[3:5] in (["skill", "lint"],) @@ -1094,6 +1139,9 @@ def _run(cmd, timeout=None): out_dir = cmd[cmd.index("-o") + 1] os.makedirs(os.path.join(out_dir, "only"), exist_ok=True) return 0, "ok", "" + eval_handled = _eval_ok(cmd, timeout, cwd) + if eval_handled is not None: + return eval_handled raise AssertionError(f"unexpected cmd: {cmd}") ### When @@ -1130,14 +1178,17 @@ def test_given_quality_id_in_ctx_when_scenario_starts_then_upstream_run_ids_atta ctx = {"review_quality": "rev_abc123", "scenario_gen": None} progress: list[dict] = [] - def _run(cmd, timeout=None): + def _run(cmd, timeout=None, cwd=None): if cmd[3:5] == ["scenario", "generate"]: return 0, '{"id": "gen_x", "status": "completed", "scenarioCount": 1}', "" if cmd[3:5] == ["scenario", "download"]: out_dir = cmd[cmd.index("-o") + 1] os.makedirs(os.path.join(out_dir, "s"), exist_ok=True) return 0, "ok", "" - return _lint_and_quality_ok(cmd, timeout) + eval_handled = _eval_ok(cmd, timeout, cwd) + if eval_handled is not None: + return eval_handled + return _lint_and_quality_ok(cmd, timeout, cwd) ### When with ( @@ -1148,7 +1199,10 @@ def _run(cmd, timeout=None): _score, rows = scanners.run_tessl(workdir, id_context=ctx, on_row_progress=progress.append) ### Then - assert progress[0]["upstream_run_ids"] == {"review_quality": "rev_abc123"} + scenario_progress = [ + p for p in progress if p.get("scanner_source") == "Tessl: Scenario Generation" + ] + assert scenario_progress[0]["upstream_run_ids"] == {"review_quality": "rev_abc123"} assert rows[2]["upstream_run_ids"] == {"review_quality": "rev_abc123"} assert ctx["scenario_gen"] == "gen_x" @@ -1168,7 +1222,7 @@ def test_given_null_quality_id_when_scenario_starts_then_upstream_key_is_null( workdir = _make_tessl_plugin(tmp_path) captured: list[list[str]] = [] - def _run(cmd, timeout=None): + def _run(cmd, timeout=None, cwd=None): captured.append(cmd) if cmd[3:5] == ["skill", "lint"]: return 0, "1 check", "" @@ -1182,6 +1236,9 @@ def _run(cmd, timeout=None): out_dir = cmd[cmd.index("-o") + 1] os.makedirs(os.path.join(out_dir, "s"), exist_ok=True) return 0, "ok", "" + eval_handled = _eval_ok(cmd, timeout, cwd) + if eval_handled is not None: + return eval_handled raise AssertionError(f"unexpected cmd: {cmd}") ### When @@ -1211,7 +1268,7 @@ def test_given_scenario_generate_fails_when_run_tessl_then_row_is_failed(tmp_pat workdir = _make_tessl_plugin(tmp_path) captured: list[list[str]] = [] - def _run(cmd, timeout=None): + def _run(cmd, timeout=None, cwd=None): captured.append(cmd) if ( cmd[3:5] in (["skill", "lint"],) @@ -1221,6 +1278,9 @@ def _run(cmd, timeout=None): return _lint_and_quality_ok(cmd, timeout) if cmd[3:5] == ["scenario", "generate"]: return 1, "", "generation exploded" + eval_handled = _eval_ok(cmd, timeout, cwd) + if eval_handled is not None: + return eval_handled raise AssertionError(f"unexpected cmd: {cmd}") ### When @@ -1273,7 +1333,7 @@ def test_given_missing_plugin_manifest_when_scenario_runs_then_failed(tmp_path) workdir = str(tmp_path / "no-plugin") os.makedirs(workdir) - def _run(cmd, timeout=None): + def _run(cmd, timeout=None, cwd=None): return _lint_and_quality_ok(cmd, timeout) ### When @@ -1346,7 +1406,7 @@ def test_given_generate_timeout_when_scenario_runs_then_interrupted_with_checkpo ### Given workdir = _make_tessl_plugin(tmp_path) - def _run(cmd, timeout=None): + def _run(cmd, timeout=None, cwd=None): if cmd[3:5] == ["scenario", "generate"]: return None, "", "timeout after 240s" if cmd[3:5] == ["scenario", "view"]: @@ -1383,7 +1443,7 @@ def test_given_resume_failed_status_when_polled_then_download_is_skipped(tmp_pat workdir = _make_tessl_plugin(tmp_path) captured: list[list[str]] = [] - def _run(cmd, timeout=None): + def _run(cmd, timeout=None, cwd=None): captured.append(cmd) if cmd[3:5] == ["scenario", "view"]: return 0, '{"id": "gen_fail", "status": "failed"}', "server failed" @@ -1419,7 +1479,7 @@ def test_given_download_fails_when_scenario_completes_then_checkpoint_retained( ### Given workdir = _make_tessl_plugin(tmp_path) - def _run(cmd, timeout=None): + def _run(cmd, timeout=None, cwd=None): if cmd[3:5] == ["scenario", "generate"]: return 0, '{"id": "gen_dl", "status": "completed"}', "" if cmd[3:5] == ["scenario", "download"]: @@ -1453,7 +1513,7 @@ def test_given_empty_evals_when_download_succeeds_then_count_comes_from_view( ### Given workdir = _make_tessl_plugin(tmp_path) - def _run(cmd, timeout=None): + def _run(cmd, timeout=None, cwd=None): if cmd[3:5] == ["scenario", "generate"]: return 0, '{"id": "gen_view_count"}', "" if cmd[3:5] == ["scenario", "download"]: @@ -1474,3 +1534,762 @@ def _run(cmd, timeout=None): assert rows[2]["status"] == "completed" assert rows[2]["checks_run"] == 5 assert rows[2]["resume_checkpoint"] is None + assert rows[3]["scanner_source"] == "Tessl: Eval" + assert rows[3]["status"] == "blocked" + + +# --- slice-50: Tessl Eval + Scenario→Eval Auto-Chain (Row 4) --- + + +def test_given_lint_review_when_run_tessl_then_eval_emitted_blocked_before_scenario( + tmp_path, +) -> None: + """ + Scenario: Eval row is emitted blocked before Scenario Generation starts. + Slice: 50 — GWT-50.0 + + Given Tessl group runner begins for a skill scan, + When Lint and Review rows are emitted, + Then an Eval row is inserted with status blocked and no tessl_run_id, + And Eval stays blocked while Scenario Generation is still running. + """ + ### Given + workdir = _make_tessl_plugin(tmp_path) + progress: list[dict] = [] + + def _run(cmd, timeout=None, cwd=None): + if cmd[3:5] == ["scenario", "generate"]: + eval_snapshots = [p for p in progress if p.get("scanner_source") == "Tessl: Eval"] + assert eval_snapshots, "Eval blocked row must be emitted before generate" + assert eval_snapshots[0]["status"] == "blocked" + assert eval_snapshots[0].get("tessl_run_id") is None + return 1, "", "scenario failed intentionally" + return _lint_and_quality_ok(cmd, timeout, cwd) + + ### When + with ( + patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_which", return_value="/usr/bin/npx"), + patch.object(scanners, "_run", side_effect=_run), + ): + _score, rows = scanners.run_tessl(workdir, on_row_progress=progress.append) + + ### Then + assert rows[3]["scanner_source"] == "Tessl: Eval" + assert rows[3]["status"] == "blocked" + assert rows[3].get("tessl_run_id") is None + + +def test_given_scenario_completed_with_evals_when_run_tessl_then_eval_auto_chains( + tmp_path, +) -> None: + """ + Scenario: First-run auto-chain from Scenario Generation into Eval. + Slice: 50 — GWT-50.1 / GWT-50.2 / GWT-50.4 / GWT-50.5 + + Given Scenario Generation completed and evals/ has scenarios, + When the auto-chain check runs, + Then Eval transitions queued→running, invokes eval run --runs 3 -y, + stamps tessl_run_id, upstream_run_ids, and score detail without failing on variance. + """ + ### Given + workdir = _make_tessl_plugin(tmp_path) + ctx = scanners._new_tessl_id_context() + progress: list[dict] = [] + captured: list[list[str]] = [] + + def _run(cmd, timeout=None, cwd=None): + captured.append(cmd) + if ( + cmd[3:5] in (["skill", "lint"],) + or cmd[3:5] == ["review", "run"] + or (len(cmd) > 5 and cmd[3:5] == ["review", "view"]) + ): + return _lint_and_quality_ok(cmd, timeout, cwd) + if cmd[3:5] == ["scenario", "generate"]: + return 0, '{"id": "gen_for_eval", "status": "completed", "scenarioCount": 2}', "" + if cmd[3:5] == ["scenario", "download"]: + out_dir = cmd[cmd.index("-o") + 1] + os.makedirs(os.path.join(out_dir, "s1"), exist_ok=True) + os.makedirs(os.path.join(out_dir, "s2"), exist_ok=True) + return 0, "ok", "" + eval_handled = _eval_ok(cmd, timeout, cwd) + if eval_handled is not None: + return eval_handled + raise AssertionError(f"unexpected cmd: {cmd}") + + ### When + with ( + patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_which", return_value="/usr/bin/npx"), + patch.object(scanners, "_run", side_effect=_run), + patch.object(scanners, "_TESSL_EVAL_POLL_SLEEP_S", 0), + ): + score, rows = scanners.run_tessl(workdir, id_context=ctx, on_row_progress=progress.append) + + ### Then + assert score == 80 + eval_row = rows[3] + assert eval_row["scanner_source"] == "Tessl: Eval" + assert eval_row["status"] == "completed" + assert eval_row["tessl_run_id"] == "eval_xyz789" + assert eval_row["tessl_run_id_at"] + assert eval_row["checks_run"] == 3 + assert eval_row["upstream_run_ids"] == { + "review_quality": "rev_abc123", + "scenario_gen": "gen_for_eval", + } + assert "baseline avg" in eval_row["detail"] + assert "with-context avg" in eval_row["detail"] + assert "delta=" in eval_row["detail"] + eval_statuses = [p["status"] for p in progress if p.get("scanner_source") == "Tessl: Eval"] + assert "blocked" in eval_statuses + assert "queued" in eval_statuses + assert "running" in eval_statuses + eval_cmds = [c for c in captured if c[3:5] == ["eval", "run"]] + assert len(eval_cmds) == 1 + assert "--runs" in eval_cmds[0] and "3" in eval_cmds[0] + assert "-y" in eval_cmds[0] + + +def test_given_scenario_failed_when_run_tessl_then_eval_stays_blocked(tmp_path) -> None: + """ + Scenario: Failed Scenario Generation leaves Eval blocked (partial ctx). + Slice: 50 — GWT-50.4b + + Given Scenario Generation failed and ctx scenario_gen is still null, + When Eval auto-chain gate runs, + Then Eval remains blocked with no eval invocation. + """ + ### Given + workdir = _make_tessl_plugin(tmp_path) + ctx = scanners._new_tessl_id_context() + captured: list[list[str]] = [] + + def _run(cmd, timeout=None, cwd=None): + captured.append(cmd) + if cmd[3:5] == ["scenario", "generate"]: + return 1, "", "boom" + return _lint_and_quality_ok(cmd, timeout, cwd) + + ### When + with ( + patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_which", return_value="/usr/bin/npx"), + patch.object(scanners, "_run", side_effect=_run), + ): + _score, rows = scanners.run_tessl(workdir, id_context=ctx) + + ### Then + assert rows[3]["status"] == "blocked" + assert ctx["scenario_gen"] is None + assert not any(c[3:5] == ["eval", "run"] for c in captured) + + +def test_given_prior_completed_eval_when_scenario_rerun_then_eval_is_stale( + tmp_path, +) -> None: + """ + Scenario: Scenario Generation re-run marks prior completed Eval as stale. + Slice: 50 — GWT-50.3 + + Given Eval previously completed with a tessl_run_id, + When Scenario Generation is re-run with a new gen id, + Then Eval status becomes stale and no new eval run is triggered. + """ + ### Given + workdir = _make_tessl_plugin(tmp_path) + prior_eval = { + "scanner_source": "Tessl: Eval", + "status": "completed", + "tessl_run_id": "eval_old", + "tessl_run_id_at": "2026-08-24T10:00:00+00:00", + "completed_at": "2026-08-24T10:05:00+00:00", + "checks_run": 2, + "detail": "baseline avg=0.5", + "upstream_run_ids": { + "review_quality": "rev_abc123", + "scenario_gen": "gen_old", + }, + } + captured: list[list[str]] = [] + + def _run(cmd, timeout=None, cwd=None): + captured.append(cmd) + if cmd[3:5] == ["scenario", "generate"]: + return 0, '{"id": "gen_new", "status": "completed", "scenarioCount": 1}', "" + if cmd[3:5] == ["scenario", "download"]: + out_dir = cmd[cmd.index("-o") + 1] + os.makedirs(os.path.join(out_dir, "s"), exist_ok=True) + return 0, "ok", "" + return _lint_and_quality_ok(cmd, timeout, cwd) + + ### When + with ( + patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_which", return_value="/usr/bin/npx"), + patch.object(scanners, "_run", side_effect=_run), + ): + _score, rows = scanners.run_tessl(workdir, prior_eval=prior_eval) + + ### Then + assert rows[3]["status"] == "stale" + assert rows[3]["tessl_run_id"] == "eval_old" + assert not any(c[3:5] == ["eval", "run"] for c in captured) + + +def test_given_interrupted_eval_when_resumed_then_polls_view_without_resubmit( + tmp_path, +) -> None: + """ + Scenario: Modal timeout resume polls eval view without re-submitting eval run. + Slice: 50 — GWT-50.2b + + Given eval run was detached with an eval_id while pending, + When the runner resumes, + Then it polls eval view until completed and does not call eval run again. + """ + ### Given + workdir = _make_tessl_plugin(tmp_path) + prior_eval = { + "scanner_source": "Tessl: Eval", + "status": "interrupted", + "tessl_run_id": "eval_resume_me", + "upstream_run_ids": { + "review_quality": "rev_abc123", + "scenario_gen": "gen_x", + }, + } + # Pre-populate evals so auto-chain gate would otherwise fire a new run. + os.makedirs(os.path.join(workdir, "evals", "s1"), exist_ok=True) + captured: list[list[str]] = [] + + def _run(cmd, timeout=None, cwd=None): + captured.append(cmd) + if cmd[3:5] == ["scenario", "generate"]: + return 0, '{"id": "gen_x", "status": "completed"}', "" + if cmd[3:5] == ["scenario", "download"]: + return 0, "ok", "" + if cmd[3:5] == ["project", "repair"]: + return 0, "{}", "" + if cmd[3:5] == ["eval", "run"]: + raise AssertionError("must not re-submit eval run while prior pending") + if cmd[3:5] == ["eval", "view"]: + assert "eval_resume_me" in cmd + return ( + 0, + '{"id": "eval_resume_me", "status": "completed", "scenarioCount": 1, ' + '"baselineAvg": 0.2, "withContextAvg": 0.5, "delta": 0.3}', + "", + ) + return _lint_and_quality_ok(cmd, timeout, cwd) + + ### When + with ( + patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_which", return_value="/usr/bin/npx"), + patch.object(scanners, "_run", side_effect=_run), + patch.object(scanners, "_TESSL_EVAL_POLL_SLEEP_S", 0), + ): + _score, rows = scanners.run_tessl(workdir, prior_eval=prior_eval) + + ### Then + assert rows[3]["status"] == "completed" + assert rows[3]["tessl_run_id"] == "eval_resume_me" + assert not any(c[3:5] == ["eval", "run"] for c in captured) + assert any(c[3:5] == ["eval", "view"] for c in captured) + + +def test_given_missing_tessl_json_when_eval_chains_then_project_create_or_needs_setup( + tmp_path, +) -> None: + """ + Scenario: Eval preflight creates Tessl project or reports needs_setup. + Slice: 50 — GWT-50.6 + + Given plugin directory has no tessl.json, + When Eval auto-chain attempts to run, + Then project create is invoked; on failure Eval is needs_setup with actionable detail. + """ + ### Given + workdir = _make_tessl_plugin(tmp_path) + os.remove(os.path.join(workdir, "tessl.json")) + captured: list[list[str]] = [] + + def _run(cmd, timeout=None, cwd=None): + captured.append(cmd) + if cmd[3:5] == ["scenario", "generate"]: + return 0, '{"id": "gen_proj", "status": "completed"}', "" + if cmd[3:5] == ["scenario", "download"]: + out_dir = cmd[cmd.index("-o") + 1] + os.makedirs(os.path.join(out_dir, "s"), exist_ok=True) + return 0, "ok", "" + if cmd[3:5] == ["project", "create"]: + assert cwd == workdir + return 1, "", "cannot create project headlessly" + return _lint_and_quality_ok(cmd, timeout, cwd) + + ### When + with ( + patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_which", return_value="/usr/bin/npx"), + patch.object(scanners, "_run", side_effect=_run), + ): + _score, rows = scanners.run_tessl(workdir) + + ### Then + assert rows[3]["status"] == "needs_setup" + assert "project" in rows[3]["detail"].lower() + assert any(c[3:5] == ["project", "create"] for c in captured) + assert not any(c[3:5] == ["eval", "run"] for c in captured) + + +def test_parse_eval_id_and_detail_from_json_shapes() -> None: + """ + Scenario: Eval JSON parsers accept nested id/score shapes. + Slice: 50 — parser helpers + + Given common Tessl eval --json envelopes, + When parsers extract id/status/checks/detail, + Then nested and list forms resolve correctly. + """ + ### Given / When / Then + assert scanners._parse_eval_id({"id": "e1"}) == "e1" + assert scanners._parse_eval_id({"eval": {"runId": "e2"}}) == "e2" + assert scanners._parse_eval_id({"evals": [{"id": "e3"}]}) == "e3" + assert scanners._parse_eval_id({"evals": ["skip"]}) is None + assert scanners._parse_eval_id("bad") is None + assert scanners._parse_eval_status({"status": "Completed"}) == "completed" + assert scanners._parse_eval_status("bad") is None + assert scanners._parse_eval_status({"eval": {}}) is None + assert scanners._parse_eval_checks_run({"scenarioCount": 4}) == 4 + assert scanners._parse_eval_checks_run("bad") is None + detail = scanners._format_eval_detail( + {"baselineAvg": 0.1, "withContextAvg": 0.4, "delta": 0.3, "runs": 3} + ) + assert "baseline avg=0.1" in detail + assert "with-context avg=0.4" in detail + assert "delta=0.3" in detail + assert "runs=3" in detail + assert scanners._eval_should_mark_stale({"status": "running"}, {"status": "completed"}) is False + assert scanners._eval_should_mark_stale({"status": "completed"}, {"status": "failed"}) is False + + +def test_eval_should_mark_stale_by_timestamp_when_gen_id_unchanged() -> None: + """ + Scenario: Stale when scenario tessl_run_id_at is newer than eval completed_at. + Slice: 50 — stale timestamp path + + Given prior Eval completed with same scenario_gen id, + When scenario_gen tessl_run_id_at is newer than eval completed_at, + Then _eval_should_mark_stale is True. + """ + ### Given + prior = { + "status": "completed", + "completed_at": "2026-08-24T10:00:00+00:00", + "upstream_run_ids": {"scenario_gen": "gen_same"}, + } + scenario = { + "status": "completed", + "tessl_run_id": "gen_same", + "tessl_run_id_at": "2026-08-24T12:00:00+00:00", + } + + ### When / Then + assert scanners._eval_should_mark_stale(prior, scenario) is True + assert ( + scanners._eval_should_mark_stale( + prior, + { + "status": "completed", + "tessl_run_id": "gen_same", + "tessl_run_id_at": "2026-08-24T09:00:00+00:00", + }, + ) + is False + ) + + +def test_given_prior_completed_unchanged_when_run_tessl_then_eval_kept( + tmp_path, +) -> None: + """ + Scenario: Unchanged Scenario Gen keeps prior completed Eval (no stale, no re-run). + Slice: 50 — keep completed + + Given prior Eval completed for the same scenario_gen id, + When Scenario Generation completes again with the same id, + Then Eval stays completed and eval run is not invoked. + """ + ### Given + workdir = _make_tessl_plugin(tmp_path) + prior_eval = { + "scanner_source": "Tessl: Eval", + "status": "completed", + "tessl_run_id": "eval_keep", + "checks_run": 2, + "detail": "kept", + "upstream_run_ids": { + "review_quality": "rev_abc123", + "scenario_gen": "gen_same", + }, + } + captured: list[list[str]] = [] + + def _run(cmd, timeout=None, cwd=None): + captured.append(cmd) + if cmd[3:5] == ["scenario", "generate"]: + return 0, '{"id": "gen_same", "status": "completed"}', "" + if cmd[3:5] == ["scenario", "view"]: + return 0, '{"id": "gen_same", "status": "completed", "scenarioCount": 1}', "" + if cmd[3:5] == ["scenario", "download"]: + out_dir = cmd[cmd.index("-o") + 1] + os.makedirs(os.path.join(out_dir, "s"), exist_ok=True) + return 0, "ok", "" + return _lint_and_quality_ok(cmd, timeout, cwd) + + ### When + with ( + patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_which", return_value="/usr/bin/npx"), + patch.object(scanners, "_run", side_effect=_run), + ): + _score, rows = scanners.run_tessl( + workdir, + prior_eval=prior_eval, + resume_checkpoint={"stage": "generated", "gen_id": "gen_same"}, + ) + + ### Then + assert rows[3]["status"] == "completed" + assert rows[3]["tessl_run_id"] == "eval_keep" + assert not any(c[3:5] == ["eval", "run"] for c in captured) + + +def test_given_eval_run_timeout_with_id_when_chained_then_interrupted(tmp_path) -> None: + """ + Scenario: Eval run timeout with captured id marks interrupted. + Slice: 50 — detach with id + + Given eval run --json times out after emitting an id, + When auto-chain finishes, + Then status is interrupted and tessl_run_id is stamped. + """ + ### Given + workdir = _make_tessl_plugin(tmp_path) + + def _run(cmd, timeout=None, cwd=None): + if cmd[3:5] == ["scenario", "generate"]: + return 0, '{"id": "gen_to", "status": "completed"}', "" + if cmd[3:5] == ["scenario", "download"]: + out_dir = cmd[cmd.index("-o") + 1] + os.makedirs(os.path.join(out_dir, "s"), exist_ok=True) + return 0, "ok", "" + if cmd[3:5] == ["project", "repair"]: + return 0, "{}", "" + if cmd[3:5] == ["eval", "run"]: + return None, '{"id": "eval_detached"}', "timeout after 240s" + return _lint_and_quality_ok(cmd, timeout, cwd) + + ### When + with ( + patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_which", return_value="/usr/bin/npx"), + patch.object(scanners, "_run", side_effect=_run), + ): + _score, rows = scanners.run_tessl(workdir) + + ### Then + assert rows[3]["status"] == "interrupted" + assert rows[3]["tessl_run_id"] == "eval_detached" + + +def test_given_eval_run_timeout_without_id_when_chained_then_timed_out(tmp_path) -> None: + """ + Scenario: Eval run timeout without id marks timed_out. + Slice: 50 — detach without id + """ + ### Given + workdir = _make_tessl_plugin(tmp_path) + + def _run(cmd, timeout=None, cwd=None): + if cmd[3:5] == ["scenario", "generate"]: + return 0, '{"id": "gen_to2", "status": "completed"}', "" + if cmd[3:5] == ["scenario", "download"]: + out_dir = cmd[cmd.index("-o") + 1] + os.makedirs(os.path.join(out_dir, "s"), exist_ok=True) + return 0, "ok", "" + if cmd[3:5] == ["project", "repair"]: + return 0, "{}", "" + if cmd[3:5] == ["eval", "run"]: + return None, "still starting", "timeout after 240s" + return _lint_and_quality_ok(cmd, timeout, cwd) + + ### When + with ( + patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_which", return_value="/usr/bin/npx"), + patch.object(scanners, "_run", side_effect=_run), + ): + _score, rows = scanners.run_tessl(workdir) + + ### Then + assert rows[3]["status"] == "timed_out" + + +def test_given_eval_run_nonzero_when_chained_then_failed(tmp_path) -> None: + """ + Scenario: Non-zero eval run without id marks failed. + Slice: 50 — eval CLI failure + """ + ### Given + workdir = _make_tessl_plugin(tmp_path) + + def _run(cmd, timeout=None, cwd=None): + if cmd[3:5] == ["scenario", "generate"]: + return 0, '{"id": "gen_fail", "status": "completed"}', "" + if cmd[3:5] == ["scenario", "download"]: + out_dir = cmd[cmd.index("-o") + 1] + os.makedirs(os.path.join(out_dir, "s"), exist_ok=True) + return 0, "ok", "" + if cmd[3:5] == ["project", "repair"]: + return 0, "{}", "" + if cmd[3:5] == ["eval", "run"]: + return 1, "", "eval exploded" + return _lint_and_quality_ok(cmd, timeout, cwd) + + ### When + with ( + patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_which", return_value="/usr/bin/npx"), + patch.object(scanners, "_run", side_effect=_run), + ): + _score, rows = scanners.run_tessl(workdir) + + ### Then + assert rows[3]["status"] == "failed" + assert "exploded" in rows[3]["detail"] + + +def test_given_eval_view_failed_when_chained_then_row_failed(tmp_path) -> None: + """ + Scenario: eval view status failed marks Eval failed (not score variance). + Slice: 50 — GWT-50.5 failure path + """ + ### Given + workdir = _make_tessl_plugin(tmp_path) + + def _run(cmd, timeout=None, cwd=None): + if cmd[3:5] == ["scenario", "generate"]: + return 0, '{"id": "gen_vf", "status": "completed"}', "" + if cmd[3:5] == ["scenario", "download"]: + out_dir = cmd[cmd.index("-o") + 1] + os.makedirs(os.path.join(out_dir, "s"), exist_ok=True) + return 0, "ok", "" + if cmd[3:5] == ["project", "repair"]: + return 0, "{}", "" + if cmd[3:5] == ["eval", "run"]: + return 0, '{"id": "eval_fail_view"}', "" + if cmd[3:5] == ["eval", "view"]: + return 0, '{"id": "eval_fail_view", "status": "failed"}', "" + return _lint_and_quality_ok(cmd, timeout, cwd) + + ### When + with ( + patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_which", return_value="/usr/bin/npx"), + patch.object(scanners, "_run", side_effect=_run), + patch.object(scanners, "_TESSL_EVAL_POLL_SLEEP_S", 0), + ): + _score, rows = scanners.run_tessl(workdir) + + ### Then + assert rows[3]["status"] == "failed" + assert rows[3]["tessl_run_id"] == "eval_fail_view" + + +def test_given_eval_run_ok_without_id_when_chained_then_completed(tmp_path) -> None: + """ + Scenario: Eval run exits 0 without parseable id still completes. + Slice: 50 — no-id success path + """ + ### Given + workdir = _make_tessl_plugin(tmp_path) + + def _run(cmd, timeout=None, cwd=None): + if cmd[3:5] == ["scenario", "generate"]: + return 0, '{"id": "gen_noid", "status": "completed"}', "" + if cmd[3:5] == ["scenario", "download"]: + out_dir = cmd[cmd.index("-o") + 1] + os.makedirs(os.path.join(out_dir, "s"), exist_ok=True) + return 0, "ok", "" + if cmd[3:5] == ["project", "repair"]: + return 0, "{}", "" + if cmd[3:5] == ["eval", "run"]: + return 0, "eval finished without json id", "" + return _lint_and_quality_ok(cmd, timeout, cwd) + + ### When + with ( + patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_which", return_value="/usr/bin/npx"), + patch.object(scanners, "_run", side_effect=_run), + ): + _score, rows = scanners.run_tessl(workdir) + + ### Then + assert rows[3]["status"] == "completed" + assert rows[3].get("tessl_run_id") is None + + +def test_ensure_tessl_project_create_writes_link(tmp_path) -> None: + """ + Scenario: Missing tessl.json — project create success returns ok. + Slice: 50 — project create happy path + """ + ### Given + workdir = str(tmp_path / "plugin") + os.makedirs(workdir) + + def _run(cmd, timeout=None, cwd=None): + assert cmd[3:5] == ["project", "create"] + assert cwd == workdir + (tmp_path / "plugin" / "tessl.json").write_text("{}") + return 0, "created", "" + + ### When + with patch.object(scanners, "_run", side_effect=_run): + ok, detail = scanners._ensure_tessl_project(workdir, "engteam") + + ### Then + assert ok is True + assert detail == "" + + +def test_poll_eval_until_terminal_fails_on_nonzero_without_status() -> None: + """ + Scenario: eval view non-zero exit without status maps to failed. + Slice: 50 — poll failure path + """ + ### Given / When + with ( + patch.object(scanners, "_run", return_value=(1, "not json", "boom")), + patch.object(scanners, "_TESSL_EVAL_POLL_SLEEP_S", 0), + ): + status, _parsed, console = scanners._poll_eval_until_terminal("eval_x") + + ### Then + assert status == "failed" + assert "boom" in console or "not json" in console + + +def test_run_tessl_eval_without_token_returns_needs_setup(tmp_path) -> None: + """ + Scenario: Direct _run_tessl_eval without credentials is needs_setup. + Slice: 50 — credential gate on eval helper + """ + ### Given + row = scanners._new_blocked_eval_row() + ctx = {"review_quality": None, "scenario_gen": "gen_1"} + + ### When + with patch.dict("os.environ", {}, clear=True): + result = scanners._run_tessl_eval(str(tmp_path), ctx, row) + + ### Then + assert result["status"] == "needs_setup" + assert "TESSL_TOKEN" in result["detail"] + + +def test_ensure_tessl_project_create_timeout_returns_false(tmp_path) -> None: + """ + Scenario: project create timeout yields actionable failure detail. + Slice: 50 — project create timeout + """ + ### Given + workdir = str(tmp_path / "bare") + os.makedirs(workdir) + + ### When + with patch.object(scanners, "_run", return_value=(None, "", "timeout after 240s")): + ok, detail = scanners._ensure_tessl_project(workdir, "engteam") + + ### Then + assert ok is False + assert "timed out" in detail.lower() or "timeout" in detail.lower() + + +def test_format_eval_detail_uses_run_count_and_fallback() -> None: + """ + Scenario: Detail formatter uses runCount and falls back when empty. + Slice: 50 — detail edge paths + """ + ### Given / When / Then + assert "runs=5" in scanners._format_eval_detail({"runCount": 5}) + assert scanners._format_eval_detail({}) == "eval completed" + assert scanners._parse_eval_score_field({"scores": {"delta": 0.2}}, "delta") == 0.2 + assert scanners._parse_eval_checks_run({"results": [1, 2]}) == 2 + assert scanners._parse_eval_status({"evaluation": {"state": "Failed"}}) == "failed" + + +def test_poll_eval_until_terminal_returns_pending_after_max_attempts() -> None: + """ + Scenario: Exhausted eval poll returns last non-terminal status. + Slice: 50 — poll max attempts + """ + ### Given / When + with ( + patch.object( + scanners, + "_run", + return_value=(0, '{"id": "e", "status": "pending"}', ""), + ), + patch.object(scanners, "_TESSL_EVAL_POLL_SLEEP_S", 0), + patch.object(scanners, "_TESSL_EVAL_POLL_MAX", 2), + ): + status, parsed, _console = scanners._poll_eval_until_terminal("e") + + ### Then + assert status == "pending" + assert parsed is not None + + +def test_given_eval_view_pending_exhausted_when_chained_then_interrupted( + tmp_path, +) -> None: + """ + Scenario: Eval view never reaches terminal → interrupted for resume. + Slice: 50 — poll interrupted + """ + ### Given + workdir = _make_tessl_plugin(tmp_path) + + def _run(cmd, timeout=None, cwd=None): + if cmd[3:5] == ["scenario", "generate"]: + return 0, '{"id": "gen_pend", "status": "completed"}', "" + if cmd[3:5] == ["scenario", "download"]: + out_dir = cmd[cmd.index("-o") + 1] + os.makedirs(os.path.join(out_dir, "s"), exist_ok=True) + return 0, "ok", "" + if cmd[3:5] == ["project", "repair"]: + return 0, "{}", "" + if cmd[3:5] == ["eval", "run"]: + return 0, '{"id": "eval_pend"}', "" + if cmd[3:5] == ["eval", "view"]: + return 0, '{"id": "eval_pend", "status": "pending"}', "" + return _lint_and_quality_ok(cmd, timeout, cwd) + + ### When + with ( + patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_which", return_value="/usr/bin/npx"), + patch.object(scanners, "_run", side_effect=_run), + patch.object(scanners, "_TESSL_EVAL_POLL_SLEEP_S", 0), + patch.object(scanners, "_TESSL_EVAL_POLL_MAX", 2), + ): + _score, rows = scanners.run_tessl(workdir) + + ### Then + assert rows[3]["status"] == "interrupted" + assert rows[3]["tessl_run_id"] == "eval_pend" diff --git a/sandbox/tests/test_ship_path_coverage.py b/sandbox/tests/test_ship_path_coverage.py index 71b5fe0..85d0683 100644 --- a/sandbox/tests/test_ship_path_coverage.py +++ b/sandbox/tests/test_ship_path_coverage.py @@ -621,6 +621,154 @@ def test_given_snyk_valid_envelope_with_no_issues_when_run_then_completed() -> N assert rows[0]["checks_run"] >= 1 +def test_given_snyk_v06_clean_scan_path_responses_when_run_then_completed() -> None: + """ + Scenario: Agent Scan v0.6 clean envelope is completed, not unreachable. + Slice: slice-11 — run_snyk v0.6 clean + + Given SNYK_TOKEN and a scan_path_responses payload with skill_risks but empty risk_indexes, + When run_snyk maps it, + Then status is completed with zero findings. + """ + ### Given + payload = { + "scan_path_responses": [ + { + "client": "/tmp/scan-target", + "path": "/tmp", + "server_risks": [], + "skill_risks": [ + { + "name": "content-distiller", + "files": [{"name": "SKILL.md", "type": "instruction"}], + "risk_indexes": {}, + } + ], + } + ] + } + + ### When + with ( + patch.dict("os.environ", {"SNYK_TOKEN": "t"}, clear=False), + patch.object(scanners, "_which", return_value=True), + patch.object(scanners, "_run", return_value=(0, json.dumps(payload), "")), + ): + findings, rows = scanners.run_snyk("/tmp", "skill") + + ### Then + assert findings == [] + assert rows[0]["status"] == "completed" + assert rows[0]["checks_run"] >= 1 + + +def test_given_snyk_v06_risk_indexes_when_run_then_red_and_amber_findings() -> None: + """ + Scenario: v0.6 risk_indexes map to Tripwire findings by score band. + Slice: slice-11 — run_snyk v0.6 risks + + Given skill risk score 1000 and server risk score 300, + When run_snyk maps them, + Then one red and one amber finding are emitted and status is completed. + """ + ### Given + payload = { + "scan_path_responses": [ + { + "path": "/tmp", + "server_risks": [ + { + "name": "github", + "entities": [{"name": "search", "type": "tool"}], + "risk_indexes": { + "dangerous_words": { + "score": 300, + "evidence": "Manipulative language in tool desc.", + "affected_tools": [0], + } + }, + } + ], + "skill_risks": [ + { + "name": "release-helper", + "files": [{"name": "SKILL.md", "type": "instruction"}], + "risk_indexes": { + "prompt_injection_skill_instructions": { + "score": 1000, + "evidence": "Hidden directives in SKILL.md.", + } + }, + } + ], + } + ] + } + + ### When + with ( + patch.dict("os.environ", {"SNYK_TOKEN": "t"}, clear=False), + patch.object(scanners, "_which", return_value=True), + patch.object(scanners, "_run", return_value=(0, json.dumps(payload), "")), + ): + findings, rows = scanners.run_snyk("/tmp", "skill") + + ### Then + assert rows[0]["status"] == "completed" + assert len(findings) == 2 + by_cat = {f["category"]: f for f in findings} + assert by_cat["prompt_injection_skill_instructions"]["severity"] == "red" + assert "Hidden directives" in by_cat["prompt_injection_skill_instructions"]["message"] + assert by_cat["dangerous_words"]["severity"] == "amber" + + +def test_given_snyk_v06_skill_unauthorized_when_run_then_skipped_credential() -> None: + """ + Scenario: v0.6 skill-level Unauthorized is a credential skip. + Slice: slice-11 — run_snyk v0.6 auth + + Given skill_risks[].error Unauthorized 401 and no risk findings, + When run_snyk maps it, + Then status is skipped_missing_credential. + """ + ### Given + payload = { + "scan_path_responses": [ + { + "path": "/tmp", + "server_risks": [], + "skill_risks": [ + { + "name": "scan-target", + "error": { + "message": ( + "Unauthorized. Please check your SNYK_TOKEN " + "environment variable or your push key." + ), + "exception": "401, message='Unauthorized'", + "is_failure": True, + "category": "analysis_error", + }, + } + ], + } + ] + } + + ### When + with ( + patch.dict("os.environ", {"SNYK_TOKEN": "t"}, clear=False), + patch.object(scanners, "_which", return_value=True), + patch.object(scanners, "_run", return_value=(1, json.dumps(payload), "")), + ): + findings, rows = scanners.run_snyk("/tmp", "skill") + + ### Then + assert findings == [] + assert rows[0]["status"] == "skipped_missing_credential" + assert "Unauthorized" in rows[0].get("detail", "") + + def test_given_no_snyk_binaries_when_cmd_then_none() -> None: """ Scenario: Neither snyk-agent-scan nor uvx → no command. @@ -700,13 +848,15 @@ def test_given_no_tessl_token_when_run_then_skipped() -> None: ### Then assert score is None - assert len(rows) == 3 + assert len(rows) == 4 assert rows[0]["scanner_source"] == "Tessl: Lint" assert rows[0]["status"] == "completed" assert rows[1]["scanner_source"] == "Tessl: Review (Quality)" assert rows[1]["status"] == "needs_setup" assert rows[2]["scanner_source"] == "Tessl: Scenario Generation" assert rows[2]["status"] == "needs_setup" + assert rows[3]["scanner_source"] == "Tessl: Eval" + assert rows[3]["status"] == "blocked" def test_given_tessl_npx_missing_when_run_then_unreachable() -> None: @@ -815,34 +965,55 @@ def _fake_all(**kwargs): def test_given_scenario_checkpoint_when_scan_item_inner_then_resume_and_progress_wired() -> None: """ - Scenario: scan_item_inner loads Tessl resume_checkpoint and persists progress rows. - Slice: 49 — scan_app mid-scan persist + Scenario: scan_item_inner loads Tessl resume_checkpoint and Eval prior row. + Slice: 49/50 — scan_app mid-scan persist - Given an existing Scenario Generation resume_checkpoint in Supabase, + Given an existing Scenario Generation resume_checkpoint and Eval row in Supabase, When _scan_item_inner runs, - Then run_all_scanners receives tessl_scenario_resume and on_scanner_progress writes. + Then run_all_scanners receives tessl_scenario_resume, tessl_prior_eval, + and on_scanner_progress writes. """ ### Given checkpoint = {"stage": "generated", "gen_id": "gen_persisted"} + prior_eval = { + "status": "interrupted", + "tessl_run_id": "eval_resume", + "tessl_run_id_at": "2026-08-25T00:00:00+00:00", + "completed_at": None, + "upstream_run_ids": {"review_quality": "rev_1", "scenario_gen": "gen_1"}, + "detail": "detached", + "checks_run": 0, + } sb = _supabase_chain(data=[{"id": "1"}]) - select_chain = MagicMock() - select_chain.eq.return_value.eq.return_value.limit.return_value.execute.return_value = ( - MagicMock(data=[{"resume_checkpoint": checkpoint}]) + select_results = iter( + [ + MagicMock(data=[{"resume_checkpoint": checkpoint}]), + MagicMock(data=[prior_eval]), + ] ) - sb.table.return_value.select.return_value = select_chain + + def _select_side_effect(*_args, **_kwargs): + chain = MagicMock() + chain.eq.return_value.eq.return_value.limit.return_value.execute.side_effect = lambda: next( + select_results + ) + return chain + + sb.table.return_value.select.side_effect = _select_side_effect captured: dict = {} def _fake_all(**kwargs): captured["resume"] = kwargs.get("tessl_scenario_resume") + captured["prior_eval"] = kwargs.get("tessl_prior_eval") assert kwargs["on_scanner_progress"] is not None kwargs["on_scanner_progress"]( { - "scanner_source": "Tessl: Scenario Generation", - "status": "interrupted", - "resume_checkpoint": checkpoint, + "scanner_source": "Tessl: Eval", + "status": "blocked", + "checks_run": 0, } ) - kwargs["on_scanner_start"](["Tessl: Scenario Generation"]) + kwargs["on_scanner_start"](["Tessl: Scenario Generation", "Tessl: Eval"]) kwargs["on_scanner_done"]([], [], None) return { "overall_status": "complete", @@ -860,9 +1031,46 @@ def _fake_all(**kwargs): ### Then assert captured["resume"] == checkpoint + assert captured["prior_eval"] == prior_eval assert sb.table.called +def test_given_resume_select_errors_when_scan_item_inner_then_continues() -> None: + """ + Scenario: Tessl resume/prior select failures are non-fatal warnings. + Slice: 50 — scan_app load resilience + + Given Supabase select for Scenario/Eval resume rows raises, + When _scan_item_inner runs, + Then run_all_scanners is still invoked with None resume/prior values. + """ + ### Given + sb = _supabase_chain(data=[{"id": "1"}]) + sb.table.return_value.select.side_effect = RuntimeError("db down") + captured: dict = {} + + def _fake_all(**kwargs): + captured["resume"] = kwargs.get("tessl_scenario_resume") + captured["prior_eval"] = kwargs.get("tessl_prior_eval") + return { + "overall_status": "complete", + "findings": [], + "scanner_rows": [], + "quality_score": None, + } + + ### When + with ( + patch.object(scan_app, "_acquire_target"), + patch.object(scan_app, "run_all_scanners", side_effect=_fake_all), + ): + scan_app._scan_item_inner(sb, "t", "skill", "run-1", "item-1", None) + + ### Then + assert captured["resume"] is None + assert captured["prior_eval"] is None + + def test_given_acquire_fails_when_scan_item_inner_then_mark_failed() -> None: """ Scenario: Acquire failure marks scan_run failed and re-raises. From b80ea0e8403ade44fb389a2d5518ef6a3176c8f8 Mon Sep 17 00:00:00 2001 From: Mani Sarkar Date: Tue, 25 Aug 2026 01:34:17 +0100 Subject: [PATCH 02/12] docs(slice-50): document Eval auto-chain as IMPLEMENTED unit Align architecture, design, STATUS, env/setup guides, and plan trackers with the Tessl Eval adapter without claiming live verification. --- CHANGELOG.md | 5 +++ docs/ARCHITECTURE.md | 4 +-- docs/STATUS.md | 36 +++++++++++-------- docs/design/tessl-5-row-expansion.md | 10 +++--- docs/plan/PROGRESS.md | 10 +++--- docs/plan/TRAIL.md | 6 ++-- .../slice-50-eval-auto-chain.md | 11 +++++- .../adapters/scanner-output-adapters.md | 3 +- docs/user-guide/env-vars.md | 10 +++--- docs/user-guide/supabase-setup.md | 4 +-- 10 files changed, 62 insertions(+), 37 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0c96a2..78d760d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,6 +36,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 avoids delayed native `title=` attributes. ### Added +- Tessl Eval auto-chain scanner row (slice 50): `run_tessl()` emits + `"Tessl: Eval"` as `blocked`, then auto-chains after Scenario Generation when + `/evals/` has scenarios — `tessl eval run --runs 3 -y --json` + + `eval view` poll, `upstream_run_ids`, project create/repair preflight; scenario + re-run marks prior completed Eval `stale` (no cascade). - Tessl Scenario Generation scanner row (slice 49): after Review (Quality), `run_tessl()` runs plugin-path `tessl scenario generate --count 3`, downloads into `/evals/`, stamps `tessl_run_id` / `upstream_run_ids`, and persists diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 3414e64..135708e 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -31,7 +31,7 @@ services — see [prerequisites](./user-guide/prerequisites.md). | **Supabase** | Postgres + Realtime system of record | MVP Live | [supabase-setup](./user-guide/supabase-setup.md) → [env-vars](./user-guide/env-vars.md) | | **Modal** | Isolated scanner sandbox compute | MVP Live | [modal-setup](./user-guide/modal-setup.md) → [env-vars](./user-guide/env-vars.md) | | **Snyk** | Skill/MCP depth scanner | Full scanner coverage | [procurement](./user-guide/env-vars.md#vendor-procurement-quick-steps) | -| **Tessl** | Skill lint (auth-free) + review run quality (`TESSL_TOKEN` + `TESSL_WORKSPACE`) + Scenario Generation (`TESSL_TOKEN` + `.tessl-plugin/plugin.json`; IMPLEMENTED unit, slice 49) | Full scanner coverage | [procurement](./user-guide/env-vars.md#vendor-procurement-quick-steps) | +| **Tessl** | Skill lint (auth-free) + review run quality (`TESSL_TOKEN` + `TESSL_WORKSPACE`) + Scenario Generation (`TESSL_TOKEN` + `.tessl-plugin/plugin.json`; IMPLEMENTED unit, slice 49) + Eval auto-chain (`tessl.json` project link; IMPLEMENTED unit, slice 50) | Full scanner coverage | [procurement](./user-guide/env-vars.md#vendor-procurement-quick-steps) | | **Cisco AI Defense** | Skill Scanner / MCP Scanner / AI Defense APIs | Full scanner coverage | [procurement](./user-guide/env-vars.md#vendor-procurement-quick-steps) | | **Superlinked SIE** | Cheap post-scan triage | Optional tiered router | [tiered-router-setup](./user-guide/tiered-router-setup.md) | | **Alibaba Cloud Model Studio** | Escalation arbitration / triage | Optional tiered router | [tiered-router-setup](./user-guide/tiered-router-setup.md) | @@ -289,7 +289,7 @@ separate `scan_run_scanners` row (slice 46 ✅ persist scan_run `a36cad9f`): [#109](https://github.com/neomatrix369/tripwire/pull/109)) uses `tessl review run quality --json --workspace` and stamps `tessl_run_id` from `tessl review view --last --json`, then seeds in-process -`_TesslIdContext["review_quality"]` for slices 49–51 (GWT-47.5). It is orthogonal to findings and to `risk_score`. **IMPLEMENTED (UI):** slice 48 synthesises "Not Available Yet" sentinel rows for Scenario Generation, Eval, and Security Review when those sources are absent from the scan_run (never stored as placeholders). **IMPLEMENTED (unit, slice 49):** Scenario Generation writes a real `scan_run_scanners` row (`scenario generate` → `download` into `/evals/`, `resume_checkpoint`, mid-scan persist). **DECIDED (runner not implemented):** slices 50–51 write Eval + Security rows — see +`_TesslIdContext["review_quality"]` for slices 49–51 (GWT-47.5). It is orthogonal to findings and to `risk_score`. **IMPLEMENTED (UI):** slice 48 synthesises "Not Available Yet" sentinel rows for Scenario Generation, Eval, and Security Review when those sources are absent from the scan_run (never stored as placeholders). **IMPLEMENTED (unit, slice 49):** Scenario Generation writes a real `scan_run_scanners` row (`scenario generate` → `download` into `/evals/`, `resume_checkpoint`, mid-scan persist). **IMPLEMENTED (unit, slice 50):** Eval starts `blocked`, auto-chains after Scenario Gen when `evals/` is populated (`eval run --runs 3 -y` + `eval view`; stale on scenario re-run; project create/repair preflight). **DECIDED (runner not implemented):** slice 51 writes Security Review rows — see [design/tessl-5-row-expansion.md](./design/tessl-5-row-expansion.md) and slices 49–51; scenario→eval pipeline is generate → download → `eval run` on disk `evals/` (sandbox-populated; host `evals/` is not a vuln-scan input — packing diff --git a/docs/STATUS.md b/docs/STATUS.md index 92714c7..d4fcf86 100644 --- a/docs/STATUS.md +++ b/docs/STATUS.md @@ -94,10 +94,12 @@ Reachable through production entry points / config: — `sandbox/scanners.py` (slice 47 ✅, [PR #109](https://github.com/neomatrix369/tripwire/pull/109); GWT-47.1–47.5) - Dashboard Tessl "Not Available Yet" placeholders — Scanner Outputs always shows - five Tessl capability rows when any Tessl DB row exists; missing - Eval / Review (Security) (and Scenario Generation until a DB row exists) are - UI-only sentinels (`status: not_available_yet`), never written as placeholders, - and counted in the header. MCP scans are unchanged. IMPLEMENTED (unit) + + five Tessl capability rows when any Tessl DB row exists; missing sources among + Scenario Generation / Eval / Review (Security) are UI-only sentinels + (`status: not_available_yet`) when absent from the scan_run (never stored as + placeholders) and counted in the header. Runners 49–50 write Scenario Gen + + Eval rows when they run; Security remains sentinel until slice 51. MCP scans + unchanged. IMPLEMENTED (unit) + VERIFIED (Mock UI 2026-08-24: `safe-changelog-writer` Scanner Outputs (7), five Tessl rows, three NAY pills, no chevron; MCP `SCANNER OUTPUTS (3)` unpadded) — `tripwire-status.js` `mergeTesslCapabilityRows`, `Tripwire.dc.html` @@ -107,7 +109,13 @@ Reachable through production entry points / config: `/evals/`, `upstream_run_ids.review_quality` from ctx, `tessl_run_id` stamp, `resume_checkpoint` + mid-scan persist via `on_scanner_progress`. Missing token → `needs_setup`; missing `.tessl-plugin/plugin.json` → `failed`. - IMPLEMENTED (unit) — `sandbox/scanners.py` / `sandbox/scan_app.py` (slice 49 🔄) + IMPLEMENTED (unit) — `sandbox/scanners.py` / `sandbox/scan_app.py` (slice 49 ✅ #112) +- Tessl Eval auto-chain — `run_tessl()` emits `"Tessl: Eval"` as `blocked` before + Scenario Generation, then auto-chains to `queued`→`running` when generation + completes and `/evals/` has scenarios; `tessl eval run --runs 3 -y + --json` + `eval view` poll; `upstream_run_ids` from ctx; project create/repair + preflight; scenario re-run marks prior completed Eval `stale` (no cascade). + IMPLEMENTED (unit) — `sandbox/scanners.py` / `sandbox/scan_app.py` (slice 50 🔨) - Live dashboard latest-state read path — `dashboard_latest_runs` view (one row per item) + batched child-table fetches in `tripwire-live.js`; replaces global `scan_runs?limit=2000` page that could miss per-item newest runs and PostgREST @@ -235,17 +243,17 @@ Not IMPLEMENTED — no production hook install path or `/tw-*` skills yet. ADR-0 Horizon A exclusion remains in force until Wave H lands and a superseding ADR records the new production entry. -**Wave L — Tessl scenario generation + eval (2026-08-24):** Row 3 -(`"Tessl: Scenario Generation"`) is **IMPLEMENTED (unit, slice 49)** — +**Wave L — Tessl scenario generation + eval (2026-08-25):** Row 3 +(`"Tessl: Scenario Generation"`) is **IMPLEMENTED (unit, slice 49 ✅ #112)** — `scenario generate --count 3` → `scenario download -o /evals/` with `resume_checkpoint` + mid-scan persist. Row 4 -(`"Tessl: Eval"`) remains **DECIDED** in slice 50 — not IMPLEMENTED. Eval -auto-chains from `blocked` when generation completes and `evals/` is -populated. Coverage Gap B (`scenario view `) resolved; Gap C (agent-assisted -generation) open. **IMPLEMENTED (slice 48):** host `evals/` is not a vuln-scan -input — `_pack_local_dir` / `_copy_local` omit root `evals/` when the skill -root has `tessl.json` or `.tessl-plugin/`. Git clone and identity hash still -see on-disk `evals/`. Spec: +(`"Tessl: Eval"`) is **IMPLEMENTED (unit, slice 50)** — starts `blocked`, +auto-chains when generation completes and `evals/` is populated; stale on +scenario re-run; resume via `eval view`. Coverage Gap B (`scenario view `) +resolved; Gap C (agent-assisted generation) open. **IMPLEMENTED (slice 48):** +host `evals/` is not a vuln-scan input — `_pack_local_dir` / `_copy_local` omit +root `evals/` when the skill root has `tessl.json` or `.tessl-plugin/`. Git +clone and identity hash still see on-disk `evals/`. Spec: [design/tessl-5-row-expansion.md](./design/tessl-5-row-expansion.md), [slices 49–50](./plan/slices/12-L-tessl-5-row-expansion/). diff --git a/docs/design/tessl-5-row-expansion.md b/docs/design/tessl-5-row-expansion.md index 0ef90ee..d600968 100644 --- a/docs/design/tessl-5-row-expansion.md +++ b/docs/design/tessl-5-row-expansion.md @@ -1,6 +1,6 @@ # Design: Tessl 5-Row Expansion -**Status**: Schema IMPLEMENTED (slice 45 ✅). Lint adapter IMPLEMENTED (slice 46 ✅ #105). Review Quality run-ID + `_TesslIdContext` seed IMPLEMENTED unit (slice 47 ✅ #109). Rows 3–5 UI sentinels IMPLEMENTED (slice 48). Scenario Generation runner IMPLEMENTED unit (slice 49). Eval + Security runners remain DECIDED / not implemented. +**Status**: Schema IMPLEMENTED (slice 45 ✅). Lint adapter IMPLEMENTED (slice 46 ✅ #105). Review Quality run-ID + `_TesslIdContext` seed IMPLEMENTED unit (slice 47 ✅ #109). Rows 3–5 UI sentinels IMPLEMENTED (slice 48). Scenario Generation runner IMPLEMENTED unit (slice 49 ✅ #112). Eval auto-chain IMPLEMENTED unit (slice 50). Security runner remains DECIDED / not implemented. **Date**: 2026-08-24 **Scope**: Design contract for replacing the single Tessl scanner row with 5 flat capability rows. Current-truth notes below mark what has shipped; remaining rows stay future-state. @@ -358,7 +358,7 @@ The single existing `"Tessl"` row is replaced by 5 flat sibling rows, in this ex | 1 | `Tessl: Lint` | live status (`completed` / `failed` / `unreachable`) | **IMPLEMENTED** (slice 46) — new row; auth-free `tessl skill lint` | | 2 | `Tessl: Review (Quality)` | live status (`completed` / `needs_setup` / …) | **IMPLEMENTED** source string (slice 46); `tessl_run_id` + `_TesslIdContext["review_quality"]` **IMPLEMENTED unit** (slice 47 ✅ #109) via `review view --last --json` | | 3 | `Tessl: Scenario Generation` | live status (`completed` / `failed` / `needs_setup` / `interrupted` / …) | **IMPLEMENTED unit (slice 49)** — `scenario generate` → download into `/evals/`; `resume_checkpoint`; DB row replaces NAY sentinel | -| 4 | `Tessl: Eval` | `Not Available Yet` | **IMPLEMENTED (UI sentinel, slice 48)** — not written to DB | +| 4 | `Tessl: Eval` | live status (`blocked` / `queued` / `running` / `completed` / `stale` / …) | **IMPLEMENTED unit (slice 50)** — auto-chain after Scenario Gen + `evals/`; `upstream_run_ids`; project create/repair preflight; DB row replaces NAY sentinel | | 5 | `Tessl: Review (Security)` | `Not Available Yet` | **IMPLEMENTED (UI sentinel, slice 48)** — not written to DB | The 5 rows appear as a contiguous block where the single `"Tessl"` row used to be. @@ -374,9 +374,9 @@ The dashboard derives the count from the **static constant list** of all 5 expec > **DECIDED (slice 48):** include "Not Available Yet" rows in the Scanner Outputs count (consistent with Cisco credential-absent rows). -### "Not Available Yet" Rendering Rules (Eval + Security until slices 50–51) +### "Not Available Yet" Rendering Rules (Security until slice 51; Scenario Gen / Eval when absent) -The dashboard holds a **static ordered list** of all 5 Tessl `scanner_source` values. For each value absent from the DB rows for the current `scan_run_id`, the `scannersView` map emits a sentinel object with `status: 'not_available_yet'`. +The dashboard holds a **static ordered list** of all 5 Tessl `scanner_source` values. For each value absent from the DB rows for the current `scan_run_id`, the `scannersView` map emits a sentinel object with `status: 'not_available_yet'`. After slices 49–50, Scenario Generation and Eval normally write real rows (including `blocked` Eval before auto-chain); Security Review remains NAY until slice 51. Rendering: - Left accent bar: neutral/muted colour (not the status colours used for active rows). @@ -411,7 +411,7 @@ Extends `scannerStatusColor` and `scannerStatusLabel` in the dashboard JS: ### `tesslQuality` Binding Scope Fix -The existing `tesslQuality` logic is implemented in `tesslInnerQuality` (`tripwire-status.js`) and is scoped to `scanner_source === "Tessl: Review (Quality)"`. Live attaches `output.quality_score` only for that source (`tripwire-live.js`). The quality score badge does not appear on Lint (slice 46 VERIFIED(unit)). Scenario Generation is written by the runner (slice 49); Eval and Security Review rows remain UI sentinels until slices 50–51 write real DB rows (slice 48 VERIFIED(unit)). +The existing `tesslQuality` logic is implemented in `tesslInnerQuality` (`tripwire-status.js`) and is scoped to `scanner_source === "Tessl: Review (Quality)"`. Live attaches `output.quality_score` only for that source (`tripwire-live.js`). The quality score badge does not appear on Lint (slice 46 VERIFIED(unit)). Scenario Generation (slice 49) and Eval (slice 50) are written by the runner; Security Review remains a UI sentinel until slice 51 writes a real DB row (slice 48 VERIFIED(unit) for the merge/sentinel path). --- diff --git a/docs/plan/PROGRESS.md b/docs/plan/PROGRESS.md index 7a62a75..660d888 100644 --- a/docs/plan/PROGRESS.md +++ b/docs/plan/PROGRESS.md @@ -6,7 +6,7 @@ | Wave | Folder | Group | Slices | Outcome | |-----:|--------|-------|--------|---------| | 11 | [`11-K-…`](slices/11-K-docs-ux-plain-language/) | **K — Docs UX plain language + compaction** | **44** | 🔀 | -| 12 | [`12-L-…`](slices/12-L-tessl-5-row-expansion/) | **L — Tessl 5-row expansion** | 45 ✅ · 46 ✅ · **47** ✅ · **48** 🔨 · 49–52 📋 | 45 ✅ · 46 ✅ · 47 ✅ · 48 🔨 | +| 12 | [`12-L-…`](slices/12-L-tessl-5-row-expansion/) | **L — Tessl 5-row expansion** | 45 ✅ · 46 ✅ · 47 ✅ · 48 ✅ · 49 ✅ · **50** 🔨 · 51–52 📋 | 45–49 ✅ · **50** 🔨 | | 9 | [`09-I-…`](slices/09-I-landing-intro-restyle/) | **I — Landing Intro + Visual Refresh** | 41 ✅ · 43 | 41 ✅ · 43 ✅ | | 10 | [`10-J-…`](slices/10-J-dashboard-data-quality/) | **J — Dashboard Data Quality Fixes** | 42 | A1–A13 ✅ | | 1 | [`01-A-…`](slices/01-A-live-path-gwt/) | **A — Live path + GWT** | 1 → 2 → 3 · 4 | ✅ · 4 📦 | @@ -27,7 +27,7 @@ | Order | Wave | # | Slice | MoSCoW | Status | |------:|-----:|---|-------|--------|--------| | 0 (docs) | K | **44** | **Docs UX plain language + compaction** | Must | 🔀 ON BRANCH · [#99](https://github.com/neomatrix369/tripwire/pull/99) · GWT-44.1–44.8 · documentarist APPROVED WITH FOLLOW-ON (DIVIO targets pending) · `slice/44-docs-ux-plain-language` | -| 0 (active) | L | **49** | **Tessl: Scenario Generation + Resume Checkpoint (Row 3)** | Should | 🔄 IN PROGRESS · quality-gates green · `slice/49-scenario-generation` | +| 0 (active) | L | **50** | **Tessl: Eval + Scenario→Eval Auto-Chain (Row 4)** | Should | 🔨 IN PROGRESS · `slice/50-eval-auto-chain` | | 0 (delta) | J | 42 | Dashboard Data Quality — Tessl quality UX + tooltips + plain labels (A9–A13) | Must | ✅ PASSED · [#98](https://github.com/neomatrix369/tripwire/pull/98) | | 1 | G | 18 | CLI Operator Evidence Contracts | Must | 📋 PLANNED | | 2 | G | 19 | Sandbox Persistence State Contract | Must | 📋 PLANNED | @@ -65,7 +65,7 @@ shared code area: slice 18 does not start until the H-wave subcommand lands | — | L | 47 | Tessl: Review (Quality) Split (Row 2) | Must | ✅ PASSED · [#109](https://github.com/neomatrix369/tripwire/pull/109) | | — | L | 48 | "Not Available Yet" Placeholder Rows (Rows 3–5) | Must | 🔨 IN PROGRESS · after-checks green · awaiting commit/PR · `slice/48-not-available-yet-ui` | | — | L | 49 | Tessl: Scenario Generation + Resume Checkpoint (Row 3) | Should | 📋 PLANNED | -| — | L | 50 | Tessl: Eval + Auto-Chain (Row 4) | Should | 📋 PLANNED | +| — | L | 50 | Tessl: Eval + Auto-Chain (Row 4) | Should | 🔨 IN PROGRESS | | — | L | 51 | Tessl: Review (Security) (Row 5) | Could | 📋 PLANNED | | — | L | 52 | ID Lineage Cross-Reads + UI Side-by-Side | Could | 📋 PLANNED | @@ -175,8 +175,8 @@ Plan", 2026-08-15). Governance blocks *merge*, not prototyping. | 46 | [slice-46-lint-adapter](slices/12-L-tessl-5-row-expansion/slice-46-lint-adapter.md) | Must | ✅ PASSED ([#105](https://github.com/neomatrix369/tripwire/pull/105)) | 2026-08-24 | 2026-08-24 | ~4 min | | 47 | [slice-47-review-quality-split](slices/12-L-tessl-5-row-expansion/slice-47-review-quality-split.md) | Must | ✅ PASSED ([#109](https://github.com/neomatrix369/tripwire/pull/109)) | 2026-08-24 | 2026-08-24 | ~4 min | | 48 | [slice-48-not-available-yet-ui](slices/12-L-tessl-5-row-expansion/slice-48-not-available-yet-ui.md) | Must | 🔨 IN PROGRESS | 2026-08-24 | — | ~3 min | -| 49 | [slice-49-scenario-generation](slices/12-L-tessl-5-row-expansion/slice-49-scenario-generation.md) | Should | 🔄 IN PROGRESS | — | — | ~6 min | -| 50 | [slice-50-eval-auto-chain](slices/12-L-tessl-5-row-expansion/slice-50-eval-auto-chain.md) | Should | 📋 PLANNED | — | — | ~5 min | +| 49 | [slice-49-scenario-generation](slices/12-L-tessl-5-row-expansion/slice-49-scenario-generation.md) | Should | ✅ PASSED ([#112](https://github.com/neomatrix369/tripwire/pull/112)) | 2026-08-24 | 2026-08-25 | ~6 min | +| 50 | [slice-50-eval-auto-chain](slices/12-L-tessl-5-row-expansion/slice-50-eval-auto-chain.md) | Should | 🔨 IN PROGRESS | 2026-08-25 | — | ~5 min | | 51 | [slice-51-review-security](slices/12-L-tessl-5-row-expansion/slice-51-review-security.md) | Could | 📋 PLANNED | — | — | ~3 min | | 52 | [slice-52-id-lineage-wiring](slices/12-L-tessl-5-row-expansion/slice-52-id-lineage-wiring.md) | Could | 📋 PLANNED | — | — | ~5 min | diff --git a/docs/plan/TRAIL.md b/docs/plan/TRAIL.md index a704cb7..8e92aff 100644 --- a/docs/plan/TRAIL.md +++ b/docs/plan/TRAIL.md @@ -131,7 +131,7 @@ Groups are ordered by when the wave ran (or will run), not by slice number. | 8 | [slice-8-scanner-skill-parse-fixtures](slices/05-E-ship-path-coverage/slice-8-scanner-skill-parse-fixtures.md) | Scanner Skill Parse Fixtures (Delta) | Must | ✅ | 7 | — | ~4 min | | 9 | [slice-9-scanner-snyk-tessl-parse-fixtures](slices/05-E-ship-path-coverage/slice-9-scanner-snyk-tessl-parse-fixtures.md) | Snyk / Tessl Parse Fixtures (Delta) | Should | 📦 closed (subsumed) | 11 | SUBSUMED by 11 | ~4 min | | 10 | [slice-10-scan-item-inner-characterization](slices/05-E-ship-path-coverage/slice-10-scan-item-inner-characterization.md) | scan_item_inner Characterization (Delta) | Should | 📦 closed (subsumed) | 11 | SUBSUMED by 11 | ~4 min | -| 11 | [slice-11-python-ship-path-coverage-95](slices/05-E-ship-path-coverage/slice-11-python-ship-path-coverage-95.md) | Python Ship-Path Coverage ≥95% | Must | ✅ | 8 (9,10 Should) | — | ~5 min | +| 11 | [slice-11-python-ship-path-coverage-95](slices/05-E-ship-path-coverage/slice-11-python-ship-path-coverage-95.md) | Python Ship-Path Coverage ≥95% (+ Snyk v0.6 delta) | Must | 🔀 delta | 8 (9,10 Should) | — | ~5 min | | 12 | [slice-12-cli-coverage-gate-95](slices/05-E-ship-path-coverage/slice-12-cli-coverage-gate-95.md) | CLI Coverage Gate ≥95% (Delta) | Must | ✅ | 6 | — | ~4 min | | 13 | [slice-13-live-acl-coverage-gate-95](slices/05-E-ship-path-coverage/slice-13-live-acl-coverage-gate-95.md) | Live ACL Coverage Gate ≥95% (Delta) | Must | ✅ | 2,3 | — | ~4 min | | 14 | [slice-14-coverage-status-docs-sync](slices/05-E-ship-path-coverage/slice-14-coverage-status-docs-sync.md) | Coverage Status + Docs Sync (Delta) | Must | ✅ | 11,12,13 | #39 | ~3 min | @@ -285,8 +285,8 @@ See `docs/design/tessl-5-row-expansion.md § Open Questions`. | 46 | [slice-46-lint-adapter](slices/12-L-tessl-5-row-expansion/slice-46-lint-adapter.md) | Tessl: Lint Adapter (Row 1) | Must | ✅ | 45 | [#105](https://github.com/neomatrix369/tripwire/pull/105) | ~4 min | | 47 | [slice-47-review-quality-split](slices/12-L-tessl-5-row-expansion/slice-47-review-quality-split.md) | Tessl: Review (Quality) Split + `tesslQuality` Scope Fix (Row 2) | Must | ✅ | 45, 46 | [#109](https://github.com/neomatrix369/tripwire/pull/109) | ~4 min | | 48 | [slice-48-not-available-yet-ui](slices/12-L-tessl-5-row-expansion/slice-48-not-available-yet-ui.md) | "Not Available Yet" Placeholder Rows (Rows 3–5) | Must | 🔨 | 47 | — | ~3 min | -| 49 | [slice-49-scenario-generation](slices/12-L-tessl-5-row-expansion/slice-49-scenario-generation.md) | Tessl: Scenario Generation + Resume Checkpoint (Row 3) | Should | 🔄 | 47, 48 | — | ~6 min | -| 50 | [slice-50-eval-auto-chain](slices/12-L-tessl-5-row-expansion/slice-50-eval-auto-chain.md) | Tessl: Eval + Scenario→Eval Auto-Chain (Row 4) | Should | 📋 | 49 | — | ~5 min | +| 49 | [slice-49-scenario-generation](slices/12-L-tessl-5-row-expansion/slice-49-scenario-generation.md) | Tessl: Scenario Generation + Resume Checkpoint (Row 3) | Should | ✅ | 47, 48 | [#112](https://github.com/neomatrix369/tripwire/pull/112) | ~6 min | +| 50 | [slice-50-eval-auto-chain](slices/12-L-tessl-5-row-expansion/slice-50-eval-auto-chain.md) | Tessl: Eval + Scenario→Eval Auto-Chain (Row 4) | Should | 🔨 | 49 | — | ~5 min | | 51 | [slice-51-review-security](slices/12-L-tessl-5-row-expansion/slice-51-review-security.md) | Tessl: Review (Security) Adapter (Row 5) | Could | 📋 | 47 | — | ~3 min | | 52 | [slice-52-id-lineage-wiring](slices/12-L-tessl-5-row-expansion/slice-52-id-lineage-wiring.md) | ID Lineage Cross-Reads + UI Side-by-Side Findings | Could | 📋 | 49, 50, 51; Gap C UI-only | — | ~5 min | diff --git a/docs/plan/slices/12-L-tessl-5-row-expansion/slice-50-eval-auto-chain.md b/docs/plan/slices/12-L-tessl-5-row-expansion/slice-50-eval-auto-chain.md index 5839d65..504cf9f 100644 --- a/docs/plan/slices/12-L-tessl-5-row-expansion/slice-50-eval-auto-chain.md +++ b/docs/plan/slices/12-L-tessl-5-row-expansion/slice-50-eval-auto-chain.md @@ -3,7 +3,7 @@ **Wave**: 12-L **MoSCoW**: Should **Depends on**: 49 -**Status**: 📋 PLANNED +**Status**: 🔨 IN PROGRESS **Read time**: ~5 min ## Context @@ -112,3 +112,12 @@ Design reference: `docs/design/tessl-5-row-expansion.md § (b) auto-chain, § (b `coverage_pct`: target ≥ 80% for new eval + auto-chain code path `complexity_tool`: ruff/radon on `sandbox/scanners.py` `doc_audit`: design doc § (b) auto-chain and Stale — mark as implemented + +## Implementation notes (2026-08-25) + +- `_run_tessl_eval` + `_resolve_tessl_eval_row` in `sandbox/scanners.py` +- `TESSL_SOURCES` includes `"Tessl: Eval"`; `run_tessl` returns 4 rows +- `scan_app` loads prior Eval row for stale/resume; progress persists blocked→queued→running +- Unit tests: GWT-50.0–50.6 in `test_scanners_status.py` +- `./scripts/quality-gates.sh` passed (coverage 95.4%) +- **nw-review**: APPROVED ([reviewer](65f6574d-325d-4809-910a-473ec52e6b6a)) — no blockers diff --git a/docs/research/adapters/scanner-output-adapters.md b/docs/research/adapters/scanner-output-adapters.md index eef1f9d..e973886 100644 --- a/docs/research/adapters/scanner-output-adapters.md +++ b/docs/research/adapters/scanner-output-adapters.md @@ -204,9 +204,10 @@ Product note that file `--output` was flaky on a tested build: re-test on pin; p | Item | Status | |---|---| -| Target | DECIDED — `items.quality_score` only (Review Quality row; Lint / Scenario Generation do not write this axis) | +| Target | DECIDED — `items.quality_score` only (Review Quality row; Lint / Scenario Generation / Eval do not write this axis) | | Capture | IMPLEMENTED (slice 47) — `tessl review run quality --json --workspace`; run ID via `review view --last --json` (fallback: run JSON `id`/`runId`/`run_id`). Requires `TESSL_TOKEN` + `TESSL_WORKSPACE`. | | Scenario Generation | IMPLEMENTED unit (slice 49) — separate `scan_run_scanners` row; plugin-path `scenario generate` / `download` into `/evals/`; no `quality_score` write | +| Eval | IMPLEMENTED unit (slice 50) — `"Tessl: Eval"` row starts `blocked`, auto-chains after Scenario Gen when `evals/` is non-empty; `eval run --runs 3 -y --json` + `eval view`; `upstream_run_ids`; project create/repair; stale on scenario re-run; no `quality_score` write | | Raw | Store JSON in Storage for audit | ### References diff --git a/docs/user-guide/env-vars.md b/docs/user-guide/env-vars.md index bcbf795..dad9042 100644 --- a/docs/user-guide/env-vars.md +++ b/docs/user-guide/env-vars.md @@ -10,7 +10,9 @@ Start here: [QUICKSTART](../../QUICKSTART.md) · Hub: [docs/README](../README.md > when you want full scanner coverage. Missing Snyk/Cisco keys report > `skipped_missing_credential`. Missing `TESSL_TOKEN` or `TESSL_WORKSPACE` still > runs Lint (auth-free) and marks Review (Quality) `needs_setup`; missing -> `TESSL_TOKEN` also marks Scenario Generation `needs_setup` — not a complete +> `TESSL_TOKEN` also marks Scenario Generation `needs_setup`; Eval stays +> `blocked` until Scenario Gen completes (then `needs_setup` if token/workspace +> or `tessl.json` project link cannot be established) — not a complete > “all clear.” > > For full Live coverage, provision all five vendors before `cp .env.example .env`. @@ -77,8 +79,8 @@ not a second environment-variable schema. | `MCP_SCANNER_LLM_MODEL` | MCP LLM routing | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | | `MCP_SCANNER_LLM_BASE_URL` | Custom MCP LLM endpoint | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | | `MCP_SCANNER_LLM_API_VERSION` | Azure-style APIs | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | -| `TESSL_TOKEN` | Tessl Review (Quality) on Modal/CI (Lint is auth-free) | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | -| `TESSL_WORKSPACE` | Tessl workspace for `tessl review run … --json` (required with `--json`; Review is `needs_setup` if absent) | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | +| `TESSL_TOKEN` | Tessl Review / Scenario Gen / Eval on Modal/CI (Lint is auth-free) | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | +| `TESSL_WORKSPACE` | Tessl workspace for `tessl review run … --json` and Eval project create (Review/Eval `needs_setup` if absent) | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | ## Tier C — Full depth (paid Cisco AI Defense) @@ -160,7 +162,7 @@ vendor to the keys you need in `.env` and where to get them. | **Supabase** (platform — MVP) | `SUPABASE_URL`, `SUPABASE_ANON_KEY`, `SUPABASE_SERVICE_ROLE_KEY`, `SUPABASE_DB_URL` | [supabase-setup](./supabase-setup.md) | | **Modal** (platform — MVP) | `MODAL_TOKEN_ID`, `MODAL_TOKEN_SECRET` (blank if interactive only) | [modal-setup](./modal-setup.md) | | **Snyk** (scanner) | `SNYK_TOKEN` | [app.snyk.io](https://app.snyk.io) → Settings → API Tokens | -| **Tessl** (scanner) | `TESSL_TOKEN`, `TESSL_WORKSPACE` | [tessl.io](https://tessl.io) → workspace → API key. Lint (`tessl skill lint`) is auth-free; token **and** workspace gate Review (Quality) (`tessl review run quality --json --workspace`; `needs_setup` if either is absent). Scenario Generation (slice 49) requires `TESSL_TOKEN` and `.tessl-plugin/plugin.json` (plugin-path `scenario generate` / `download`; no `--workspace` on that path). **DECIDED (slice 50, not shipped):** Eval will also require token + linked Tessl project (`tessl.json`) — see [tessl-5-row-expansion](../design/tessl-5-row-expansion.md). | +| **Tessl** (scanner) | `TESSL_TOKEN`, `TESSL_WORKSPACE` | [tessl.io](https://tessl.io) → workspace → API key. Lint (`tessl skill lint`) is auth-free; token **and** workspace gate Review (Quality) (`tessl review run quality --json --workspace`; `needs_setup` if either is absent). Scenario Generation (slice 49) requires `TESSL_TOKEN` and `.tessl-plugin/plugin.json` (plugin-path `scenario generate` / `download`; no `--workspace` on that path). Eval (slice 50) auto-chains after Scenario Gen when `evals/` is populated; requires token, workspace, and a linked Tessl project (`tessl.json` — adapter runs `project create` / `project repair`) — see [tessl-5-row-expansion](../design/tessl-5-row-expansion.md). | | **Cisco Skill / MCP LLM** (scanner Tier B) | `SKILL_SCANNER_LLM_API_KEY`, `SKILL_SCANNER_LLM_MODEL`, `SKILL_SCANNER_LLM_PROVIDER`, `SKILL_SCANNER_LLM_BASE_URL`; `MCP_SCANNER_LLM_API_KEY`, `MCP_SCANNER_LLM_MODEL`, `MCP_SCANNER_LLM_BASE_URL` | Any OpenAI-compatible or Azure LLM — not the same as AI Defense cloud keys below | | **Cisco AI Defense** (scanner Tier C) | `AI_DEFENSE_API_KEY`, `MCP_SCANNER_API_KEY`; optional `AI_DEFENSE_API_URL`, `MCP_SCANNER_ENDPOINT` | [developer.cisco.com](https://developer.cisco.com) → AI Defense | | **Ossprey** (malware — access OPEN/pending) | `OSSPREY_API_KEY` (`ospy_…`) | Access not yet available — leave blank; adapter reports `skipped_missing_credential` | diff --git a/docs/user-guide/supabase-setup.md b/docs/user-guide/supabase-setup.md index 80e4d90..889d4ae 100644 --- a/docs/user-guide/supabase-setup.md +++ b/docs/user-guide/supabase-setup.md @@ -87,8 +87,8 @@ truncate responses. Symptoms: that scan run - Card shows a recent **Last scan** time but no scanner list - New Tessl rows (`Tessl: Lint`, `Tessl: Review (Quality)`, - `Tessl: Scenario Generation`) missing while older `"Tessl"` rows still appear - on other items + `Tessl: Scenario Generation`, `Tessl: Eval`) missing while older `"Tessl"` rows + still appear on other items This limit is **not** in `.env` or repo config — it is a **project setting** on Supabase. From 3dc331c9452575bb25f018be54b7eaa73d2ebefb Mon Sep 17 00:00:00 2001 From: Mani Sarkar Date: Tue, 25 Aug 2026 01:48:07 +0100 Subject: [PATCH 03/12] fix(slice-50): pass --workspace on tessl scenario generate Live Tessl CLI requires --workspace for plugin-path scenario generate; gate Scenario Gen on TESSL_WORKSPACE instead of failing at the CLI. --- docs/design/tessl-5-row-expansion.md | 4 ++-- docs/user-guide/env-vars.md | 12 +++++------ sandbox/scanners.py | 29 +++++++++++++++++++++------ sandbox/tests/test_scanners_status.py | 15 +++++++++++--- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/docs/design/tessl-5-row-expansion.md b/docs/design/tessl-5-row-expansion.md index d600968..1284bdd 100644 --- a/docs/design/tessl-5-row-expansion.md +++ b/docs/design/tessl-5-row-expansion.md @@ -255,7 +255,7 @@ Verified against [Tessl CLI reference](https://docs.tessl.io/reference/cli-comma 7. Security (slice 51) → upstream_run_ids={review_quality}; review run security; stamp tessl_run_id ``` -**Not supported by Tessl CLI**: passing `gen_id` to `eval run`. Eval always consumes on-disk scenarios. **`--workspace`** on `scenario generate` is for repo mode (`org/repo --commits …`), not plugin-path generation. +**Not supported by Tessl CLI**: passing `gen_id` to `eval run`. Eval always consumes on-disk scenarios. **`--workspace`** is **required** on plugin-path `scenario generate` (live CLI: `tessl scenario generate ./my-plugin --workspace acme`); Tripwire passes `TESSL_WORKSPACE`. ### ID carry-forward contract (MUST — slices 47–51) @@ -328,7 +328,7 @@ Each feature that reads from a prior feature's persisted state does so by: **What is read**: Same Quality Review `tessl_run_id` lookup; `tessl review view --json` to retrieve Quality findings. -**Threading findings into scenario generation**: The **plain CLI form** (`tessl scenario generate [--count N]`) has no context-injection flag. `--workspace` applies to **repo** generation (`org/repo --commits …`), not plugin-path generation. To thread Quality findings into scenario generation, the **agent-assisted path** (`tessl install tessl-labs/tessl-skill-eval-scenarios`) is the only documented channel. +**Threading findings into scenario generation**: The **plain CLI form** (`tessl scenario generate --workspace [--count N]`) has no context-injection flag for Quality findings. To thread Quality findings into scenario generation, the **agent-assisted path** (`tessl install tessl-labs/tessl-skill-eval-scenarios`) is the only documented channel. **Caveat — agent-assisted path in headless sandbox**: This path is designed around an interactive agent prompt. Whether it can be scripted from Tripwire's headless Modal sandbox orchestration is **unverified** (Coverage Gap C). Until verified, the plain CLI form is used for scenario generation, and the Quality findings are surfaced in the UI as context for human review of the generated scenarios rather than injected into the CLI call. diff --git a/docs/user-guide/env-vars.md b/docs/user-guide/env-vars.md index dad9042..2556018 100644 --- a/docs/user-guide/env-vars.md +++ b/docs/user-guide/env-vars.md @@ -10,10 +10,10 @@ Start here: [QUICKSTART](../../QUICKSTART.md) · Hub: [docs/README](../README.md > when you want full scanner coverage. Missing Snyk/Cisco keys report > `skipped_missing_credential`. Missing `TESSL_TOKEN` or `TESSL_WORKSPACE` still > runs Lint (auth-free) and marks Review (Quality) `needs_setup`; missing -> `TESSL_TOKEN` also marks Scenario Generation `needs_setup`; Eval stays -> `blocked` until Scenario Gen completes (then `needs_setup` if token/workspace -> or `tessl.json` project link cannot be established) — not a complete -> “all clear.” +> `TESSL_TOKEN` or `TESSL_WORKSPACE` also marks Scenario Generation `needs_setup`; +> Eval stays `blocked` until Scenario Gen completes (then `needs_setup` if +> token/workspace or `tessl.json` project link cannot be established) — not a +> complete “all clear.” > > For full Live coverage, provision all five vendors before `cp .env.example .env`. > @@ -80,7 +80,7 @@ not a second environment-variable schema. | `MCP_SCANNER_LLM_BASE_URL` | Custom MCP LLM endpoint | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | | `MCP_SCANNER_LLM_API_VERSION` | Azure-style APIs | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | | `TESSL_TOKEN` | Tessl Review / Scenario Gen / Eval on Modal/CI (Lint is auth-free) | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | -| `TESSL_WORKSPACE` | Tessl workspace for `tessl review run … --json` and Eval project create (Review/Eval `needs_setup` if absent) | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | +| `TESSL_WORKSPACE` | Tessl workspace for `tessl review run … --workspace`, `scenario generate … --workspace`, and Eval project create (Review/Scenario/Eval `needs_setup` if absent) | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | ## Tier C — Full depth (paid Cisco AI Defense) @@ -162,7 +162,7 @@ vendor to the keys you need in `.env` and where to get them. | **Supabase** (platform — MVP) | `SUPABASE_URL`, `SUPABASE_ANON_KEY`, `SUPABASE_SERVICE_ROLE_KEY`, `SUPABASE_DB_URL` | [supabase-setup](./supabase-setup.md) | | **Modal** (platform — MVP) | `MODAL_TOKEN_ID`, `MODAL_TOKEN_SECRET` (blank if interactive only) | [modal-setup](./modal-setup.md) | | **Snyk** (scanner) | `SNYK_TOKEN` | [app.snyk.io](https://app.snyk.io) → Settings → API Tokens | -| **Tessl** (scanner) | `TESSL_TOKEN`, `TESSL_WORKSPACE` | [tessl.io](https://tessl.io) → workspace → API key. Lint (`tessl skill lint`) is auth-free; token **and** workspace gate Review (Quality) (`tessl review run quality --json --workspace`; `needs_setup` if either is absent). Scenario Generation (slice 49) requires `TESSL_TOKEN` and `.tessl-plugin/plugin.json` (plugin-path `scenario generate` / `download`; no `--workspace` on that path). Eval (slice 50) auto-chains after Scenario Gen when `evals/` is populated; requires token, workspace, and a linked Tessl project (`tessl.json` — adapter runs `project create` / `project repair`) — see [tessl-5-row-expansion](../design/tessl-5-row-expansion.md). | +| **Tessl** (scanner) | `TESSL_TOKEN`, `TESSL_WORKSPACE` | [tessl.io](https://tessl.io) → workspace → API key. Lint (`tessl skill lint`) is auth-free; token **and** workspace gate Review (Quality) (`tessl review run quality --json --workspace`; `needs_setup` if either is absent). Scenario Generation requires `TESSL_TOKEN`, `TESSL_WORKSPACE` (`scenario generate … --workspace`), and `.tessl-plugin/plugin.json`. Eval (slice 50) auto-chains after Scenario Gen when `evals/` is populated; requires token, workspace, and a linked Tessl project (`tessl.json` — adapter runs `project create` / `project repair`) — see [tessl-5-row-expansion](../design/tessl-5-row-expansion.md). | | **Cisco Skill / MCP LLM** (scanner Tier B) | `SKILL_SCANNER_LLM_API_KEY`, `SKILL_SCANNER_LLM_MODEL`, `SKILL_SCANNER_LLM_PROVIDER`, `SKILL_SCANNER_LLM_BASE_URL`; `MCP_SCANNER_LLM_API_KEY`, `MCP_SCANNER_LLM_MODEL`, `MCP_SCANNER_LLM_BASE_URL` | Any OpenAI-compatible or Azure LLM — not the same as AI Defense cloud keys below | | **Cisco AI Defense** (scanner Tier C) | `AI_DEFENSE_API_KEY`, `MCP_SCANNER_API_KEY`; optional `AI_DEFENSE_API_URL`, `MCP_SCANNER_ENDPOINT` | [developer.cisco.com](https://developer.cisco.com) → AI Defense | | **Ossprey** (malware — access OPEN/pending) | `OSSPREY_API_KEY` (`ospy_…`) | Access not yet available — leave blank; adapter reports `skipped_missing_credential` | diff --git a/sandbox/scanners.py b/sandbox/scanners.py index ac3802f..db694e6 100644 --- a/sandbox/scanners.py +++ b/sandbox/scanners.py @@ -755,7 +755,12 @@ def _parse_scenario_gen_id(parsed) -> str | None: return _parse_tessl_run_id(parsed) -def _tessl_scenario_generate_argv(workdir: str, count: int = _TESSL_SCENARIO_COUNT) -> list[str]: +def _tessl_scenario_generate_argv( + workdir: str, + workspace: str, + count: int = _TESSL_SCENARIO_COUNT, +) -> list[str]: + """Build ``tessl scenario generate`` argv (plugin path requires ``--workspace``).""" return [ "npx", "--yes", @@ -763,6 +768,8 @@ def _tessl_scenario_generate_argv(workdir: str, count: int = _TESSL_SCENARIO_COU "scenario", "generate", workdir, + "--workspace", + workspace, "--count", str(count), ] @@ -869,13 +876,21 @@ def _scenario_checkpoint_row( def _scenario_gen_preflight(workdir: str, upstream_run_ids: dict) -> dict | None: - """Return an early terminal row when token/plugin prerequisites fail.""" + """Return an early terminal row when token/workspace/plugin prerequisites fail.""" if not os.environ.get("TESSL_TOKEN"): skipped: dict = _skipped( _TESSL_SCENARIO_SOURCE, reason="needs_setup", detail="TESSL_TOKEN required" ) skipped["upstream_run_ids"] = upstream_run_ids return skipped + if not (os.environ.get("TESSL_WORKSPACE") or "").strip(): + skipped = _skipped( + _TESSL_SCENARIO_SOURCE, + reason="needs_setup", + detail="TESSL_WORKSPACE required for scenario generate --workspace", + ) + skipped["upstream_run_ids"] = upstream_run_ids + return skipped if _has_tessl_plugin_manifest(workdir): return None return { @@ -918,7 +933,8 @@ def _ensure_scenario_generated( row["checks_run"] = count_hint return gen_id, None - code, out, err = _run(_tessl_scenario_generate_argv(workdir)) + workspace = (os.environ.get("TESSL_WORKSPACE") or "").strip() + code, out, err = _run(_tessl_scenario_generate_argv(workdir, workspace)) console = _build_console(out, err) if console: consoles.append(console) @@ -1509,9 +1525,10 @@ def run_tessl( Lint is synchronous and never requires TESSL_TOKEN; it always runs when npx is available. Review Quality requires TESSL_TOKEN and TESSL_WORKSPACE; its row transitions to needs_setup when either is absent. Scenario Generation - requires TESSL_TOKEN and ``.tessl-plugin/plugin.json``. Eval starts ``blocked`` - and auto-chains after Scenario Generation completes with scenarios in - ``evals/`` (first run only; re-runs mark prior completed Eval as ``stale``). + requires TESSL_TOKEN, TESSL_WORKSPACE (``--workspace`` on generate), and + ``.tessl-plugin/plugin.json``. Eval starts ``blocked`` and auto-chains after + Scenario Generation completes with scenarios in ``evals/`` (first run only; + re-runs mark prior completed Eval as ``stale``). Returns (quality_score, [lint_row, review_row, scenario_row, eval_row]). quality_score is None when Review Quality did not complete successfully. diff --git a/sandbox/tests/test_scanners_status.py b/sandbox/tests/test_scanners_status.py index 10f0a7e..9af74f2 100644 --- a/sandbox/tests/test_scanners_status.py +++ b/sandbox/tests/test_scanners_status.py @@ -802,12 +802,13 @@ def test_run_tessl_falls_back_to_run_json_id_when_view_last_fails() -> None: def test_run_tessl_without_workspace_emits_review_needs_setup() -> None: """ - Scenario: TESSL_WORKSPACE absent — Review (Quality) is needs_setup. + Scenario: TESSL_WORKSPACE absent — Review (Quality) and Scenario Gen are needs_setup. Slice: 47 — GWT-47.3 needs_setup for missing review config + Slice: 49/50 — scenario generate also requires --workspace Given TESSL_TOKEN is set and TESSL_WORKSPACE is absent, When run_tessl is called, - Then Review (Quality) status is needs_setup and no review subprocess runs. + Then Review (Quality) and Scenario Generation are needs_setup and no review/generate runs. """ ### Given ran: list[list[str]] = [] @@ -828,7 +829,10 @@ def _record(cmd, timeout=None): assert score is None assert rows[1]["scanner_source"] == "Tessl: Review (Quality)" assert rows[1]["status"] == "needs_setup" + assert rows[2]["scanner_source"] == "Tessl: Scenario Generation" + assert rows[2]["status"] == "needs_setup" assert all(c[3:5] != ["review", "run"] for c in ran) + assert all(c[3:5] != ["scenario", "generate"] for c in ran) def test_given_quality_review_completes_when_run_tessl_then_ctx_review_quality_is_stamped_id() -> ( @@ -1005,7 +1009,8 @@ def _run(cmd, timeout=None, cwd=None): if cmd[3:5] == ["scenario", "generate"]: assert "--count" in cmd and "3" in cmd assert workdir in cmd - assert "--workspace" not in cmd + assert "--workspace" in cmd + assert cmd[cmd.index("--workspace") + 1] == "engteam" return 0, '{"id": "gen_abc123", "status": "completed", "scenarioCount": 3}', "" if cmd[3:5] == ["scenario", "view"]: return 0, '{"id": "gen_abc123", "status": "completed", "scenarioCount": 3}', "" @@ -1390,6 +1395,10 @@ def test_parse_scenario_gen_id_and_status_from_json_shapes() -> None: assert scanners._parse_scenario_status(None) is None assert scanners._parse_scenario_count("nope") is None assert scanners._tessl_scenario_view_argv(None)[-3:] == ["--last", "--mine", "--json"] + gen_argv = scanners._tessl_scenario_generate_argv("/plugin", "acme", count=3) + assert gen_argv[3:6] == ["scenario", "generate", "/plugin"] + assert gen_argv[gen_argv.index("--workspace") + 1] == "acme" + assert gen_argv[gen_argv.index("--count") + 1] == "3" def test_given_generate_timeout_when_scenario_runs_then_interrupted_with_checkpoint( From 3b6ec6d5b805d8d1e90600966f7c59b0c22901a9 Mon Sep 17 00:00:00 2001 From: Mani Sarkar Date: Tue, 25 Aug 2026 01:53:59 +0100 Subject: [PATCH 04/12] fix(slice-50): resolve Tessl --workspace via whoami/list TESSL_WORKSPACE is an optional override; default to the personal workspace from tessl whoami + workspace list (usually the username). --- docs/ARCHITECTURE.md | 2 +- docs/design/tessl-5-row-expansion.md | 2 +- docs/user-guide/env-vars.md | 16 +-- sandbox/scanners.py | 161 ++++++++++++++++++++++---- sandbox/tests/test_scanners_status.py | 87 +++++++++++++- 5 files changed, 229 insertions(+), 39 deletions(-) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 135708e..73c6d1f 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -31,7 +31,7 @@ services — see [prerequisites](./user-guide/prerequisites.md). | **Supabase** | Postgres + Realtime system of record | MVP Live | [supabase-setup](./user-guide/supabase-setup.md) → [env-vars](./user-guide/env-vars.md) | | **Modal** | Isolated scanner sandbox compute | MVP Live | [modal-setup](./user-guide/modal-setup.md) → [env-vars](./user-guide/env-vars.md) | | **Snyk** | Skill/MCP depth scanner | Full scanner coverage | [procurement](./user-guide/env-vars.md#vendor-procurement-quick-steps) | -| **Tessl** | Skill lint (auth-free) + review run quality (`TESSL_TOKEN` + `TESSL_WORKSPACE`) + Scenario Generation (`TESSL_TOKEN` + `.tessl-plugin/plugin.json`; IMPLEMENTED unit, slice 49) + Eval auto-chain (`tessl.json` project link; IMPLEMENTED unit, slice 50) | Full scanner coverage | [procurement](./user-guide/env-vars.md#vendor-procurement-quick-steps) | +| **Tessl** | Skill lint (auth-free) + review / scenario / eval (`TESSL_TOKEN`; `--workspace` from optional `TESSL_WORKSPACE` or `whoami`+`workspace list`) + Scenario Generation (`.tessl-plugin/plugin.json`; IMPLEMENTED unit, slice 49) + Eval auto-chain (`tessl.json` project link; IMPLEMENTED unit, slice 50) | Full scanner coverage | [procurement](./user-guide/env-vars.md#vendor-procurement-quick-steps) | | **Cisco AI Defense** | Skill Scanner / MCP Scanner / AI Defense APIs | Full scanner coverage | [procurement](./user-guide/env-vars.md#vendor-procurement-quick-steps) | | **Superlinked SIE** | Cheap post-scan triage | Optional tiered router | [tiered-router-setup](./user-guide/tiered-router-setup.md) | | **Alibaba Cloud Model Studio** | Escalation arbitration / triage | Optional tiered router | [tiered-router-setup](./user-guide/tiered-router-setup.md) | diff --git a/docs/design/tessl-5-row-expansion.md b/docs/design/tessl-5-row-expansion.md index 1284bdd..e4cf67e 100644 --- a/docs/design/tessl-5-row-expansion.md +++ b/docs/design/tessl-5-row-expansion.md @@ -255,7 +255,7 @@ Verified against [Tessl CLI reference](https://docs.tessl.io/reference/cli-comma 7. Security (slice 51) → upstream_run_ids={review_quality}; review run security; stamp tessl_run_id ``` -**Not supported by Tessl CLI**: passing `gen_id` to `eval run`. Eval always consumes on-disk scenarios. **`--workspace`** is **required** on plugin-path `scenario generate` (live CLI: `tessl scenario generate ./my-plugin --workspace acme`); Tripwire passes `TESSL_WORKSPACE`. +**Not supported by Tessl CLI**: passing `gen_id` to `eval run`. Eval always consumes on-disk scenarios. **`--workspace`** is **required** outside interactive mode for plugin-path `scenario generate` (live CLI). Tripwire resolves it via optional `TESSL_WORKSPACE` or `tessl whoami` + `tessl workspace list` (personal workspace is usually the username). ### ID carry-forward contract (MUST — slices 47–51) diff --git a/docs/user-guide/env-vars.md b/docs/user-guide/env-vars.md index 2556018..d86dd6f 100644 --- a/docs/user-guide/env-vars.md +++ b/docs/user-guide/env-vars.md @@ -8,12 +8,12 @@ Start here: [QUICKSTART](../../QUICKSTART.md) · Hub: [docs/README](../README.md > **Minimum Viable Live:** Supabase + Modal keys only. Add Snyk / Tessl / Cisco > when you want full scanner coverage. Missing Snyk/Cisco keys report -> `skipped_missing_credential`. Missing `TESSL_TOKEN` or `TESSL_WORKSPACE` still -> runs Lint (auth-free) and marks Review (Quality) `needs_setup`; missing -> `TESSL_TOKEN` or `TESSL_WORKSPACE` also marks Scenario Generation `needs_setup`; -> Eval stays `blocked` until Scenario Gen completes (then `needs_setup` if -> token/workspace or `tessl.json` project link cannot be established) — not a -> complete “all clear.” +> `skipped_missing_credential`. Missing `TESSL_TOKEN` still runs Lint (auth-free) +> and marks Review (Quality) / Scenario Generation `needs_setup`. `TESSL_WORKSPACE` +> is optional — when unset, Tripwire resolves workspace via `tessl whoami` + +> `tessl workspace list` (usually the Tessl username). Eval stays `blocked` until +> Scenario Gen completes (then `needs_setup` if token or workspace/project link +> cannot be established) — not a complete “all clear.” > > For full Live coverage, provision all five vendors before `cp .env.example .env`. > @@ -80,7 +80,7 @@ not a second environment-variable schema. | `MCP_SCANNER_LLM_BASE_URL` | Custom MCP LLM endpoint | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | | `MCP_SCANNER_LLM_API_VERSION` | Azure-style APIs | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | | `TESSL_TOKEN` | Tessl Review / Scenario Gen / Eval on Modal/CI (Lint is auth-free) | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | -| `TESSL_WORKSPACE` | Tessl workspace for `tessl review run … --workspace`, `scenario generate … --workspace`, and Eval project create (Review/Scenario/Eval `needs_setup` if absent) | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | +| `TESSL_WORKSPACE` | Optional override for Tessl `--workspace` (Review / Scenario Gen / project create). When unset, resolved via `tessl whoami` + `workspace list` (usually the username) | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | ## Tier C — Full depth (paid Cisco AI Defense) @@ -162,7 +162,7 @@ vendor to the keys you need in `.env` and where to get them. | **Supabase** (platform — MVP) | `SUPABASE_URL`, `SUPABASE_ANON_KEY`, `SUPABASE_SERVICE_ROLE_KEY`, `SUPABASE_DB_URL` | [supabase-setup](./supabase-setup.md) | | **Modal** (platform — MVP) | `MODAL_TOKEN_ID`, `MODAL_TOKEN_SECRET` (blank if interactive only) | [modal-setup](./modal-setup.md) | | **Snyk** (scanner) | `SNYK_TOKEN` | [app.snyk.io](https://app.snyk.io) → Settings → API Tokens | -| **Tessl** (scanner) | `TESSL_TOKEN`, `TESSL_WORKSPACE` | [tessl.io](https://tessl.io) → workspace → API key. Lint (`tessl skill lint`) is auth-free; token **and** workspace gate Review (Quality) (`tessl review run quality --json --workspace`; `needs_setup` if either is absent). Scenario Generation requires `TESSL_TOKEN`, `TESSL_WORKSPACE` (`scenario generate … --workspace`), and `.tessl-plugin/plugin.json`. Eval (slice 50) auto-chains after Scenario Gen when `evals/` is populated; requires token, workspace, and a linked Tessl project (`tessl.json` — adapter runs `project create` / `project repair`) — see [tessl-5-row-expansion](../design/tessl-5-row-expansion.md). | +| **Tessl** (scanner) | `TESSL_TOKEN` (required); `TESSL_WORKSPACE` (optional override) | [tessl.io](https://tessl.io) → API key. Lint (`tessl skill lint`) is auth-free. Token gates Review / Scenario / Eval. Workspace for `--workspace` comes from `TESSL_WORKSPACE` or `tessl whoami` + `workspace list` (personal workspace ≈ username). Scenario Gen also needs `.tessl-plugin/plugin.json`. Eval auto-chains after Scenario Gen when `evals/` is populated; needs a linked Tessl project (`tessl.json` — adapter runs `project create` / `repair`) — see [tessl-5-row-expansion](../design/tessl-5-row-expansion.md). | | **Cisco Skill / MCP LLM** (scanner Tier B) | `SKILL_SCANNER_LLM_API_KEY`, `SKILL_SCANNER_LLM_MODEL`, `SKILL_SCANNER_LLM_PROVIDER`, `SKILL_SCANNER_LLM_BASE_URL`; `MCP_SCANNER_LLM_API_KEY`, `MCP_SCANNER_LLM_MODEL`, `MCP_SCANNER_LLM_BASE_URL` | Any OpenAI-compatible or Azure LLM — not the same as AI Defense cloud keys below | | **Cisco AI Defense** (scanner Tier C) | `AI_DEFENSE_API_KEY`, `MCP_SCANNER_API_KEY`; optional `AI_DEFENSE_API_URL`, `MCP_SCANNER_ENDPOINT` | [developer.cisco.com](https://developer.cisco.com) → AI Defense | | **Ossprey** (malware — access OPEN/pending) | `OSSPREY_API_KEY` (`ospy_…`) | Access not yet available — leave blank; adapter reports `skipped_missing_credential` | diff --git a/sandbox/scanners.py b/sandbox/scanners.py index db694e6..c46bdad 100644 --- a/sandbox/scanners.py +++ b/sandbox/scanners.py @@ -641,6 +641,83 @@ def _parse_tessl_run_id(parsed) -> str | None: return None +def _parse_tessl_whoami_username(parsed) -> str | None: + """Extract username from ``tessl whoami --json`` (usually the personal workspace).""" + if not isinstance(parsed, dict): + return None + user = parsed.get("user") + if isinstance(user, dict): + username = user.get("username") + if isinstance(username, str) and username.strip(): + return username.strip() + username = parsed.get("username") + if isinstance(username, str) and username.strip(): + return username.strip() + return None + + +def _parse_tessl_workspace_list(parsed) -> list[dict]: + """Normalize ``tessl workspace list --json`` to a list of workspace dicts.""" + if isinstance(parsed, list): + return [ws for ws in parsed if isinstance(ws, dict)] + if isinstance(parsed, dict): + workspaces = parsed.get("workspaces") + if isinstance(workspaces, list): + return [ws for ws in workspaces if isinstance(ws, dict)] + return [] + + +def _pick_tessl_workspace(workspaces: list[dict], username: str | None) -> str | None: + """Prefer username-named workspace, then one with scenario/review actions, else first.""" + named: list[dict] = [] + for ws in workspaces: + name = ws.get("name") + if isinstance(name, str) and name.strip(): + named.append(ws) + if not named: + return None + if username: + for ws in named: + if ws["name"].strip() == username: + return username + for action in ("generate_eval_scenarios", "run_review"): + for ws in named: + actions = ws.get("allowedActions") or [] + if isinstance(actions, list) and action in actions: + return ws["name"].strip() + return named[0]["name"].strip() + + +def _resolve_tessl_workspace() -> tuple[str | None, str]: + """Resolve Tessl workspace for ``--workspace``. + + Prefer optional ``TESSL_WORKSPACE`` override; otherwise query + ``tessl whoami --json`` + ``tessl workspace list --json`` (personal workspace + is usually the authenticated username). + """ + env_ws = (os.environ.get("TESSL_WORKSPACE") or "").strip() + if env_ws: + return env_ws, "" + + username = None + who_code, who_out, who_err = _run(["npx", "--yes", "tessl@latest", "whoami", "--json"]) + if who_code == 0: + username = _parse_tessl_whoami_username(_safe_json(who_out)) + + list_code, list_out, list_err = _run( + ["npx", "--yes", "tessl@latest", "workspace", "list", "--json"] + ) + if list_code != 0: + detail = (list_err or list_out or who_err or who_out or "workspace list failed").strip() + return None, detail[:4000] + + workspaces = _parse_tessl_workspace_list(_safe_json(list_out)) + picked = _pick_tessl_workspace(workspaces, username) + if not picked: + return None, "no Tessl workspaces available for this account" + return picked, "" + + def _tessl_review_run_argv(judge_type: str, workdir: str, workspace: str, force: bool) -> list[str]: argv = [ "npx", @@ -875,7 +952,9 @@ def _scenario_checkpoint_row( return row -def _scenario_gen_preflight(workdir: str, upstream_run_ids: dict) -> dict | None: +def _scenario_gen_preflight( + workdir: str, upstream_run_ids: dict, workspace: str | None +) -> dict | None: """Return an early terminal row when token/workspace/plugin prerequisites fail.""" if not os.environ.get("TESSL_TOKEN"): skipped: dict = _skipped( @@ -883,11 +962,11 @@ def _scenario_gen_preflight(workdir: str, upstream_run_ids: dict) -> dict | None ) skipped["upstream_run_ids"] = upstream_run_ids return skipped - if not (os.environ.get("TESSL_WORKSPACE") or "").strip(): + if not workspace: skipped = _skipped( _TESSL_SCENARIO_SOURCE, reason="needs_setup", - detail="TESSL_WORKSPACE required for scenario generate --workspace", + detail="Tessl workspace unresolved — set TESSL_WORKSPACE or ensure tessl login", ) skipped["upstream_run_ids"] = upstream_run_ids return skipped @@ -912,6 +991,7 @@ def _ensure_scenario_generated( stage: str | None, consoles: list[str], on_progress, + workspace: str, ) -> tuple[str | None, dict | None]: """Ensure generation completed. Returns (gen_id, early_row_or_None).""" if stage == "generated" and gen_id: @@ -933,7 +1013,6 @@ def _ensure_scenario_generated( row["checks_run"] = count_hint return gen_id, None - workspace = (os.environ.get("TESSL_WORKSPACE") or "").strip() code, out, err = _run(_tessl_scenario_generate_argv(workdir, workspace)) console = _build_console(out, err) if console: @@ -1015,6 +1094,7 @@ def _run_tessl_scenario_gen( workdir: str, ctx: dict[str, str | None], *, + workspace: str | None = None, resume_checkpoint: dict | None = None, on_progress=None, ) -> dict: @@ -1027,7 +1107,7 @@ def _run_tessl_scenario_gen( _attach_upstream_run_ids(row, ctx, "review_quality") _emit_tessl_row_progress(on_progress, row) - gated = _scenario_gen_preflight(workdir, row["upstream_run_ids"]) + gated = _scenario_gen_preflight(workdir, row["upstream_run_ids"], workspace) if gated is not None: return gated @@ -1036,7 +1116,9 @@ def _run_tessl_scenario_gen( stage = checkpoint.get("stage") if isinstance(checkpoint.get("stage"), str) else None consoles: list[str] = [] - gen_id, early = _ensure_scenario_generated(workdir, row, gen_id, stage, consoles, on_progress) + gen_id, early = _ensure_scenario_generated( + workdir, row, gen_id, stage, consoles, on_progress, workspace or "" + ) if early is not None: return early if not gen_id: @@ -1284,6 +1366,7 @@ def _run_tessl_eval( ctx: dict[str, str | None], row: dict, *, + workspace: str | None = None, resume_eval_id: str | None = None, on_progress=None, ) -> dict: @@ -1292,10 +1375,16 @@ def _run_tessl_eval( row["status"] = "queued" _emit_tessl_row_progress(on_progress, row) - workspace = (os.environ.get("TESSL_WORKSPACE") or "").strip() - if not os.environ.get("TESSL_TOKEN") or not workspace: + if not os.environ.get("TESSL_TOKEN"): row["status"] = "needs_setup" - row["detail"] = "TESSL_TOKEN and TESSL_WORKSPACE required for eval" + row["detail"] = "TESSL_TOKEN required for eval" + _emit_tessl_row_progress(on_progress, row) + return row + if not workspace: + row["status"] = "needs_setup" + row["detail"] = ( + "Tessl workspace unresolved — set TESSL_WORKSPACE or ensure tessl login" + ) _emit_tessl_row_progress(on_progress, row) return row @@ -1394,6 +1483,7 @@ def _resolve_tessl_eval_row( eval_row: dict, scenario_row: dict, *, + workspace: str | None = None, prior_eval: dict | None = None, on_progress=None, ) -> dict: @@ -1435,13 +1525,20 @@ def _resolve_tessl_eval_row( if resume_id: return _run_tessl_eval( - workdir, ctx, eval_row, resume_eval_id=resume_id, on_progress=on_progress + workdir, + ctx, + eval_row, + workspace=workspace, + resume_eval_id=resume_id, + on_progress=on_progress, ) scenario_ok = scenario_row.get("status") == "completed" has_scenarios = _count_evals_scenarios(workdir) > 0 if scenario_ok and has_scenarios and eval_row.get("status") in {"blocked", "not_started"}: - return _run_tessl_eval(workdir, ctx, eval_row, on_progress=on_progress) + return _run_tessl_eval( + workdir, ctx, eval_row, workspace=workspace, on_progress=on_progress + ) _emit_tessl_row_progress(on_progress, eval_row) return eval_row @@ -1523,12 +1620,13 @@ def run_tessl( """Run Tessl Lint, Review Quality, Scenario Generation, then Eval auto-chain. Lint is synchronous and never requires TESSL_TOKEN; it always runs when npx - is available. Review Quality requires TESSL_TOKEN and TESSL_WORKSPACE; its - row transitions to needs_setup when either is absent. Scenario Generation - requires TESSL_TOKEN, TESSL_WORKSPACE (``--workspace`` on generate), and - ``.tessl-plugin/plugin.json``. Eval starts ``blocked`` and auto-chains after - Scenario Generation completes with scenarios in ``evals/`` (first run only; - re-runs mark prior completed Eval as ``stale``). + is available. Review Quality, Scenario Generation, and Eval need + ``TESSL_TOKEN``. Workspace for ``--workspace`` comes from optional + ``TESSL_WORKSPACE`` or is resolved via ``tessl whoami`` + ``workspace list`` + (personal workspace is usually the authenticated username). Scenario + Generation also needs ``.tessl-plugin/plugin.json``. Eval starts ``blocked`` + and auto-chains after Scenario Generation completes with scenarios in + ``evals/`` (first run only; re-runs mark prior completed Eval as ``stale``). Returns (quality_score, [lint_row, review_row, scenario_row, eval_row]). quality_score is None when Review Quality did not complete successfully. @@ -1569,25 +1667,39 @@ def run_tessl( lint_row["console_output"] = console rows.append(lint_row) - # --- Tessl: Review (Quality) (TESSL_TOKEN + TESSL_WORKSPACE required) --- + # --- Tessl: Review (Quality) (TESSL_TOKEN + resolved workspace) --- score = None - if not os.environ.get("TESSL_TOKEN") or not (os.environ.get("TESSL_WORKSPACE") or "").strip(): + workspace: str | None = None + workspace_detail = "" + if not os.environ.get("TESSL_TOKEN"): rows.append(_skipped("Tessl: Review (Quality)", reason="needs_setup")) _update_tessl_id_context(ctx, "review_quality", None) else: - workspace = os.environ["TESSL_WORKSPACE"].strip() - score, review_row = _run_tessl_review("quality", workdir, workspace) - rows.append(review_row) - _update_tessl_id_context(ctx, "review_quality", review_row.get("tessl_run_id")) + workspace, workspace_detail = _resolve_tessl_workspace() + if not workspace: + rows.append( + _skipped( + "Tessl: Review (Quality)", + reason="needs_setup", + detail=workspace_detail + or "Tessl workspace unresolved — set TESSL_WORKSPACE or ensure tessl login", + ) + ) + _update_tessl_id_context(ctx, "review_quality", None) + else: + score, review_row = _run_tessl_review("quality", workdir, workspace) + rows.append(review_row) + _update_tessl_id_context(ctx, "review_quality", review_row.get("tessl_run_id")) # --- Tessl: Eval (blocked until Scenario Generation completes) --- eval_row = _new_blocked_eval_row() _emit_tessl_row_progress(on_row_progress, eval_row) - # --- Tessl: Scenario Generation (token + plugin manifest) --- + # --- Tessl: Scenario Generation (token + plugin manifest + workspace) --- scenario_row = _run_tessl_scenario_gen( workdir, ctx, + workspace=workspace, resume_checkpoint=resume_checkpoint, on_progress=on_row_progress, ) @@ -1598,6 +1710,7 @@ def run_tessl( ctx, eval_row, scenario_row, + workspace=workspace, prior_eval=prior_eval, on_progress=on_row_progress, ) diff --git a/sandbox/tests/test_scanners_status.py b/sandbox/tests/test_scanners_status.py index 9af74f2..37437c1 100644 --- a/sandbox/tests/test_scanners_status.py +++ b/sandbox/tests/test_scanners_status.py @@ -14,6 +14,7 @@ from __future__ import annotations +import json import os from unittest.mock import patch @@ -802,19 +803,21 @@ def test_run_tessl_falls_back_to_run_json_id_when_view_last_fails() -> None: def test_run_tessl_without_workspace_emits_review_needs_setup() -> None: """ - Scenario: TESSL_WORKSPACE absent — Review (Quality) and Scenario Gen are needs_setup. - Slice: 47 — GWT-47.3 needs_setup for missing review config - Slice: 49/50 — scenario generate also requires --workspace + Scenario: Workspace unresolved — Review (Quality) and Scenario Gen are needs_setup. + Slice: 47 — GWT-47.3 needs_setup when workspace cannot be resolved + Slice: 49/50 — scenario generate also needs a resolved --workspace - Given TESSL_TOKEN is set and TESSL_WORKSPACE is absent, + Given TESSL_TOKEN is set, TESSL_WORKSPACE is absent, and workspace list fails, When run_tessl is called, Then Review (Quality) and Scenario Generation are needs_setup and no review/generate runs. """ ### Given ran: list[list[str]] = [] - def _record(cmd, timeout=None): + def _record(cmd, timeout=None, cwd=None): ran.append(cmd) + if cmd[3:5] == ["whoami"] or cmd[3:5] == ["workspace", "list"]: + return 1, "", "not authenticated" return 0, "1 check", "" ### When @@ -835,6 +838,72 @@ def _record(cmd, timeout=None): assert all(c[3:5] != ["scenario", "generate"] for c in ran) +def test_run_tessl_resolves_workspace_from_whoami_when_env_unset(tmp_path) -> None: + """ + Scenario: No TESSL_WORKSPACE — resolve personal workspace via whoami + list. + Given TESSL_TOKEN only and CLI returns username-matching workspace, + When run_tessl runs Review, + Then --workspace uses the username from whoami (not an env var). + """ + ### Given + workdir = _make_tessl_plugin(tmp_path) + captured: list[list[str]] = [] + + def _run(cmd, timeout=None, cwd=None): + captured.append(cmd) + if cmd[3] == "whoami": + return 0, '{"authenticated": true, "user": {"username": "neomatrix369"}}', "" + if cmd[3:5] == ["workspace", "list"]: + return ( + 0, + json.dumps( + { + "workspaces": [ + {"name": "other", "allowedActions": ["view"]}, + { + "name": "neomatrix369", + "allowedActions": [ + "generate_eval_scenarios", + "run_review", + ], + }, + ] + } + ), + "", + ) + if cmd[3:5] in (["skill", "lint"],) or cmd[3:5] == ["review", "run"]: + return _lint_and_quality_ok(cmd, timeout) + if len(cmd) > 5 and cmd[3:5] == ["review", "view"]: + return _lint_and_quality_ok(cmd, timeout) + if cmd[3:5] == ["scenario", "generate"]: + assert cmd[cmd.index("--workspace") + 1] == "neomatrix369" + return 0, '{"id": "gen_ws", "status": "completed", "scenarioCount": 1}', "" + if cmd[3:5] == ["scenario", "download"]: + out_dir = cmd[cmd.index("-o") + 1] + os.makedirs(os.path.join(out_dir, "s1"), exist_ok=True) + return 0, "ok", "" + eval_handled = _eval_ok(cmd, timeout, cwd) + if eval_handled is not None: + return eval_handled + raise AssertionError(f"unexpected cmd: {cmd}") + + ### When + with ( + patch.dict("os.environ", {"TESSL_TOKEN": "t"}, clear=True), + patch.object(scanners, "_which", return_value="/usr/bin/npx"), + patch.object(scanners, "_run", side_effect=_run), + ): + score, rows = scanners.run_tessl(workdir) + + ### Then + assert score == 80 + review_cmds = [c for c in captured if c[3:5] == ["review", "run"]] + assert review_cmds + assert review_cmds[0][review_cmds[0].index("--workspace") + 1] == "neomatrix369" + assert rows[2]["status"] == "completed" + + def test_given_quality_review_completes_when_run_tessl_then_ctx_review_quality_is_stamped_id() -> ( None ): @@ -1399,6 +1468,14 @@ def test_parse_scenario_gen_id_and_status_from_json_shapes() -> None: assert gen_argv[3:6] == ["scenario", "generate", "/plugin"] assert gen_argv[gen_argv.index("--workspace") + 1] == "acme" assert gen_argv[gen_argv.index("--count") + 1] == "3" + assert ( + scanners._pick_tessl_workspace( + [{"name": "other"}, {"name": "neomatrix369"}], + "neomatrix369", + ) + == "neomatrix369" + ) + assert scanners._parse_tessl_whoami_username({"user": {"username": "alice"}}) == "alice" def test_given_generate_timeout_when_scenario_runs_then_interrupted_with_checkpoint( From bf657b6b512781715fe085020f9ebe42b5d1330b Mon Sep 17 00:00:00 2001 From: Mani Sarkar Date: Tue, 25 Aug 2026 01:55:11 +0100 Subject: [PATCH 05/12] test(slice-50): cover Tessl workspace resolve fallbacks Raise sandbox coverage back above 95% for whoami/list parsing and eval unmet-workspace needs_setup. --- sandbox/tests/test_scanners_status.py | 94 +++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) diff --git a/sandbox/tests/test_scanners_status.py b/sandbox/tests/test_scanners_status.py index 37437c1..db72873 100644 --- a/sandbox/tests/test_scanners_status.py +++ b/sandbox/tests/test_scanners_status.py @@ -2288,6 +2288,100 @@ def test_run_tessl_eval_without_token_returns_needs_setup(tmp_path) -> None: assert "TESSL_TOKEN" in result["detail"] +def test_run_tessl_eval_without_workspace_returns_needs_setup(tmp_path) -> None: + """ + Scenario: Eval with token but unresolved workspace is needs_setup. + Slice: 50 — workspace gate on eval helper + """ + ### Given + row = scanners._new_blocked_eval_row() + ctx = {"review_quality": "rev_1", "scenario_gen": "gen_1"} + + ### When + with patch.dict("os.environ", {"TESSL_TOKEN": "t"}, clear=True): + result = scanners._run_tessl_eval(str(tmp_path), ctx, row, workspace=None) + + ### Then + assert result["status"] == "needs_setup" + assert "workspace" in result["detail"].lower() + + +def test_resolve_tessl_workspace_helpers_cover_fallback_paths() -> None: + """ + Scenario: Workspace resolve helpers handle list shapes and action fallback. + Slice: 50 — whoami/list parsing coverage + """ + ### Given / When / Then + assert scanners._parse_tessl_whoami_username(None) is None + assert scanners._parse_tessl_whoami_username({"username": "top"}) == "top" + assert scanners._parse_tessl_whoami_username({"user": {"username": " "}}) is None + assert scanners._parse_tessl_workspace_list([{"name": "a"}, "skip"]) == [{"name": "a"}] + assert scanners._parse_tessl_workspace_list({"workspaces": [{"name": "b"}]}) == [ + {"name": "b"} + ] + assert scanners._parse_tessl_workspace_list({"workspaces": "bad"}) == [] + assert scanners._parse_tessl_workspace_list("nope") == [] + assert scanners._pick_tessl_workspace([], None) is None + assert ( + scanners._pick_tessl_workspace( + [{"name": "team", "allowedActions": ["run_review"]}], + "missing", + ) + == "team" + ) + assert ( + scanners._pick_tessl_workspace( + [{"name": "first"}, {"name": "second"}], + None, + ) + == "first" + ) + assert scanners._pick_tessl_workspace([{"name": " "}, {}], "x") is None + + with patch.dict("os.environ", {"TESSL_WORKSPACE": "from-env"}, clear=True): + assert scanners._resolve_tessl_workspace() == ("from-env", "") + + def _run_empty(cmd, timeout=None, cwd=None): + if cmd[3] == "whoami": + return 0, '{"authenticated": true}', "" + return 0, '{"workspaces": []}', "" + + with ( + patch.dict("os.environ", {}, clear=True), + patch.object(scanners, "_run", side_effect=_run_empty), + ): + ws, detail = scanners._resolve_tessl_workspace() + assert ws is None + assert "no Tessl workspaces" in detail + + def _run_action_pick(cmd, timeout=None, cwd=None): + if cmd[3] == "whoami": + return 1, "", "whoami failed" + return ( + 0, + json.dumps( + { + "workspaces": [ + {"name": "view-only", "allowedActions": ["view"]}, + { + "name": "publisher", + "allowedActions": ["generate_eval_scenarios"], + }, + ] + } + ), + "", + ) + + with ( + patch.dict("os.environ", {}, clear=True), + patch.object(scanners, "_run", side_effect=_run_action_pick), + ): + ws, detail = scanners._resolve_tessl_workspace() + assert ws == "publisher" + assert detail == "" + + def test_ensure_tessl_project_create_timeout_returns_false(tmp_path) -> None: """ Scenario: project create timeout yields actionable failure detail. From 162cabdf1b5321cddde86dbb9e51962afe269f7b Mon Sep 17 00:00:00 2001 From: Mani Sarkar Date: Tue, 25 Aug 2026 01:57:45 +0100 Subject: [PATCH 06/12] feat(dashboard): make Findings section collapsible with rotating chevron Lets operators collapse long finding lists in the detail panel; larger chevron rotates on toggle so expand state is obvious. --- prototypes/dc-dashboard/Tripwire.dc.html | 72 +++++++++++++++++------- 1 file changed, 53 insertions(+), 19 deletions(-) diff --git a/prototypes/dc-dashboard/Tripwire.dc.html b/prototypes/dc-dashboard/Tripwire.dc.html index b0c2c59..35fd092 100644 --- a/prototypes/dc-dashboard/Tripwire.dc.html +++ b/prototypes/dc-dashboard/Tripwire.dc.html @@ -73,6 +73,26 @@ 0%, 100% { opacity: 1; } 50% { opacity: 0.4; } } + .findings-chevron { + display: inline-flex; + align-items: center; + justify-content: center; + flex-shrink: 0; + width: 28px; + height: 28px; + font-size: 18px; + font-weight: 700; + line-height: 1; + color: var(--cta-ink); + transition: transform 180ms ease, color 150ms ease; + transform-origin: 50% 55%; + } + .findings-toggle:hover .findings-chevron { + color: var(--text-primary); + } + @media (prefers-reduced-motion: reduce) { + .findings-chevron { transition: none; } + } @media (prefers-reduced-motion: reduce) { *, *::before, *::after { animation-duration: 0.01ms !important; @@ -808,24 +828,33 @@

{{ selectedView.findingsHeadingLabel }} - -
-
- {{ f.category }}{{ f.scanner }} -
-
{{ f.message }}
- -
tool: {{ f.entity_name }}
- - -
{{ f.file_path }}:{{ f.location }}
- - -
{{ f.snippet }}
- +
+
+ +
{{ selectedView.findingsHeadingLabel }}
- + +
+ +
+
+ {{ f.category }}{{ f.scanner }} +
+
{{ f.message }}
+ +
tool: {{ f.entity_name }}
+ + +
{{ f.file_path }}:{{ f.location }}
+ + +
{{ f.snippet }}
+ +
+ +
+ +
Sandbox evidence
@@ -1004,6 +1033,7 @@

{ const mode = e.target.value; sessionStorage.setItem('tripwire-data-source-mode', mode); - this.setState({ dataSourceMode: mode, selectedId: null, expandedScanners: {} }); + this.setState({ dataSourceMode: mode, selectedId: null, expandedScanners: {}, findingsExpanded: true }); this._loadDataForMode(mode); }; @@ -1131,7 +1161,7 @@

this.setState(s => ({ findingsExpanded: !s.findingsExpanded })); simulateGuardCall = (id) => { this.setState(s => { @@ -1667,6 +1698,9 @@

0 if scenario_ok and has_scenarios and eval_row.get("status") in {"blocked", "not_started"}: - return _run_tessl_eval( - workdir, ctx, eval_row, workspace=workspace, on_progress=on_progress - ) + return _run_tessl_eval(workdir, ctx, eval_row, workspace=workspace, on_progress=on_progress) _emit_tessl_row_progress(on_progress, eval_row) return eval_row diff --git a/sandbox/tests/test_scanners_status.py b/sandbox/tests/test_scanners_status.py index db72873..0c06cbb 100644 --- a/sandbox/tests/test_scanners_status.py +++ b/sandbox/tests/test_scanners_status.py @@ -2316,9 +2316,7 @@ def test_resolve_tessl_workspace_helpers_cover_fallback_paths() -> None: assert scanners._parse_tessl_whoami_username({"username": "top"}) == "top" assert scanners._parse_tessl_whoami_username({"user": {"username": " "}}) is None assert scanners._parse_tessl_workspace_list([{"name": "a"}, "skip"]) == [{"name": "a"}] - assert scanners._parse_tessl_workspace_list({"workspaces": [{"name": "b"}]}) == [ - {"name": "b"} - ] + assert scanners._parse_tessl_workspace_list({"workspaces": [{"name": "b"}]}) == [{"name": "b"}] assert scanners._parse_tessl_workspace_list({"workspaces": "bad"}) == [] assert scanners._parse_tessl_workspace_list("nope") == [] assert scanners._pick_tessl_workspace([], None) is None From fc595e36e15a6c4e820ba9252f0df6e0c50a1a8d Mon Sep 17 00:00:00 2001 From: Mani Sarkar Date: Tue, 25 Aug 2026 02:17:28 +0100 Subject: [PATCH 09/12] fix(slice-50): ignore invalid TESSL_WORKSPACE and annotate not-found errors Stale Modal overrides caused Workspace not found; validate against workspace list and surface user=/workspace= on identity failures. --- docs/user-guide/env-vars.md | 7 +- sandbox/scanners.py | 159 ++++++++++++++++++++--- sandbox/tests/test_scanners_status.py | 149 ++++++++++++++++++++- sandbox/tests/test_ship_path_coverage.py | 5 + 4 files changed, 298 insertions(+), 22 deletions(-) diff --git a/docs/user-guide/env-vars.md b/docs/user-guide/env-vars.md index d86dd6f..2174e4f 100644 --- a/docs/user-guide/env-vars.md +++ b/docs/user-guide/env-vars.md @@ -10,8 +10,9 @@ Start here: [QUICKSTART](../../QUICKSTART.md) · Hub: [docs/README](../README.md > when you want full scanner coverage. Missing Snyk/Cisco keys report > `skipped_missing_credential`. Missing `TESSL_TOKEN` still runs Lint (auth-free) > and marks Review (Quality) / Scenario Generation `needs_setup`. `TESSL_WORKSPACE` -> is optional — when unset, Tripwire resolves workspace via `tessl whoami` + -> `tessl workspace list` (usually the Tessl username). Eval stays `blocked` until +> is optional — when unset (or set to a name not in `tessl workspace list`), +> Tripwire resolves workspace via `tessl whoami` + `tessl workspace list` +> (usually the Tessl username). Eval stays `blocked` until > Scenario Gen completes (then `needs_setup` if token or workspace/project link > cannot be established) — not a complete “all clear.” > @@ -80,7 +81,7 @@ not a second environment-variable schema. | `MCP_SCANNER_LLM_BASE_URL` | Custom MCP LLM endpoint | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | | `MCP_SCANNER_LLM_API_VERSION` | Azure-style APIs | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | | `TESSL_TOKEN` | Tessl Review / Scenario Gen / Eval on Modal/CI (Lint is auth-free) | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | -| `TESSL_WORKSPACE` | Optional override for Tessl `--workspace` (Review / Scenario Gen / project create). When unset, resolved via `tessl whoami` + `workspace list` (usually the username) | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | +| `TESSL_WORKSPACE` | Optional override for Tessl `--workspace` (Review / Scenario Gen / project create). Must match a workspace from `tessl workspace list` (name or id); invalid values are ignored and auto-resolved (usually the username) | [Vendor procurement quick-steps](#vendor-procurement-quick-steps) in this file | ## Tier C — Full depth (paid Cisco AI Defense) diff --git a/sandbox/scanners.py b/sandbox/scanners.py index a8a94bd..9d7146a 100644 --- a/sandbox/scanners.py +++ b/sandbox/scanners.py @@ -656,6 +656,57 @@ def _parse_tessl_whoami_username(parsed) -> str | None: return None +# Last identity seen by ``_resolve_tessl_workspace`` — used to annotate CLI errors. +_tessl_last_username: str | None = None +_tessl_last_workspace: str | None = None + +_TESSL_IDENTITY_ERROR_RE = re.compile( + r"(?:workspace|user)\s+not\s+found", + re.IGNORECASE, +) + + +def _remember_tessl_identity( + *, username: str | None = None, workspace: str | None = None +) -> None: + """Cache Tessl whoami/workspace for identity-error annotations.""" + global _tessl_last_username, _tessl_last_workspace + if username is not None: + _tessl_last_username = username or None + if workspace is not None: + _tessl_last_workspace = workspace or None + + +def _annotate_tessl_cli_detail( + detail: str, + *, + workspace: str | None = None, + username: str | None = None, +) -> str: + """Append user=/workspace= when Tessl reports user/workspace not found. + + Also prints the annotated line so Modal/container logs show which identity + was attempted (raw Tessl stderr often omits the name). + """ + text = (detail or "").strip() + if not text: + return "" + if not _TESSL_IDENTITY_ERROR_RE.search(text): + return text[:4000] + user = (username if username is not None else _tessl_last_username) or None + ws = (workspace if workspace is not None else _tessl_last_workspace) or None + bits: list[str] = [] + if user: + bits.append(f"user={user}") + if ws: + bits.append(f"workspace={ws}") + if not bits: + return text[:4000] + annotated = f"{text} ({', '.join(bits)})" + print(f"[tessl] {annotated}", flush=True) + return annotated[:4000] + + def _parse_tessl_workspace_list(parsed) -> list[dict]: """Normalize ``tessl workspace list --json`` to a list of workspace dicts.""" if isinstance(parsed, list): @@ -688,33 +739,86 @@ def _pick_tessl_workspace(workspaces: list[dict], username: str | None) -> str | return named[0]["name"].strip() +def _match_tessl_workspace(workspaces: list[dict], needle: str) -> str | None: + """Return canonical workspace name (or id) if *needle* matches a list entry name or id.""" + want = needle.strip() + if not want: + return None + for ws in workspaces: + name = ws.get("name") + wid = ws.get("id") + name_ok = isinstance(name, str) and name.strip() == want + id_ok = isinstance(wid, str) and wid.strip() == want + if name_ok or id_ok: + if isinstance(name, str) and name.strip(): + return name.strip() + if isinstance(wid, str) and wid.strip(): + return wid.strip() + return None + + def _resolve_tessl_workspace() -> tuple[str | None, str]: """Resolve Tessl workspace for ``--workspace``. - Prefer optional ``TESSL_WORKSPACE`` override; otherwise query - ``tessl whoami --json`` + ``tessl workspace list --json`` (personal workspace - is usually the authenticated username). + Optional ``TESSL_WORKSPACE`` is an override only when it matches a workspace + the authenticated account can see (name or id). Invalid overrides are ignored + so Modal secrets with stale example values (e.g. ``engteam``) do not produce + ``Workspace not found``. Otherwise resolve via ``tessl whoami`` + + ``tessl workspace list`` (personal workspace is usually the username). """ + global _tessl_last_username, _tessl_last_workspace + _tessl_last_username = None + _tessl_last_workspace = None + env_ws = (os.environ.get("TESSL_WORKSPACE") or "").strip() - if env_ws: - return env_ws, "" username = None who_code, who_out, who_err = _run(["npx", "--yes", "tessl@latest", "whoami", "--json"]) if who_code == 0: username = _parse_tessl_whoami_username(_safe_json(who_out)) + _remember_tessl_identity(username=username) list_code, list_out, list_err = _run( ["npx", "--yes", "tessl@latest", "workspace", "list", "--json"] ) if list_code != 0: + # Cannot validate membership — honour env override if present. + if env_ws: + _remember_tessl_identity(workspace=env_ws) + return env_ws, "" detail = (list_err or list_out or who_err or who_out or "workspace list failed").strip() - return None, detail[:4000] + return None, _annotate_tessl_cli_detail(detail[:4000], username=username) workspaces = _parse_tessl_workspace_list(_safe_json(list_out)) + available = [ + ws["name"].strip() + for ws in workspaces + if isinstance(ws.get("name"), str) and ws["name"].strip() + ] + + if env_ws: + matched = _match_tessl_workspace(workspaces, env_ws) + if matched: + _remember_tessl_identity(workspace=matched) + return matched, "" + print( + f"[tessl] ignoring TESSL_WORKSPACE={env_ws!r} — not in workspace list " + f"(available: {', '.join(available) or 'none'}); falling back to auto-resolve", + flush=True, + ) + picked = _pick_tessl_workspace(workspaces, username) if not picked: - return None, "no Tessl workspaces available for this account" + detail = "no Tessl workspaces available for this account" + if env_ws: + detail = ( + f"TESSL_WORKSPACE={env_ws!r} not found and no fallback workspace " + f"(available: {', '.join(available) or 'none'})" + ) + return None, _annotate_tessl_cli_detail( + detail, workspace=env_ws or None, username=username + ) + _remember_tessl_identity(workspace=picked) return picked, "" @@ -1022,7 +1126,10 @@ def _ensure_scenario_generated( early = _scenario_checkpoint_row( row, status="interrupted", - detail=(err or out or "scenario generate timed out").strip(), + detail=_annotate_tessl_cli_detail( + (err or out or "scenario generate timed out").strip(), + workspace=workspace, + ), gen_id=timed_out_id, consoles=consoles, on_progress=on_progress, @@ -1030,7 +1137,10 @@ def _ensure_scenario_generated( return timed_out_id, early if code != 0: row["status"] = "failed" - row["detail"] = (err or out or "scenario generate exited non-zero").strip()[:4000] + row["detail"] = _annotate_tessl_cli_detail( + (err or out or "scenario generate exited non-zero").strip(), + workspace=workspace, + )[:4000] return None, _scenario_row_with_console(row, consoles) captured_id = _capture_scenario_gen_id(out, gen_id) if not captured_id: @@ -1279,7 +1389,10 @@ def _ensure_tessl_project(workdir: str, workspace: str) -> tuple[bool, str]: if _has_tessl_project_link(workdir): code, out, err = _run(_tessl_project_repair_argv(), cwd=workdir) if code not in (0, None) and not _has_tessl_project_link(workdir): - return False, (err or out or "tessl project repair failed").strip() + return False, _annotate_tessl_cli_detail( + (err or out or "tessl project repair failed").strip(), + workspace=workspace, + ) return True, "" project_name = os.path.basename(os.path.abspath(workdir)) or "tripwire-scan" @@ -1287,9 +1400,15 @@ def _ensure_tessl_project(workdir: str, workspace: str) -> tuple[bool, str]: if code == 0 and _has_tessl_project_link(workdir): return True, "" if code is None: - return False, (err or out or "tessl project create timed out").strip() + return False, _annotate_tessl_cli_detail( + (err or out or "tessl project create timed out").strip(), + workspace=workspace, + ) detail = (err or out or "tessl project create failed").strip() - return False, detail or "tessl.json missing — project create/repair required before eval" + return False, _annotate_tessl_cli_detail( + detail or "tessl.json missing — project create/repair required before eval", + workspace=workspace, + ) def _eval_should_mark_stale(prior_eval: dict, scenario_row: dict) -> bool: @@ -1413,7 +1532,10 @@ def _run_tessl_eval( eval_id=eval_id, parsed=parsed_run if isinstance(parsed_run, dict) else None, consoles=consoles, - detail=(err or out or "eval run timed out").strip(), + detail=_annotate_tessl_cli_detail( + (err or out or "eval run timed out").strip(), + workspace=workspace, + ), on_progress=on_progress, ) if code != 0 and not eval_id: @@ -1423,7 +1545,10 @@ def _run_tessl_eval( eval_id=None, parsed=None, consoles=consoles, - detail=(err or out or "eval run exited non-zero").strip(), + detail=_annotate_tessl_cli_detail( + (err or out or "eval run exited non-zero").strip(), + workspace=workspace, + ), on_progress=on_progress, ) if not eval_id: @@ -1545,7 +1670,11 @@ def _finish_tessl_review( ) -> tuple[float | None, dict]: console = _build_console(out, err) if code != 0: - return None, _unreachable(source, err or out, console_output=console) + return None, _unreachable( + source, + _annotate_tessl_cli_detail(err or out, workspace=workspace), + console_output=console, + ) parsed = _safe_json(out) score = _tessl_quality_score(parsed) if judge_type == "quality" else None if judge_type == "quality" and score is None: diff --git a/sandbox/tests/test_scanners_status.py b/sandbox/tests/test_scanners_status.py index 0c06cbb..c1214bf 100644 --- a/sandbox/tests/test_scanners_status.py +++ b/sandbox/tests/test_scanners_status.py @@ -20,6 +20,27 @@ import scanners +# Satisfies whoami + workspace list when tests set TESSL_WORKSPACE=engteam. +_TESSL_WS_RESOLVE_OK = [ + (0, '{"authenticated": true, "user": {"username": "engteam"}}', ""), + ( + 0, + '{"workspaces": [{"name": "engteam", "id": "ws_eng", ' + '"allowedActions": ["generate_eval_scenarios", "run_review"]}]}', + "", + ), +] + + +def _tessl_workspace_cli_ok(cmd, timeout=None, cwd=None): + """Satisfy whoami + workspace list for tests that set TESSL_WORKSPACE=engteam.""" + del timeout, cwd + if len(cmd) > 3 and cmd[3] == "whoami": + return _TESSL_WS_RESOLVE_OK[0] + if cmd[3:5] == ["workspace", "list"]: + return _TESSL_WS_RESOLVE_OK[1] + return None + def test_given_unreachable_stderr_when_building_row_then_detail_is_capped() -> None: """ @@ -452,6 +473,7 @@ def test_run_tessl_logs_diagnostic_when_score_is_none(capsys) -> None: ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "fake-token", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object( scanners, @@ -538,6 +560,7 @@ def test_run_tessl_with_token_emits_lint_and_review_rows() -> None: ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "tok-abc", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object( scanners, @@ -707,6 +730,9 @@ def test_run_tessl_review_quality_invokes_review_run_quality() -> None: captured: list[list[str]] = [] def _capture_run(cmd, timeout=None, cwd=None): + resolved = _tessl_workspace_cli_ok(cmd, timeout, cwd) + if resolved is not None: + return resolved captured.append(cmd) if cmd[3:5] == ["skill", "lint"]: return 0, "1 check", "" @@ -722,6 +748,7 @@ def _capture_run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_capture_run), ): @@ -755,6 +782,7 @@ def test_run_tessl_captures_run_id_from_view_last_json() -> None: ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object( scanners, @@ -784,6 +812,7 @@ def test_run_tessl_falls_back_to_run_json_id_when_view_last_fails() -> None: ### Given / When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object( scanners, @@ -816,7 +845,7 @@ def test_run_tessl_without_workspace_emits_review_needs_setup() -> None: def _record(cmd, timeout=None, cwd=None): ran.append(cmd) - if cmd[3:5] == ["whoami"] or cmd[3:5] == ["workspace", "list"]: + if cmd[3] == "whoami" or cmd[3:5] == ["workspace", "list"]: return 1, "", "not authenticated" return 0, "1 check", "" @@ -925,6 +954,7 @@ def test_given_quality_review_completes_when_run_tessl_then_ctx_review_quality_i ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object( scanners, @@ -1016,6 +1046,9 @@ def _make_tessl_plugin(tmp_path) -> str: def _lint_and_quality_ok(cmd, timeout=None, cwd=None): + resolved = _tessl_workspace_cli_ok(cmd, timeout, cwd) + if resolved is not None: + return resolved del timeout, cwd if cmd[3:5] == ["skill", "lint"]: return 0, "1 check", "" @@ -1068,6 +1101,9 @@ def test_given_plugin_when_scenario_gen_succeeds_then_download_stamps_and_clears captured: list[list[str]] = [] def _run(cmd, timeout=None, cwd=None): + resolved = _tessl_workspace_cli_ok(cmd, timeout, cwd) + if resolved is not None: + return resolved captured.append(cmd) if ( cmd[3:5] in (["skill", "lint"],) @@ -1100,6 +1136,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -1139,6 +1176,9 @@ def test_given_resume_generated_when_run_tessl_then_skips_generate_and_downloads captured: list[list[str]] = [] def _run(cmd, timeout=None, cwd=None): + resolved = _tessl_workspace_cli_ok(cmd, timeout, cwd) + if resolved is not None: + return resolved captured.append(cmd) if ( cmd[3:5] in (["skill", "lint"],) @@ -1162,6 +1202,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), patch.object(scanners, "_TESSL_SCENARIO_POLL_SLEEP_S", 0), @@ -1199,6 +1240,9 @@ def test_given_in_progress_checkpoint_when_resumed_then_polls_before_download( captured: list[list[str]] = [] def _run(cmd, timeout=None, cwd=None): + resolved = _tessl_workspace_cli_ok(cmd, timeout, cwd) + if resolved is not None: + return resolved captured.append(cmd) if ( cmd[3:5] in (["skill", "lint"],) @@ -1221,6 +1265,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), patch.object(scanners, "_TESSL_SCENARIO_POLL_SLEEP_S", 0), @@ -1253,6 +1298,9 @@ def test_given_quality_id_in_ctx_when_scenario_starts_then_upstream_run_ids_atta progress: list[dict] = [] def _run(cmd, timeout=None, cwd=None): + resolved = _tessl_workspace_cli_ok(cmd, timeout, cwd) + if resolved is not None: + return resolved if cmd[3:5] == ["scenario", "generate"]: return 0, '{"id": "gen_x", "status": "completed", "scenarioCount": 1}', "" if cmd[3:5] == ["scenario", "download"]: @@ -1267,6 +1315,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -1318,6 +1367,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -1343,6 +1393,9 @@ def test_given_scenario_generate_fails_when_run_tessl_then_row_is_failed(tmp_pat captured: list[list[str]] = [] def _run(cmd, timeout=None, cwd=None): + resolved = _tessl_workspace_cli_ok(cmd, timeout, cwd) + if resolved is not None: + return resolved captured.append(cmd) if ( cmd[3:5] in (["skill", "lint"],) @@ -1351,7 +1404,7 @@ def _run(cmd, timeout=None, cwd=None): ): return _lint_and_quality_ok(cmd, timeout) if cmd[3:5] == ["scenario", "generate"]: - return 1, "", "generation exploded" + return 1, "", "Failed to generate scenarios\nWorkspace not found" eval_handled = _eval_ok(cmd, timeout, cwd) if eval_handled is not None: return eval_handled @@ -1360,6 +1413,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -1367,7 +1421,8 @@ def _run(cmd, timeout=None, cwd=None): ### Then assert rows[2]["status"] == "failed" - assert "generation exploded" in rows[2]["detail"] + assert "Workspace not found" in rows[2]["detail"] + assert "workspace=engteam" in rows[2]["detail"] assert all(c[3:5] != ["scenario", "download"] for c in captured) @@ -1408,11 +1463,15 @@ def test_given_missing_plugin_manifest_when_scenario_runs_then_failed(tmp_path) os.makedirs(workdir) def _run(cmd, timeout=None, cwd=None): + resolved = _tessl_workspace_cli_ok(cmd, timeout, cwd) + if resolved is not None: + return resolved return _lint_and_quality_ok(cmd, timeout) ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -1502,6 +1561,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -1538,6 +1598,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), patch.object(scanners, "_TESSL_SCENARIO_POLL_SLEEP_S", 0), @@ -1575,6 +1636,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -1611,6 +1673,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -1655,6 +1718,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -1707,6 +1771,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), patch.object(scanners, "_TESSL_EVAL_POLL_SLEEP_S", 0), @@ -1761,6 +1826,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -1813,6 +1879,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -1873,6 +1940,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), patch.object(scanners, "_TESSL_EVAL_POLL_SLEEP_S", 0), @@ -1918,6 +1986,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -2038,6 +2107,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -2081,6 +2151,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -2115,6 +2186,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -2148,6 +2220,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -2184,6 +2257,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), patch.object(scanners, "_TESSL_EVAL_POLL_SLEEP_S", 0), @@ -2219,6 +2293,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), ): @@ -2337,7 +2412,14 @@ def test_resolve_tessl_workspace_helpers_cover_fallback_paths() -> None: assert scanners._pick_tessl_workspace([{"name": " "}, {}], "x") is None with patch.dict("os.environ", {"TESSL_WORKSPACE": "from-env"}, clear=True): - assert scanners._resolve_tessl_workspace() == ("from-env", "") + # Env-only short-circuit removed — resolve always lists; mock list membership. + def _run_env_ok(cmd, timeout=None, cwd=None): + if cmd[3] == "whoami": + return 0, '{"user": {"username": "from-env"}}', "" + return 0, '{"workspaces": [{"name": "from-env"}]}', "" + + with patch.object(scanners, "_run", side_effect=_run_env_ok): + assert scanners._resolve_tessl_workspace() == ("from-env", "") def _run_empty(cmd, timeout=None, cwd=None): if cmd[3] == "whoami": @@ -2379,6 +2461,64 @@ def _run_action_pick(cmd, timeout=None, cwd=None): assert ws == "publisher" assert detail == "" + def _run_ignore_bad_env(cmd, timeout=None, cwd=None): + if cmd[3] == "whoami": + return 0, '{"user": {"username": "neomatrix369"}}', "" + return ( + 0, + json.dumps( + { + "workspaces": [ + { + "name": "neomatrix369", + "id": "019c-ws", + "allowedActions": ["generate_eval_scenarios"], + } + ] + } + ), + "", + ) + + with ( + patch.dict("os.environ", {"TESSL_WORKSPACE": "engteam"}, clear=True), + patch.object(scanners, "_run", side_effect=_run_ignore_bad_env), + ): + ws, detail = scanners._resolve_tessl_workspace() + assert ws == "neomatrix369" + assert detail == "" + assert ( + scanners._match_tessl_workspace([{"name": "neomatrix369", "id": "019c-ws"}], "019c-ws") + == "neomatrix369" + ) + + +def test_annotate_tessl_cli_detail_adds_user_and_workspace() -> None: + """ + Scenario: Tessl identity errors include attempted user/workspace in detail. + Slice: 50 — workspace/user not found diagnostics + """ + assert ( + scanners._annotate_tessl_cli_detail( + "Failed to generate scenarios\nWorkspace not found", + workspace="engteam", + username="neomatrix369", + ) + == "Failed to generate scenarios\nWorkspace not found " + "(user=neomatrix369, workspace=engteam)" + ) + assert ( + scanners._annotate_tessl_cli_detail( + "User not found", + workspace="acme", + username="alice", + ) + == "User not found (user=alice, workspace=acme)" + ) + assert scanners._annotate_tessl_cli_detail("generation exploded", workspace="engteam") == ( + "generation exploded" + ) + def test_ensure_tessl_project_create_timeout_returns_false(tmp_path) -> None: """ @@ -2461,6 +2601,7 @@ def _run(cmd, timeout=None, cwd=None): ### When with ( patch.dict("os.environ", {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value="/usr/bin/npx"), patch.object(scanners, "_run", side_effect=_run), patch.object(scanners, "_TESSL_EVAL_POLL_SLEEP_S", 0), diff --git a/sandbox/tests/test_ship_path_coverage.py b/sandbox/tests/test_ship_path_coverage.py index 85d0683..dec4275 100644 --- a/sandbox/tests/test_ship_path_coverage.py +++ b/sandbox/tests/test_ship_path_coverage.py @@ -812,6 +812,7 @@ def test_given_tessl_token_when_run_ok_then_quality_score() -> None: {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}, clear=False, ), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value=True), patch.object( scanners, @@ -871,6 +872,7 @@ def test_given_tessl_npx_missing_when_run_then_unreachable() -> None: {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}, clear=False, ), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value=False), ): score, rows = scanners.run_tessl("/tmp") @@ -892,6 +894,7 @@ def test_given_tessl_nonzero_when_run_then_unreachable() -> None: {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}, clear=False, ), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value=True), patch.object( scanners, "_run", side_effect=[(0, "0 checks — 0 findings", ""), (1, "", "fail")] @@ -1345,6 +1348,7 @@ def test_given_tessl_no_console_when_completed_then_no_console_key() -> None: {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}, clear=False, ), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value=True), patch.object( scanners, "_run", return_value=(0, json.dumps({"score": 1, "id": "rev_c"}), "") @@ -1374,6 +1378,7 @@ def test_given_tessl_empty_success_output_when_run_then_not_reported_completed() {"TESSL_TOKEN": "t", "TESSL_WORKSPACE": "engteam"}, clear=False, ), + patch.object(scanners, "_resolve_tessl_workspace", return_value=("engteam", "")), patch.object(scanners, "_which", return_value=True), patch.object( scanners, "_run", side_effect=[(0, "0 checks — 0 findings", ""), (0, "{}", "")] From c7378c8d8accb696f4a02d5f2d9a2d813bd9dd6b Mon Sep 17 00:00:00 2001 From: Mani Sarkar Date: Tue, 25 Aug 2026 02:27:13 +0100 Subject: [PATCH 10/12] style: ruff format Tessl identity helpers in scanners --- sandbox/scanners.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/sandbox/scanners.py b/sandbox/scanners.py index 9d7146a..7db9034 100644 --- a/sandbox/scanners.py +++ b/sandbox/scanners.py @@ -666,9 +666,7 @@ def _parse_tessl_whoami_username(parsed) -> str | None: ) -def _remember_tessl_identity( - *, username: str | None = None, workspace: str | None = None -) -> None: +def _remember_tessl_identity(*, username: str | None = None, workspace: str | None = None) -> None: """Cache Tessl whoami/workspace for identity-error annotations.""" global _tessl_last_username, _tessl_last_workspace if username is not None: @@ -815,9 +813,7 @@ def _resolve_tessl_workspace() -> tuple[str | None, str]: f"TESSL_WORKSPACE={env_ws!r} not found and no fallback workspace " f"(available: {', '.join(available) or 'none'})" ) - return None, _annotate_tessl_cli_detail( - detail, workspace=env_ws or None, username=username - ) + return None, _annotate_tessl_cli_detail(detail, workspace=env_ws or None, username=username) _remember_tessl_identity(workspace=picked) return picked, "" From 95e6067b2232a3f4123f760f1d65194475e7dd6e Mon Sep 17 00:00:00 2001 From: Mani Sarkar Date: Tue, 25 Aug 2026 02:29:39 +0100 Subject: [PATCH 11/12] fix(slice-50): satisfy mypy no-any-return in workspace pick Narrow dict name values with isinstance before returning str | None. --- sandbox/scanners.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sandbox/scanners.py b/sandbox/scanners.py index 7db9034..e473802 100644 --- a/sandbox/scanners.py +++ b/sandbox/scanners.py @@ -733,8 +733,13 @@ def _pick_tessl_workspace(workspaces: list[dict], username: str | None) -> str | for ws in named: actions = ws.get("allowedActions") or [] if isinstance(actions, list) and action in actions: - return ws["name"].strip() - return named[0]["name"].strip() + name = ws.get("name") + if isinstance(name, str) and name.strip(): + return name.strip() + first = named[0].get("name") + if isinstance(first, str) and first.strip(): + return first.strip() + return None def _match_tessl_workspace(workspaces: list[dict], needle: str) -> str | None: From 80c52dcddc2ff8d3d5a21effede3d36af5ce67be Mon Sep 17 00:00:00 2001 From: Mani Sarkar Date: Tue, 25 Aug 2026 02:31:43 +0100 Subject: [PATCH 12/12] fix(slice-50): keep mypy clean without dropping sandbox coverage Use str() for picked workspace names and cover annotate/match edge cases. --- sandbox/scanners.py | 10 +++------- sandbox/tests/test_scanners_status.py | 6 ++++++ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/sandbox/scanners.py b/sandbox/scanners.py index e473802..2143e6a 100644 --- a/sandbox/scanners.py +++ b/sandbox/scanners.py @@ -733,13 +733,9 @@ def _pick_tessl_workspace(workspaces: list[dict], username: str | None) -> str | for ws in named: actions = ws.get("allowedActions") or [] if isinstance(actions, list) and action in actions: - name = ws.get("name") - if isinstance(name, str) and name.strip(): - return name.strip() - first = named[0].get("name") - if isinstance(first, str) and first.strip(): - return first.strip() - return None + # named entries always carry a non-empty str name (filtered above). + return str(ws["name"]).strip() + return str(named[0]["name"]).strip() def _match_tessl_workspace(workspaces: list[dict], needle: str) -> str | None: diff --git a/sandbox/tests/test_scanners_status.py b/sandbox/tests/test_scanners_status.py index c1214bf..0ec59d1 100644 --- a/sandbox/tests/test_scanners_status.py +++ b/sandbox/tests/test_scanners_status.py @@ -2518,6 +2518,12 @@ def test_annotate_tessl_cli_detail_adds_user_and_workspace() -> None: assert scanners._annotate_tessl_cli_detail("generation exploded", workspace="engteam") == ( "generation exploded" ) + assert scanners._annotate_tessl_cli_detail("") == "" + scanners._tessl_last_username = None + scanners._tessl_last_workspace = None + assert scanners._annotate_tessl_cli_detail("Workspace not found") == "Workspace not found" + assert scanners._match_tessl_workspace([{"name": "a"}], " ") is None + assert scanners._match_tessl_workspace([{"id": "id-1"}], "id-1") == "id-1" def test_ensure_tessl_project_create_timeout_returns_false(tmp_path) -> None: